diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e3ad3019e..07d7281b6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,8 @@ variables: # target test config file, used by assign test job CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml" + # Versioned esp-idf-doc env image to use for all document building jobs + ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v3" # before each job, we need to check if this job is filtered by bot stage/job filter diff --git a/docs/build_docs.py b/docs/build_docs.py index c0a6263a45..d2dfb1d25c 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -9,6 +9,20 @@ # # Specific custom docs functionality should be added in conf_common.py or in a Sphinx extension, not here. # +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from __future__ import print_function import argparse import locale diff --git a/docs/conf_common.py b/docs/conf_common.py index adc6e276d0..4b0964fa2e 100644 --- a/docs/conf_common.py +++ b/docs/conf_common.py @@ -21,6 +21,7 @@ import os import os.path import re import subprocess +from sanitize_version import sanitize_version from idf_extensions.util import download_file_if_missing # build_docs on the CI server sometimes fails under Python3. This is a workaround: @@ -107,17 +108,7 @@ version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8') # The 'release' version is the same as version for non-CI builds, but for CI # builds on a branch then it's replaced with the branch name -try: - # default to using the Gitlab CI branch or tag if one is present - release = os.environ['CI_COMMIT_REF_NAME'] - - # emulate RTD's naming scheme for branches, master->latest, etc - release = release.replace('/', '-') - if release == "master": - release = "latest" -except KeyError: - # otherwise, fall back to the full git version (no branch info) - release = version +release = sanitize_version(version) print('Version: {0} Release: {1}'.format(version, release)) diff --git a/docs/sanitize_version.py b/docs/sanitize_version.py new file mode 100644 index 0000000000..90e9101678 --- /dev/null +++ b/docs/sanitize_version.py @@ -0,0 +1,43 @@ +# Tiny Python module to sanitize a Git version into something that can be used in a URL +# +# (this is used in multiple places: conf_common.py and in tools/ci/docs_deploy +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + + +def sanitize_version(original_version): + """ Given a version (probably output from 'git describe --always' or similar), return + a URL-safe sanitized version. (this is used as 'release' config variable when building + the docs.) + + Will override the original version with the Gitlab CI CI_COMMIT_REF_NAME environment variable if + this is present. + + Also follows the RTD-ism that master branch is named 'latest' + + """ + + try: + version = os.environ['CI_COMMIT_REF_NAME'] + except KeyError: + version = original_version + + if version == "master": + return "latest" + + version = version.replace('/', '-') + + return version diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 028e8a7928..8de0b0f9e6 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -232,7 +232,7 @@ build_test_apps_esp32s2: .build_docs_template: &build_docs_template stage: build - image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2 + image: $ESP_IDF_DOC_ENV_IMAGE tags: - build_docs artifacts: diff --git a/tools/ci/config/deploy.yml b/tools/ci/config/deploy.yml index dfbee61449..16be520556 100644 --- a/tools/ci/config/deploy.yml +++ b/tools/ci/config/deploy.yml @@ -70,63 +70,32 @@ push_to_github: - git remote add github git@github.com:espressif/esp-idf.git - tools/ci/push_to_github.sh -.upload_doc_archive: &upload_doc_archive | - cd docs/_build/$DOCLANG/$DOCTGT - mv html $DOCTGT - tar czf $DOCTGT.tar.gz $DOCTGT - # arrange for URL format language/version/target - ssh $DOCS_SERVER -x "mkdir -p $DOCS_PATH/$DOCLANG/$GIT_VER" - scp $DOCTGT.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$GIT_VER" - ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$GIT_VER && tar xzf ${DOCTGT}.tar.gz && rm -f ../latest && ln -s . ../latest" - cd - - -deploy_docs: +.deploy_docs_template: + extends: .before_script_lesser stage: deploy - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + image: $ESP_IDF_DOC_ENV_IMAGE tags: - deploy - shiny - only: - refs: - - master - - /^release\/v/ - - /^v\d+\.\d+(\.\d+)?($|-)/ - - triggers - variables: - - $BOT_TRIGGER_WITH_LABEL == null - - $BOT_LABEL_BUILD_DOCS dependencies: - build_docs_en_esp32 - build_docs_en_esp32s2 - build_docs_zh_CN_esp32 - build_docs_zh_CN_esp32s2 - extends: .before_script_lesser + variables: + DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" + PYTHONUNBUFFERED: 1 script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh - - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64 + - echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config + - echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config - export GIT_VER=$(git describe --always) - - DOCLANG=en; DOCTGT=esp32 - - *upload_doc_archive + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py - - DOCLANG=en; DOCTGT=esp32s2 - - *upload_doc_archive - - - DOCLANG=zh_CN; DOCTGT=esp32 - - *upload_doc_archive - - - DOCLANG=zh_CN; DOCTGT=esp32s2 - - *upload_doc_archive - - # log link to doc URLs - - echo "[document $TYPE][en][esp32] $DOCS_URL_BASE/en/${GIT_VER}/esp32/index.html" - - echo "[document $TYPE][en][esp32s2] $DOCS_URL_BASE/en/${GIT_VER}/esp32s2/index.html" - - echo "[document $TYPE][zh_CN][esp32] $DOCS_URL_BASE/zh_CN/${GIT_VER}/esp32/index.html" - - echo "[document $TYPE][zh_CN][esp32s2] $DOCS_URL_BASE/zh_CN/${GIT_VER}/esp32s2/index.html" # deploys docs to CI_DOCKER_REGISTRY webserver, for internal review deploy_docs_preview: @@ -139,26 +108,29 @@ deploy_docs_preview: - $BOT_LABEL_BUILD_DOCS variables: TYPE: "preview" - # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_PATH for preview server so we keep them - DOCS_DEPLOY_KEY: "$DOCS_DEPLOY_KEY" - DOCS_SERVER: "$DOCS_SERVER" - DOCS_PATH: "$DOCS_PATH" - DOCS_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf" + # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview' + DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY" + DOCS_DEPLOY_SERVER: "$DOCS_SERVER" + DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER" + DOCS_DEPLOY_PATH: "$DOCS_PATH" + DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf" # deploy docs to production webserver deploy_docs_production: extends: .deploy_docs_template only: refs: + # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ variables: TYPE: "preview" - DOCS_DEPLOY_KEY: "WIP" - DOCS_SERVER: "WIP" - DOCS_PATH: "WIP" - DOCS_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf" + DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY" + DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER" + DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER" + DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH" + DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf" deploy_test_result: stage: deploy diff --git a/tools/ci/config/post_deploy.yml b/tools/ci/config/post_deploy.yml index ccb3fc198a..782be335d1 100644 --- a/tools/ci/config/post_deploy.yml +++ b/tools/ci/config/post_deploy.yml @@ -1,6 +1,6 @@ .check_doc_links_template: &check_doc_links_template stage: post_deploy - image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2 + image: $ESP_IDF_DOC_ENV_IMAGE tags: [ "build", "amd64", "internet" ] only: - master diff --git a/tools/ci/deploy_docs.py b/tools/ci/deploy_docs.py new file mode 100755 index 0000000000..207af7a733 --- /dev/null +++ b/tools/ci/deploy_docs.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# +# CI script to deploy docs to a webserver. Not useful outside of CI environment +# +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import glob +import os +import os.path +import re +import stat +import sys +import subprocess +import tarfile +import packaging.version + + +def env(variable, default=None): + """ Shortcut to return the expanded version of an environment variable """ + return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable]) + + +# import sanitize_version from the docs directory, shared with here +sys.path.append(os.path.join(env("IDF_PATH"), "docs")) +from sanitize_version import sanitize_version # noqa + + +def main(): + # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI + git_ver = env("GIT_VER") # output of git describe --always + ci_ver = env("CI_COMMIT_REF_NAME", git_ver) # branch or tag we're building for (used for 'release' & URL) + + version = sanitize_version(ci_ver) + print("Git version: {}".format(git_ver)) + print("CI Version: {}".format(ci_ver)) + print("Deployment version: {}".format(version)) + + if not version: + raise RuntimeError("A version is needed to deploy") + + build_dir = env("DOCS_BUILD_DIR") # top-level local build dir, where docs have already been built + + if not build_dir: + raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy") + + url_base = env("DOCS_DEPLOY_URL_BASE") # base for HTTP URLs, used to print the URL to the log after deploying + + docs_server = env("DOCS_DEPLOY_SERVER") # ssh server to deploy to + docs_user = env("DOCS_DEPLOY_SERVER_USER") + docs_path = env("DOCS_DEPLOY_PATH") # filesystem path on DOCS_SERVER + + if not docs_server: + raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy") + + if not docs_user: + raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy") + + docs_server = "{}@{}".format(docs_user, docs_server) + + if not docs_path: + raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy") + + print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path)) + + tarball_path, version_urls = build_doc_tarball(version, build_dir) + + deploy(version, tarball_path, docs_path, docs_server) + + print("Docs URLs:") + for vurl in version_urls: + url = "{}/{}/index.html".format(url_base, vurl) # (index.html needed for the preview server) + url = re.sub(r"([^:])//", r"\1/", url) # get rid of any // that isn't in the https:// part + print(url) + + # note: it would be neater to use symlinks for stable, but because of the directory order + # (language first) it's kind of a pain to do on a remote server, so we just repeat the + # process but call the version 'stable' this time + if is_stable_version(version): + print("Deploying again as stable version...") + tarball_path, version_urls = build_doc_tarball("stable", build_dir) + deploy("stable", tarball_path, docs_path, docs_server) + + +def deploy(version, tarball_path, docs_path, docs_server): + + def run_ssh(commands): + """ Log into docs_server and run a sequence of commands using ssh """ + print("Running ssh: {}".format(commands)) + subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True) + + # copy the version tarball to the server + run_ssh(["mkdir -p {}".format(docs_path)]) + print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path))) + subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True) + + tarball_name = os.path.basename(tarball_path) + + run_ssh(["cd {}".format(docs_path), + "rm -rf ./*/{}".format(version), # remove any pre-existing docs matching this version + "tar -zxvf {}".format(tarball_name), # untar the archive with the new docs + "rm {}".format(tarball_name)]) + + # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc + # as the version will be invalid for a window of time. Better to do it atomically, but this is + # another thing made much more complex by the directory structure putting language before version... + + +def build_doc_tarball(version, build_dir): + """ Make a tar.gz archive of the docs, in the directory structure used to deploy as + the given version """ + version_paths = [] + tarball_path = "{}/{}.tar.gz".format(build_dir, version) + + # find all the 'html/' directories under build_dir + html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True) + print("Found %d html directories" % len(html_dirs)) + + def not_sources_dir(ti): + """ Filter the _sources directories out of the tarballs """ + if ti.name.endswith("/_sources"): + return None + + ti.mode |= stat.S_IWGRP # make everything group-writeable + return ti + + try: + os.remove(tarball_path) + except OSError: + pass + + with tarfile.open(tarball_path, "w:gz") as tarball: + for html_dir in html_dirs: + # html_dir has the form '///html/' + target_dirname = os.path.dirname(os.path.dirname(html_dir)) + target = os.path.basename(target_dirname) + language = os.path.basename(os.path.dirname(target_dirname)) + + # when deploying, we want the top-level directory layout 'language/version/target' + archive_path = "{}/{}/{}".format(language, version, target) + print("Archiving '{}' as '{}'...".format(html_dir, archive_path)) + tarball.add(html_dir, archive_path, filter=not_sources_dir) + version_paths.append(archive_path) + + return (os.path.abspath(tarball_path), version_paths) + + +def is_stable_version(version): + """ Heuristic for whether this is the latest stable release """ + if not version.startswith("v"): + return False # branch name + if "-" in version: + return False # prerelease tag + + git_out = subprocess.run(["git", "tag", "-l"], capture_output=True, check=True) + + versions = [v.strip() for v in git_out.stdout.decode("utf-8").split("\n")] + versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)] # include vX.Y.Z only + + versions = [packaging.version.parse(v) for v in versions] + + max_version = max(versions) + + if max_version.public != version[1:]: + print("Stable version is v{}. This version is {}.".format(max_version.public, version)) + return False + else: + print("This version {} is the stable version".format(version)) + return True + + +if __name__ == "__main__": + main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 11a80cc18a..c0ca2c4ba0 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -45,6 +45,7 @@ tools/ci/check_idf_version.sh tools/ci/check_public_headers.sh tools/ci/check_ut_cmake_make.sh tools/ci/checkout_project_ref.py +tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/fix_empty_prototypes.sh tools/ci/get-full-sources.sh