docs: self host docs

Move doc hosting from readthedocs to espressif servers

Update CI, Sphinx configs and add IDF Sphinx theme
customer/11kv_v4.0_xiaomi
Marius Vikhammer 2020-08-28 15:29:37 +08:00
rodzic a6bddd68d2
commit 8e7e0973db
20 zmienionych plików z 1494 dodań i 118 usunięć

Wyświetl plik

@ -60,6 +60,9 @@ variables:
CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v3.1"
# Versioned esp-idf-doc env image to use for all document building jobs
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7"
# before each job, we need to check if this job is filtered by bot stage/job filter

Wyświetl plik

@ -266,6 +266,7 @@ MACRO_EXPANSION = YES
EXPAND_ONLY_PREDEF = YES
PREDEFINED = \
__attribute__(x)= \
_Static_assert()= \
IDF_DEPRECATED(X)= \
IRAM_ATTR= \
configSUPPORT_DYNAMIC_ALLOCATION=1 \

785
docs/_static/espressif2.pdf vendored 100644

File diff suppressed because one or more lines are too long

Wyświetl plik

@ -20,16 +20,21 @@ import sys
import os
import re
import subprocess
from sanitize_version import sanitize_version
from get_github_rev import get_github_rev
# Note: If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute
from local_util import run_cmd_get_output, copy_if_modified
from local_util import copy_if_modified
# build_docs on the CI server sometimes fails under Python3. This is a workaround:
sys.setrecursionlimit(3500)
config_dir = os.path.abspath(os.path.dirname(__file__))
try:
builddir = os.environ['BUILDDIR']
except KeyError:
@ -146,6 +151,7 @@ suppress_warnings = ['image.nonlocal_uri']
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['breathe',
'sphinx_idf_theme',
'link-roles',
'sphinxcontrib.blockdiag',
'sphinxcontrib.seqdiag',
@ -154,6 +160,7 @@ extensions = ['breathe',
'sphinxcontrib.rackdiag',
'sphinxcontrib.packetdiag',
'html_redirects',
'latex_builder',
]
# Enabling this fixes cropping of blockdiag edge labels
@ -188,16 +195,14 @@ master_doc = 'index'
# built documents.
#
# Readthedocs largely ignores 'version' and 'release', and displays one of
# 'latest', tag name, or branch name, depending on the build type.
# Still, this is useful for non-RTD builds.
# This is supposed to be "the short X.Y version", but it's the only version
# This is the full exact version, canonical git version description
# visible when you open index.html.
# Display full version to make things less confusing.
version = run_cmd_get_output('git describe')
# The full version, including alpha/beta/rc tags.
# If needed, nearest tag is returned by 'git describe --abbrev=0'.
release = version
version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8')
# The 'release' version is the same as version for non-CI builds, but for CI
# builds on a branch then it's replaced with the branch name
release = sanitize_version(version)
print('Version: {0} Release: {1}'.format(version, release))
# There are two options for replacing |today|: either, you set today to some
@ -228,6 +233,13 @@ exclude_patterns = ['_build','README.md']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# Extra options required by sphinx_idf_theme
project_slug = 'esp-idf'
versions_url = 'https://dl.espressif.com/dl/esp-idf/idf_versions.js'
languages = ['en', 'zh_CN']
# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []
@ -250,7 +262,15 @@ html_redirect_pages = [tuple(line.split(' ')) for line in lines]
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'
html_theme = 'sphinx_idf_theme'
# context used by sphinx_idf_theme
html_context = {
"display_github": True, # Add 'Edit on Github' link instead of 'View page source'
"github_user": "espressif",
"github_repo": "esp-idf",
"github_version": get_github_rev(),
}
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@ -333,28 +353,41 @@ htmlhelp_basename = 'ReadtheDocsTemplatedoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
# 'papersize': 'letterpaper',
#
# The font size ('10pt', '11pt' or '12pt').
# 'pointsize': '10pt',
#
# Additional stuff for the LaTeX preamble.
# 'preamble': '',
}
latex_template_dir = os.path.join(config_dir, 'latex_templates')
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'ReadtheDocsTemplate.tex', u'Read the Docs Template Documentation',
u'Read the Docs', 'manual'),
]
preamble = ''
with open(os.path.join(latex_template_dir, 'preamble.tex')) as f:
preamble = f.read()
titlepage = ''
with open(os.path.join(latex_template_dir, 'titlepage.tex')) as f:
titlepage = f.read()
latex_elements = {
'papersize': 'a4paper',
# Latex figure (float) alignment
'figure_align':'htbp',
'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
'fncychap': '\\usepackage[Sonny]{fncychap}',
'preamble': preamble,
'maketitle': titlepage,
}
# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None
# The name of an image file (relative to this directory) to place at the bottom of
# the title page.
latex_logo = "../_static/espressif2.pdf"
latex_engine = 'xelatex'
latex_use_xindy = False
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
@ -415,3 +448,24 @@ texinfo_documents = [
def setup(app):
app.add_stylesheet('theme_overrides.css')
generate_version_specific_includes(app)
# Not all config variables are set when setup is called
app.connect('config-inited', setup_config_values)
app.connect('config-inited', setup_html_context)
def setup_config_values(app, config):
# Sets up global config values needed by other extensions
idf_target_title_dict = {
'esp32': 'ESP32',
}
app.add_config_value('idf_target_title_dict', idf_target_title_dict, 'env')
pdf_name = "esp-idf-{}-{}-{}".format(app.config.language, app.config.version, "esp32")
app.add_config_value('pdf_file', pdf_name, 'env')
def setup_html_context(app, config):
# Setup path for 'edit on github'-link
app.config.html_context['conf_py_path'] = "/docs/{}/".format(app.config.language)

Wyświetl plik

@ -17,7 +17,7 @@ except ImportError:
# General information about the project.
project = u'ESP-IDF Programming Guide'
copyright = u'2016 - 2020, Espressif Systems (Shanghai) CO., LTD'
copyright = u'2016 - 2021, Espressif Systems (Shanghai) CO., LTD'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.

Wyświetl plik

@ -189,34 +189,29 @@ def write_version_note(template, out_dir, version, ver_type, is_stable):
def get_version():
"""
Returns a tuple of (name of branch/tag, type branch/tag, is_stable)
Returns a tuple of (name of branch/tag/commit-id, type branch/tag/commit, is_stable)
"""
# Trust what RTD says our version is, if it is set
version = os.environ.get("READTHEDOCS_VERSION", None)
if version == "latest":
return ("master", "branch", False)
# Otherwise, use git to look for a tag
# Use git to look for a tag
try:
tag = subprocess.check_output(["git", "describe", "--exact-match"]).strip()
tag = subprocess.check_output(["git", "describe", "--exact-match"]).strip().decode('utf-8')
is_stable = re.match(r"v[0-9\.]+$", tag) is not None
return (tag, "tag", is_stable)
except subprocess.CalledProcessError:
pass
# No tag, look for a branch
refs = subprocess.check_output(["git", "for-each-ref", "--points-at", "HEAD", "--format", "%(refname)"])
print("refs:\n%s" % refs)
refs = refs.split(b"\n")
# Note: this looks for branches in 'origin' because GitLab CI doesn't check out a local branch
branches = [r.replace(b"refs/remotes/origin/",b"").strip() for r in refs if r.startswith(b"refs/remotes/origin/")]
if len(branches) == 0:
# last resort, return the commit (may happen on Gitlab CI sometimes, unclear why)
return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip(), "commit", False)
if "master" in branches:
return ("master", "branch", False)
else:
return (branches[0], "branch", False) # take whatever the first branch is
# No tag, look at branch name from CI, this will give the correct branch name even if the ref for the branch we
# merge into has moved forward before the pipeline runs
branch = os.environ.get("CI_COMMIT_REF_NAME", None)
if branch is not None:
return (branch, "branch", False)
# Try to find the branch name even if docs are built locally
branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode('utf-8')
if branch != "HEAD":
return (branch, "branch", False)
# As a last resort we return commit SHA-1, should never happen in CI/docs that should be published
return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip().decode('utf-8'), "commit", False)
if __name__ == "__main__":

Wyświetl plik

@ -0,0 +1,15 @@
import subprocess
# Get revision used for constructing github URLs
def get_github_rev():
path = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode('utf-8')
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip().decode('utf-8')
except subprocess.CalledProcessError:
tag = None
print('Git commit ID: ', path)
if tag:
print('Git tag: ', tag)
return tag
return path

Wyświetl plik

@ -0,0 +1,55 @@
from sphinx.builders.latex import LaTeXBuilder
import os
# Overrides the default Sphinx latex build
class IdfLatexBuilder(LaTeXBuilder):
def __init__(self, app):
# Sets up the latex_documents config value, done here instead of conf.py since it depends on the runtime value 'idf_target'
self.init_latex_documents(app)
super().__init__(app)
def init_latex_documents(self, app):
file_name = app.config.pdf_file + '.tex'
if app.config.language == 'zh_CN':
latex_documents = [('index', file_name, u'ESP-IDF 编程指南', u'乐鑫信息科技', 'manual')]
else:
# Default to english naming
latex_documents = [('index', file_name, u'ESP-IDF Programming Guide', u'Espressif Systems', 'manual')]
app.config.latex_documents = latex_documents
def prepare_latex_macros(self, package_path, config):
PACKAGE_NAME = "espidf.sty"
latex_package = ''
with open(package_path, 'r') as template:
latex_package = template.read()
idf_target_title = "ESP32"
latex_package = latex_package.replace('<idf_target_title>', idf_target_title)
# Release name for the PDF front page, remove '_' as this is used for subscript in Latex
idf_release_name = "Release {}".format(config.version.replace('_', '-'))
latex_package = latex_package.replace('<idf_release_name>', idf_release_name)
with open(os.path.join(self.outdir, PACKAGE_NAME), 'w') as package_file:
package_file.write(latex_package)
def finish(self):
super().finish()
TEMPLATE_PATH = "../latex_templates/espidf.sty"
self.prepare_latex_macros(os.path.join(self.confdir,TEMPLATE_PATH), self.config)
def setup(app):
app.add_builder(IdfLatexBuilder, override=True)
return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}

Wyświetl plik

@ -0,0 +1,7 @@
\NeedsTeXFormat{LaTeX2e}[1995/12/01]
\ProvidesPackage{espidf}[2020/03/25 v0.1.0 LaTeX package (ESP-IDF markup)]
\newcommand{\idfTarget}{<idf_target_title>}
\newcommand{\idfReleaseName}{<idf_release_name>}
\endinput

Wyświetl plik

@ -0,0 +1,129 @@
% package with esp-idf specific macros
\usepackage{espidf}
\setcounter{secnumdepth}{2}
\setcounter{tocdepth}{2}
\usepackage{amsmath,amsfonts,amssymb,amsthm}
\usepackage{graphicx}
%%% reduce spaces for Table of contents, figures and tables
%%% it is used "\addtocontents{toc}{\vskip -1.2cm}" etc. in the document
\usepackage[notlot,nottoc,notlof]{}
\usepackage{color}
\usepackage{transparent}
\usepackage{eso-pic}
\usepackage{lipsum}
%%% Needed for displaying Chinese in English documentation
\usepackage{xeCJK}
\usepackage{footnotebackref} %%link at the footnote to go to the place of footnote in the text
%% spacing between line
\usepackage{setspace}
\singlespacing
\definecolor{myred}{RGB}{229, 32, 26}
\definecolor{mygrayy}{RGB}{127, 127, 127}
\definecolor{myblack}{RGB}{64, 64, 64}
%%%%%%%%%%% datetime
\usepackage{datetime}
\newdateformat{MonthYearFormat}{%
\monthname[\THEMONTH], \THEYEAR}
%% RO, LE will not work for 'oneside' layout.
%% Change oneside to twoside in document class
\usepackage{fancyhdr}
\pagestyle{fancy}
\fancyhf{}
% Header and footer
\makeatletter
\fancypagestyle{normal}{
\fancyhf{}
\fancyhead[L]{\nouppercase{\leftmark}}
\fancyfoot[C]{\py@HeaderFamily\thepage \\ \href{https://www.espressif.com/en/company/documents/documentation_feedback?docId=4287&sections=&version=\idfReleaseName}{Submit Document Feedback}}
\fancyfoot[L]{Espressif Systems}
\fancyfoot[R]{\idfReleaseName}
\renewcommand{\headrulewidth}{0.4pt}
\renewcommand{\footrulewidth}{0.4pt}
}
\makeatother
\renewcommand{\headrulewidth}{0.5pt}
\renewcommand{\footrulewidth}{0.5pt}
% Define a spacing for section, subsection and subsubsection
% http://tex.stackexchange.com/questions/108684/spacing-before-and-after-section-titles
\titlespacing*{\section}{0pt}{6pt plus 0pt minus 0pt}{6pt plus 0pt minus 0pt}
\titlespacing*{\subsection}{0pt}{18pt plus 64pt minus 0pt}{0pt}
\titlespacing*{\subsubsection}{0pt}{12pt plus 0pt minus 0pt}{0pt}
\titlespacing*{\paragraph} {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
\titlespacing*{\subparagraph} {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
% Define the colors of table of contents
% This is helpful to understand http://tex.stackexchange.com/questions/110253/what-the-first-argument-for-lsubsection-actually-is
\definecolor{LochmaraColor}{HTML}{1020A0}
% Hyperlinks
\hypersetup{
colorlinks = true,
allcolors = {LochmaraColor},
}
\RequirePackage{tocbibind} %%% comment this to remove page number for following
\addto\captionsenglish{\renewcommand{\contentsname}{Table of contents}}
\addto\captionsenglish{\renewcommand{\listfigurename}{List of figures}}
\addto\captionsenglish{\renewcommand{\listtablename}{List of tables}}
% \addto\captionsenglish{\renewcommand{\chaptername}{Chapter}}
%%reduce spacing for itemize
\usepackage{enumitem}
\setlist{nosep}
%%%%%%%%%%% Quote Styles at the top of chapter
\usepackage{epigraph}
\setlength{\epigraphwidth}{0.8\columnwidth}
\newcommand{\chapterquote}[2]{\epigraphhead[60]{\epigraph{\textit{#1}}{\textbf {\textit{--#2}}}}}
%%%%%%%%%%% Quote for all places except Chapter
\newcommand{\sectionquote}[2]{{\quote{\textit{``#1''}}{\textbf {\textit{--#2}}}}}
% Insert 22pt white space before roc title. \titlespacing at line 65 changes it by -22 later on.
\renewcommand*\contentsname{\hspace{0pt}Contents}
% Define section, subsection and subsubsection font size and color
\usepackage{sectsty}
\definecolor{AllportsColor}{HTML}{A02010}
\allsectionsfont{\color{AllportsColor}}
\usepackage{titlesec}
\titleformat{\section}
{\color{AllportsColor}\LARGE\bfseries}{\thesection.}{1em}{}
\titleformat{\subsection}
{\color{AllportsColor}\Large\bfseries}{\thesubsection.}{1em}{}
\titleformat{\subsubsection}
{\color{AllportsColor}\large\bfseries}{\thesubsubsection.}{1em}{}
\titleformat{\paragraph}
{\color{AllportsColor}\large\bfseries}{\theparagraph}{1em}{}
\titleformat{\subparagraph}
{\normalfont\normalsize\bfseries}{\thesubparagraph}{1em}{}
\titleformat{\subsubparagraph}
{\normalfont\normalsize\bfseries}{\thesubsubparagraph}{1em}{}

Wyświetl plik

@ -0,0 +1,39 @@
\makeatletter
\newgeometry{left=0cm,right=0cm,bottom=2cm}
\cfoot{www.espressif.com}
\renewcommand{\headrulewidth}{0pt}
{\color{myred}\rule{30pt}{2.1cm}}
\hspace{0.2cm}
\begin{minipage}[b]{18cm}
{\fontsize{36pt}{48pt}\textbf{\idfTarget}}\\
{\fontsize{28pt}{18pt}\textbf{\color{mygrayy}\@title}}
\end{minipage}
\hspace{\stretch{1}}
\vspace{48em}
\begin{flushright}
\setlength\parindent{8em}
\begin{minipage}[b]{2cm}
\sphinxlogo
\end{minipage}
\hspace{0.2cm}
\rule{3pt}{1.9cm}
\hspace{0.2cm}
\begin{minipage}[b]{7cm}
{\large{\idfReleaseName}}\smallskip\newline
{\large{\@author}}\smallskip\newline
{\large{\@date}}\smallskip
\end{minipage}
{\color{myred}\rule{30pt}{1.9cm}}
\end{flushright}
\restoregeometry
\makeatother

Wyświetl plik

@ -1,17 +1,15 @@
# This is a list of python packages used to generate documentation. This file is used with pip:
# pip install --user -r requirements.txt
#
sphinx>=1.8.4
breathe==4.11.1
sphinx-rtd-theme
sphinx==2.3.1
breathe==4.14.1
sphinx-notfound-page
sphinxcontrib-blockdiag>=1.5.5, <2.0.0
sphinxcontrib-seqdiag>=0.8.5, <2.0.0
sphinxcontrib-actdiag>=0.8.5, <2.0.0
sphinxcontrib-nwdiag>=0.9.5, <2.0.0
blockdiag>=1.5.4, <2.0.0
seqdiag>=0.9.6, <2.0.0
actdiag>=0.5.4, <2.0.0
nwdiag>=1.0.4, <2.0.0
sphinxcontrib-blockdiag==2.0.0
sphinxcontrib-seqdiag==2.0.0
sphinxcontrib-actdiag==2.0.0
sphinxcontrib-nwdiag==2.0.0
sphinxcontrib-wavedrom==2.0.0
nwdiag==2.0.0
recommonmark
future>=0.16.0 # for ../tools/gen_esp_err_to_name.py
sphinx_idf_theme==0.2

Wyświetl plik

@ -0,0 +1,43 @@
# Tiny Python module to sanitize a Git version into something that can be used in a URL
#
# (this is used in multiple places: conf_common.py and in tools/ci/docs_deploy
#
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
def sanitize_version(original_version):
""" Given a version (probably output from 'git describe --always' or similar), return
a URL-safe sanitized version. (this is used as 'release' config variable when building
the docs.)
Will override the original version with the Gitlab CI CI_COMMIT_REF_NAME environment variable if
this is present.
Also follows the RTD-ism that master branch is named 'latest'
"""
try:
version = os.environ['CI_COMMIT_REF_NAME']
except KeyError:
version = original_version
if version == "master":
return "latest"
version = version.replace('/', '-')
return version

Wyświetl plik

@ -119,9 +119,9 @@ DUT1 终端::
Waiting for signal: [dut2 mac address]!
Please input parameter value from any board send this signal and press "Enter" key.
DUT2 终端::
DUT2 终端::
Send signal: [dut2 mac address][10:20:30:40:50:60]!
@ -189,7 +189,7 @@ DUT2 终端::
当单元测试应用程序空闲时,输入回车键,它会打印出测试菜单,其中包含所有的测试项目。
.. code:: bash
.. code::
Here's the test menu, pick your combo:
(1) "esp_ota_begin() verifies arguments" [ota]
@ -236,7 +236,7 @@ DUT2 终端::
一旦选择了多设备测试用例,它会打印一个子菜单:
.. code:: bash
.. code::
Running gpio master/slave test example...
gpio master/slave test example
@ -247,7 +247,7 @@ DUT2 终端::
与多设备测试用例相似,多阶段测试用例也会打印子菜单:
.. code:: bash
.. code::
Running reset reason check for deepsleep...
reset reason check for deepsleep

Wyświetl plik

@ -17,7 +17,7 @@ C 文件可以包含多个测试用例。测试文件的名字要以 “test”
测试用例需要通过 C 文件中特定的函数来添加,如下所示:
.. code:: c
.. code-block:: c
TEST_CASE("test name", "[module name]"
{
@ -179,7 +179,7 @@ DUT2slave终端
当单元测试应用程序空闲时,输入回车键,它会打印出测试菜单,其中包含所有的测试项目。
.. code:: bash
.. code::
Here's the test menu, pick your combo:
(1) "esp_ota_begin() verifies arguments" [ota]
@ -226,7 +226,7 @@ DUT2slave终端
一旦选择了多设备测试用例,它会打印一个子菜单:
.. code:: bash
.. code::
Running gpio master/slave test example...
gpio master/slave test example
@ -237,7 +237,7 @@ DUT2slave终端
与多设备测试用例相似,多阶段测试用例也会打印子菜单:
.. code:: bash
.. code::
Running reset reason check for deepsleep...
reset reason check for deepsleep

Wyświetl plik

@ -17,7 +17,7 @@ except ImportError:
# General information about the project.
project = u'ESP-IDF 编程指南'
copyright = u'2016 - 2020 乐鑫信息科技(上海)股份有限公司'
copyright = u'2016 - 2021 乐鑫信息科技(上海)股份有限公司'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.

Wyświetl plik

@ -198,25 +198,17 @@ build_examples_cmake_esp32:
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
build_docs:
.build_docs_template: &build_docs_template
stage: build
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
image: $ESP_IDF_DOC_ENV_IMAGE
tags:
- build_docs
artifacts:
when: always
paths:
# English version of documentation
- docs/en/doxygen-warning-log.txt
- docs/en/sphinx-warning-log.txt
- docs/en/sphinx-warning-log-sanitized.txt
- docs/en/_build/html
- docs/sphinx-err-*
# Chinese version of documentation
- docs/zh_CN/doxygen-warning-log.txt
- docs/zh_CN/sphinx-warning-log.txt
- docs/zh_CN/sphinx-warning-log-sanitized.txt
- docs/zh_CN/_build/html
- docs/*/*.txt
- docs/_build/*/html/*
- docs/_build/*/latex/*
expire_in: 4 days
only:
variables:
@ -224,17 +216,31 @@ build_docs:
- $BOT_LABEL_BUILD
- $BOT_LABEL_BUILD_DOCS
- $BOT_LABEL_REGULAR_TEST
dependencies: []
script:
# Active python 3.6.10 env as this is where Sphinx is installed
- source /opt/pyenv/activate && pyenv global 3.6.10
# Setup a build dir with both languages to simplify deployment
- cd docs
- mkdir -p _build/$DOCLANG
- ./check_lang_folder_sync.sh
- cd en
- cd $DOCLANG
- make gh-linkcheck
- make html
- ../check_doc_warnings.sh
- cd ../zh_CN
- make gh-linkcheck
- make html
- make latexpdf LATEXMKOPTS="--f --interaction=nonstopmode --quiet --outdir=build"
- ../check_doc_warnings.sh
- mv -f _build/* ../_build/$DOCLANG
build_docs_en:
extends: .build_docs_template
variables:
DOCLANG: "en"
build_docs_zh_CN:
extends: .build_docs_template
variables:
DOCLANG: "zh_CN"
verify_cmake_style:
extends: .check_job_template

Wyświetl plik

@ -73,45 +73,67 @@ push_to_github:
- git remote add github git@github.com:espressif/esp-idf.git
- tools/ci/push_to_github.sh
deploy_docs:
.deploy_docs_template:
extends: .before_script_lesser
stage: deploy
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
image: $ESP_IDF_DOC_ENV_IMAGE
tags:
- deploy
- shiny
dependencies:
- build_docs_en
- build_docs_zh_CN
variables:
DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
PYTHONUNBUFFERED: 1
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
- export GIT_VER=$(git describe --always)
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
deploy_docs_preview:
extends: .deploy_docs_template
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD_DOCS
dependencies:
- build_docs
extends: .before_script_lesser
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
- export GIT_VER=$(git describe --always)
- cd docs/en/_build/
- mv html $GIT_VER
- tar czvf $GIT_VER.tar.gz $GIT_VER
- scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
- ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
- cd ../../zh_CN/_build/
- mv html $GIT_VER
- tar czvf $GIT_VER.tar.gz $GIT_VER
- scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
- ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
# add link to preview doc
- echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
- echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
variables:
TYPE: "preview"
# older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
DOCS_DEPLOY_PATH: "$DOCS_PATH"
DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
# deploy docs to production webserver
deploy_docs_production:
extends: .deploy_docs_template
only:
refs:
# The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
variables:
- $BOT_TRIGGER_WITH_LABEL == null
variables:
TYPE: "preview"
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
deploy_test_result:
stage: deploy

Wyświetl plik

@ -0,0 +1,223 @@
#!/usr/bin/env python3
#
# CI script to deploy docs to a webserver. Not useful outside of CI environment
#
#
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import glob
import os
import os.path
import re
import stat
import sys
import subprocess
import tarfile
import packaging.version
def env(variable, default=None):
""" Shortcut to return the expanded version of an environment variable """
return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
# import sanitize_version from the docs directory, shared with here
sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
from sanitize_version import sanitize_version # noqa
def main():
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
git_ver = env("GIT_VER") # output of git describe --always
ci_ver = env("CI_COMMIT_REF_NAME", git_ver) # branch or tag we're building for (used for 'release' & URL)
version = sanitize_version(ci_ver)
print("Git version: {}".format(git_ver))
print("CI Version: {}".format(ci_ver))
print("Deployment version: {}".format(version))
if not version:
raise RuntimeError("A version is needed to deploy")
build_dir = env("DOCS_BUILD_DIR") # top-level local build dir, where docs have already been built
if not build_dir:
raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
url_base = env("DOCS_DEPLOY_URL_BASE") # base for HTTP URLs, used to print the URL to the log after deploying
docs_server = env("DOCS_DEPLOY_SERVER") # ssh server to deploy to
docs_user = env("DOCS_DEPLOY_SERVER_USER")
docs_path = env("DOCS_DEPLOY_PATH") # filesystem path on DOCS_SERVER
if not docs_server:
raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
if not docs_user:
raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
docs_server = "{}@{}".format(docs_user, docs_server)
if not docs_path:
raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
deploy(version, tarball_path, docs_path, docs_server)
print("Docs URLs:")
doc_deploy_type = os.getenv('TYPE')
for vurl in version_urls:
language, _, = vurl.split('/')
tag = '{}'.format(language)
url = "{}/{}/index.html".format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r"([^:])//", r"\1/", url) # get rid of any // that isn't in the https:// part
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
# note: it would be neater to use symlinks for stable, but because of the directory order
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
# process but call the version 'stable' this time
if is_stable_version(version):
print("Deploying again as stable version...")
tarball_path, version_urls = build_doc_tarball("stable", git_ver, build_dir)
deploy("stable", tarball_path, docs_path, docs_server)
def deploy(version, tarball_path, docs_path, docs_server):
def run_ssh(commands):
""" Log into docs_server and run a sequence of commands using ssh """
print("Running ssh: {}".format(commands))
subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
# copy the version tarball to the server
run_ssh(["mkdir -p {}".format(docs_path)])
print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
tarball_name = os.path.basename(tarball_path)
run_ssh(["cd {}".format(docs_path),
"rm -rf ./*/{}".format(version), # remove any pre-existing docs matching this version
"tar -zxvf {}".format(tarball_name), # untar the archive with the new docs
"rm {}".format(tarball_name)])
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
# as the version will be invalid for a window of time. Better to do it atomically, but this is
# another thing made much more complex by the directory structure putting language before version...
def build_doc_tarball(version, git_ver, build_dir):
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
the given version """
version_paths = []
tarball_path = "{}/{}.tar.gz".format(build_dir, version)
# find all the 'html/' directories under build_dir
html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
print("Found %d html directories" % len(html_dirs))
pdfs = glob.glob("{}/**/latex/build/*.pdf".format(build_dir), recursive=True)
print("Found %d PDFs in latex directories" % len(pdfs))
# add symlink for stable and latest and adds them to PDF blob
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
def not_sources_dir(ti):
""" Filter the _sources directories out of the tarballs """
if ti.name.endswith("/_sources"):
return None
ti.mode |= stat.S_IWGRP # make everything group-writeable
return ti
try:
os.remove(tarball_path)
except OSError:
pass
with tarfile.open(tarball_path, "w:gz") as tarball:
for html_dir in html_dirs:
# html_dir has the form '<ignored>/<language>/html/'
language_dirname = os.path.dirname(os.path.dirname(html_dir))
language = os.path.basename(language_dirname)
# when deploying, we want the top-level directory layout 'language/version'
archive_path = "{}/{}".format(language, version)
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
tarball.add(html_dir, archive_path, filter=not_sources_dir)
version_paths.append(archive_path)
for pdf_path in pdfs:
# pdf_path has the form '<ignored>/<language>/<target>/latex/build'
latex_dirname = os.path.dirname(pdf_path)
pdf_filename = os.path.basename(pdf_path)
language_dirname = os.path.dirname(os.path.dirname(latex_dirname))
language = os.path.basename(language_dirname)
# when deploying, we want the layout 'language/version/pdf'
archive_path = "{}/{}/{}".format(language, version, pdf_filename)
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
tarball.add(pdf_path, archive_path)
for symlink in symlinks:
os.unlink(symlink)
return (os.path.abspath(tarball_path), version_paths)
def create_and_add_symlinks(version, git_ver, pdfs):
""" Create symbolic links for PDFs for 'latest' and 'stable' releases """
symlinks = []
if 'stable' in version or 'latest' in version:
for pdf_path in pdfs:
symlink_path = pdf_path.replace(git_ver, version)
os.symlink(pdf_path, symlink_path)
symlinks.append(symlink_path)
pdfs.extend(symlinks)
print("Found %d PDFs in latex directories after adding symlink" % len(pdfs))
return symlinks
def is_stable_version(version):
""" Heuristic for whether this is the latest stable release """
if not version.startswith("v"):
return False # branch name
if "-" in version:
return False # prerelease tag
git_out = subprocess.check_output(["git", "tag", "-l"]).decode("utf-8")
versions = [v.strip() for v in git_out.split("\n")]
versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)] # include vX.Y.Z only
versions = [packaging.version.parse(v) for v in versions]
max_version = max(versions)
if max_version.public != version[1:]:
print("Stable version is v{}. This version is {}.".format(max_version.public, version))
return False
else:
print("This version {} is the stable version".format(version))
return True
if __name__ == "__main__":
main()

Wyświetl plik

@ -40,6 +40,7 @@ tools/ci/check_examples_cmake_make.sh
tools/ci/check_idf_version.sh
tools/ci/check_ut_cmake_make.sh
tools/ci/checkout_project_ref.py
tools/ci/deploy_docs.py
tools/ci/envsubst.py
tools/ci/get-full-sources.sh
tools/ci/get_supported_examples.sh