docs: add fast build feature and activate it in CI

Adds feature for skipping include of doxygen headers into sphinx build when env variable is set. Builds incomplete docs (no function API documention) but speeds up building (20min to 1<min). Add this fast build as the default way of building docs in CI on non protected branches.
2021-04-27 12:24:22 +08:00 · 2021-04-27 12:24:22 +08:00 · 89179dc286
commit 89179dc286
--- a/.gitlab/ci/dependencies/dependencies.yml
+++ b/.gitlab/ci/dependencies/dependencies.yml
@ -49,22 +49,6 @@
    - build_system
    - macos

-"build:docs":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  patterns:
-    - docs
-  deploy:
-    - production
-
-"build:docs:label-only":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  deploy:
-    - preview
-
 # ---------------
 # Build Test Jobs
 # ---------------
--- a/.gitlab/ci/docs.yml
+++ b/.gitlab/ci/docs.yml
@ -1,8 +1,45 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_full: &if-label-docs_full
+  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_full
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
 # stage: pre_check
 check_readme_links:
  extends:
    - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
  tags: ["build", "amd64", "internet"]
  allow_failure: true
  script:
@ -11,7 +48,7 @@ check_readme_links:
 check_docs_lang_sync:
  extends:
    - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
  script:
    - cd docs
    - ./check_lang_folder_sync.sh
@ -34,7 +71,7 @@ check_docs_gh_links:
  extends:
    - .pre_check_job_template
    - .build_docs_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
  script:
    - cd docs
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@ -45,7 +82,6 @@ check_docs_gh_links:
 .build_docs_build_stage_template:
  extends:
    - .build_docs_template
-    - .rules:build:docs
  stage: build_doc
  needs:
    - job: check_docs_lang_sync
@ -53,9 +89,14 @@ check_docs_gh_links:
    - job: check_docs_gh_links
      artifacts: false

-build_docs_html:
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
  extends:
    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
  artifacts:
    when: always
    paths:
@ -65,10 +106,32 @@ build_docs_html:
  variables:
    DOC_BUILDERS: "html"

+build_docs_html_fast:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_full
+      when: never
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
+
 build_docs_pdf:
  extends:
    - .build_docs_build_stage_template
-    - .rules:build:docs:label-only
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
  artifacts:
    when: always
    paths:
@ -98,10 +161,12 @@ build_docs_pdf:
 deploy_docs_preview:
  extends:
    - .deploy_docs_template
-    - .rules:build:docs:label-only-preview
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - build_docs_html
+  rules:
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+  dependencies:
+    - build_docs_html_fast
+    - build_docs_html_full
    - build_docs_pdf
  variables:
    TYPE: "preview"
@ -117,11 +182,12 @@ deploy_docs_production:
  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
  extends:
    - .deploy_docs_template
-    - .rules:build:docs-production
+  rules:
+    - <<: *if-protected-no_label
  stage: post_deploy
  dependencies:  # set dependencies to null to avoid missing artifacts issue
  needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
+    - build_docs_html_full
    - build_docs_pdf
    - job: push_to_github
      artifacts: false
--- a/.gitlab/ci/rules.yml
+++ b/.gitlab/ci/rules.yml
@ -1,16 +1,6 @@
 ############
 # Patterns #
 ############
-.patterns-docs: &patterns-docs
-  - ".gitlab/ci/docs.yml"
-  - "docs/**/*"
-  - "components/**/*.h"
-  - "components/**/Kconfig*"
-  - "components/**/CMakeList.txt"
-  - "components/**/sdkconfig*"
-  - "tools/kconfig_new/**/*"
-  - "CONTRIBUTING.rst"
-
 .patterns-c-files: &patterns-c-files
  - ".gitlab/ci/static-code-analysis.yml"
  - "tools/ci/static-analysis-rules.yml"
@ -258,9 +248,6 @@
 .if-label-build: &if-label-build
  if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'

-.if-label-build_docs: &if-label-build_docs
-  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-component_ut: &if-label-component_ut
  if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i'

@ -294,9 +281,6 @@
 .if-label-docker: &if-label-docker
  if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i'

-.if-label-docs: &if-label-docs
-  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-example_test: &if-label-example_test
  if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i'

@ -486,29 +470,6 @@
    - <<: *if-dev-push
      changes: *patterns-docker

-.rules:build:docs:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
-
-.rules:build:docs-production:
-  rules:
-    - <<: *if-protected-no_label
-
-.rules:build:docs:label-only:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
-.rules:build:docs:label-only-preview:
-  rules:
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
 .rules:build:example_test-esp32:
  rules:
    - <<: *if-protected
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@ -100,6 +100,7 @@ def main():

    build_parser = action_parsers.add_parser('build', help='Build documentation')
    build_parser.add_argument('--check-warnings-only', '-w', action='store_true')
+    build_parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build')

    action_parsers.add_parser('linkcheck', help='Check links (a current IDF revision should be uploaded to GitHub)')

@ -124,6 +125,11 @@ def main():
    if args.action == 'build' or args.action is None:
        if args.action is None:
            args.check_warnings_only = False
+            args.fast_build = False
+
+        if args.fast_build:
+            os.environ['DOCS_FAST_BUILD'] = 'y'
+
        sys.exit(action_build(args))

    if args.action == 'linkcheck':
--- a/docs/en/contribute/documenting-code.rst
+++ b/docs/en/contribute/documenting-code.rst
@ -513,6 +513,16 @@ As well as wildcards::

 Note that this is a feature intended to simply testing and debugging during writing of documentation. The HTML output won't be perfect, i.e. it will not build a proper index that lists all the documents, and any references to documents that are not built will result in warnings.

+Fast build
+""""""""""
+Another trick to speed up building is to skip including doxygen generated API documention into the Sphinx build process, skipping this drastically reduces build times.
+
+This can be achieved by adding the fast-build argument::
+
+    ./build_docs.py build -f
+
+or by setting the environment variable `DOCS_FAST_BUILD`. Note that the the `-f` argument is a subargument to `build` and thus must be listed after `build`.
+
 Building PDF
 """"""""""""

--- a/docs/idf_extensions/run_doxygen.py
+++ b/docs/idf_extensions/run_doxygen.py
@ -73,6 +73,8 @@ def convert_api_xml_to_inc(app, doxyfiles):
    xml_directory_path = '{}/xml'.format(build_dir)
    inc_directory_path = '{}/inc'.format(build_dir)

+    fast_build = os.environ.get('DOCS_FAST_BUILD', None)
+
    if not os.path.isdir(xml_directory_path):
        raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path))

@ -96,6 +98,13 @@ def convert_api_xml_to_inc(app, doxyfiles):
            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
                inc_file.write(rst_output)

+        # For fast builds we wipe the doxygen api documention.
+        # Parsing this output during the sphinx build process is
+        # what takes 95% of the build time
+        if fast_build:
+            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
+                inc_file.write('')
+

 def get_doxyfile_input_paths(app, doxyfile_path):
    """Get contents of Doxyfile's INPUT statement.