docs: add fast build feature and activate it in CI

Adds feature for skipping include of doxygen headers into sphinx build
when env variable is set.

Builds incomplete docs (no function API documention)
but speeds up building (20min to 1<min).

Add this fast build as the default way of building docs in CI on
non protected branches.

.gitlab/ci/dependencies/dependencies.yml

@@ -49,22 +49,6 @@
     - build_system
     - macos
 
-"build:docs":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  patterns:
-    - docs
-  deploy:
-    - production
-
-"build:docs:label-only":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  deploy:
-    - preview
-
 # ---------------
 # Build Test Jobs
 # ---------------

.gitlab/ci/docs.yml

@@ -1,8 +1,45 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_full: &if-label-docs_full
+  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_full
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
 # stage: pre_check
 check_readme_links:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   tags: ["build", "amd64", "internet"]
   allow_failure: true
   script:
@@ -11,7 +48,7 @@ check_readme_links:
 check_docs_lang_sync:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ./check_lang_folder_sync.sh
@@ -34,7 +71,7 @@ check_docs_gh_links:
   extends:
     - .pre_check_job_template
     - .build_docs_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@@ -45,7 +82,6 @@ check_docs_gh_links:
 .build_docs_build_stage_template:
   extends:
     - .build_docs_template
-    - .rules:build:docs
   stage: build_doc
   needs:
     - job: check_docs_lang_sync
@@ -53,9 +89,14 @@ check_docs_gh_links:
     - job: check_docs_gh_links
       artifacts: false
 
-build_docs_html:
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
   extends:
     - .build_docs_build_stage_template
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -65,10 +106,32 @@ build_docs_html:
   variables:
     DOC_BUILDERS: "html"
 
+build_docs_html_fast:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_full
+      when: never
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
+
 build_docs_pdf:
   extends:
     - .build_docs_build_stage_template
-    - .rules:build:docs:label-only
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -98,10 +161,12 @@ build_docs_pdf:
 deploy_docs_preview:
   extends:
     - .deploy_docs_template
-    - .rules:build:docs:label-only-preview
+  rules:
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
+    - <<: *if-label-build_docs
-  needs:
+    - <<: *if-label-docs
-    - build_docs_html
+  dependencies:
+    - build_docs_html_fast
+    - build_docs_html_full
     - build_docs_pdf
   variables:
     TYPE: "preview"
@@ -117,11 +182,12 @@ deploy_docs_production:
   # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
   extends:
     - .deploy_docs_template
-    - .rules:build:docs-production
+  rules:
+    - <<: *if-protected-no_label
   stage: post_deploy
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
+    - build_docs_html_full
     - build_docs_pdf
     - job: push_to_github
       artifacts: false

.gitlab/ci/rules.yml

@@ -1,16 +1,6 @@
 ############
 # Patterns #
 ############
-.patterns-docs: &patterns-docs
-  - ".gitlab/ci/docs.yml"
-  - "docs/**/*"
-  - "components/**/*.h"
-  - "components/**/Kconfig*"
-  - "components/**/CMakeList.txt"
-  - "components/**/sdkconfig*"
-  - "tools/kconfig_new/**/*"
-  - "CONTRIBUTING.rst"
-
 .patterns-c-files: &patterns-c-files
   - ".gitlab/ci/static-code-analysis.yml"
   - "tools/ci/static-analysis-rules.yml"
@@ -258,9 +248,6 @@
 .if-label-build: &if-label-build
   if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
 
-.if-label-build_docs: &if-label-build_docs
-  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-component_ut: &if-label-component_ut
   if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i'
 
@@ -294,9 +281,6 @@
 .if-label-docker: &if-label-docker
   if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i'
 
-.if-label-docs: &if-label-docs
-  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-example_test: &if-label-example_test
   if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i'
 
@@ -486,29 +470,6 @@
     - <<: *if-dev-push
       changes: *patterns-docker
 
-.rules:build:docs:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
-
-.rules:build:docs-production:
-  rules:
-    - <<: *if-protected-no_label
-
-.rules:build:docs:label-only:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
-.rules:build:docs:label-only-preview:
-  rules:
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
 .rules:build:example_test-esp32:
   rules:
     - <<: *if-protected

docs/build_docs.py

@@ -100,6 +100,7 @@ def main():
 
     build_parser = action_parsers.add_parser('build', help='Build documentation')
     build_parser.add_argument('--check-warnings-only', '-w', action='store_true')
+    build_parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build')
 
     action_parsers.add_parser('linkcheck', help='Check links (a current IDF revision should be uploaded to GitHub)')
 
@@ -124,6 +125,11 @@ def main():
     if args.action == 'build' or args.action is None:
         if args.action is None:
             args.check_warnings_only = False
+            args.fast_build = False
+
+        if args.fast_build:
+            os.environ['DOCS_FAST_BUILD'] = 'y'
+
         sys.exit(action_build(args))
 
     if args.action == 'linkcheck':

docs/en/contribute/documenting-code.rst

@@ -513,6 +513,16 @@ As well as wildcards::
 
 Note that this is a feature intended to simply testing and debugging during writing of documentation. The HTML output won't be perfect, i.e. it will not build a proper index that lists all the documents, and any references to documents that are not built will result in warnings.
 
+Fast build
+""""""""""
+Another trick to speed up building is to skip including doxygen generated API documention into the Sphinx build process, skipping this drastically reduces build times.
+
+This can be achieved by adding the fast-build argument::
+
+    ./build_docs.py build -f
+
+or by setting the environment variable `DOCS_FAST_BUILD`. Note that the the `-f` argument is a subargument to `build` and thus must be listed after `build`.
+
 Building PDF
 """"""""""""
 

docs/idf_extensions/run_doxygen.py

@@ -73,6 +73,8 @@ def convert_api_xml_to_inc(app, doxyfiles):
     xml_directory_path = '{}/xml'.format(build_dir)
     inc_directory_path = '{}/inc'.format(build_dir)
 
+    fast_build = os.environ.get('DOCS_FAST_BUILD', None)
+
     if not os.path.isdir(xml_directory_path):
         raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path))
 
@@ -96,6 +98,13 @@ def convert_api_xml_to_inc(app, doxyfiles):
             with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
                 inc_file.write(rst_output)
 
+        # For fast builds we wipe the doxygen api documention.
+        # Parsing this output during the sphinx build process is
+        # what takes 95% of the build time
+        if fast_build:
+            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
+                inc_file.write('')
+
 
 def get_doxyfile_input_paths(app, doxyfile_path):
     """Get contents of Doxyfile's INPUT statement.