diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index ac6ea4b825..d6fb3700af 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -49,22 +49,6 @@ - build_system - macos -"build:docs": - labels: - - docs - - build_docs # for backward compatibility - patterns: - - docs - deploy: - - production - -"build:docs:label-only": - labels: - - docs - - build_docs # for backward compatibility - deploy: - - preview - # --------------- # Build Test Jobs # --------------- diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 9d53498a65..a0ce8e715e 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -1,8 +1,45 @@ +.patterns-docs: &patterns-docs + - ".gitlab/ci/docs.yml" + - "docs/**/*" + - "components/**/*.h" + - "components/**/Kconfig*" + - "components/**/CMakeList.txt" + - "components/**/sdkconfig*" + - "tools/kconfig_new/**/*" + - "CONTRIBUTING.rst" + +.if-protected: &if-protected + if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)' + +.if-protected-no_label: &if-protected-no_label + if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null' + +.if-label-build_docs: &if-label-build_docs + if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i' + +.if-label-docs: &if-label-docs + if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i' + +.if-label-docs_full: &if-label-docs_full + if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i' + +.if-dev-push: &if-dev-push + if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' + +.doc-rules:build:docs: + rules: + - <<: *if-protected + - <<: *if-label-build_docs + - <<: *if-label-docs + - <<: *if-label-docs_full + - <<: *if-dev-push + changes: *patterns-docs + # stage: pre_check check_readme_links: extends: - .pre_check_job_template - - .rules:build:docs + - .doc-rules:build:docs tags: ["build", "amd64", "internet"] allow_failure: true script: @@ -11,7 +48,7 @@ check_readme_links: check_docs_lang_sync: extends: - .pre_check_job_template - - .rules:build:docs + - .doc-rules:build:docs script: - cd docs - ./check_lang_folder_sync.sh @@ -34,7 +71,7 @@ check_docs_gh_links: extends: - .pre_check_job_template - .build_docs_template - - .rules:build:docs + - .doc-rules:build:docs script: - cd docs - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt @@ -45,7 +82,6 @@ check_docs_gh_links: .build_docs_build_stage_template: extends: - .build_docs_template - - .rules:build:docs stage: build_doc needs: - job: check_docs_lang_sync @@ -53,9 +89,14 @@ check_docs_gh_links: - job: check_docs_gh_links artifacts: false -build_docs_html: +# Doc jobs have a lot of special cases, we specify rules here directly instead +# in dependencies.yml to simplify things +build_docs_html_full: extends: - .build_docs_build_stage_template + rules: + - <<: *if-protected + - <<: *if-label-docs_full artifacts: when: always paths: @@ -65,10 +106,32 @@ build_docs_html: variables: DOC_BUILDERS: "html" +build_docs_html_fast: + extends: + - .build_docs_build_stage_template + rules: + - <<: *if-label-docs_full + when: never + - <<: *if-label-build_docs + - <<: *if-label-docs + - <<: *if-dev-push + changes: *patterns-docs + artifacts: + when: always + paths: + - docs/_build/*/*/*.txt + - docs/_build/*/*/html/* + expire_in: 4 days + variables: + DOC_BUILDERS: "html" + DOCS_FAST_BUILD: "yes" + build_docs_pdf: extends: - .build_docs_build_stage_template - - .rules:build:docs:label-only + rules: + - <<: *if-protected + - <<: *if-label-docs_full artifacts: when: always paths: @@ -98,10 +161,12 @@ build_docs_pdf: deploy_docs_preview: extends: - .deploy_docs_template - - .rules:build:docs:label-only-preview - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - build_docs_html + rules: + - <<: *if-label-build_docs + - <<: *if-label-docs + dependencies: + - build_docs_html_fast + - build_docs_html_full - build_docs_pdf variables: TYPE: "preview" @@ -117,11 +182,12 @@ deploy_docs_production: # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings extends: - .deploy_docs_template - - .rules:build:docs-production + rules: + - <<: *if-protected-no_label stage: post_deploy dependencies: # set dependencies to null to avoid missing artifacts issue needs: # ensure runs after push_to_github succeeded - - build_docs_html + - build_docs_html_full - build_docs_pdf - job: push_to_github artifacts: false diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 3ae2eb0f27..8c2d26730e 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -1,16 +1,6 @@ ############ # Patterns # ############ -.patterns-docs: &patterns-docs - - ".gitlab/ci/docs.yml" - - "docs/**/*" - - "components/**/*.h" - - "components/**/Kconfig*" - - "components/**/CMakeList.txt" - - "components/**/sdkconfig*" - - "tools/kconfig_new/**/*" - - "CONTRIBUTING.rst" - .patterns-c-files: &patterns-c-files - ".gitlab/ci/static-code-analysis.yml" - "tools/ci/static-analysis-rules.yml" @@ -258,9 +248,6 @@ .if-label-build: &if-label-build if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i' -.if-label-build_docs: &if-label-build_docs - if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i' - .if-label-component_ut: &if-label-component_ut if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i' @@ -294,9 +281,6 @@ .if-label-docker: &if-label-docker if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i' -.if-label-docs: &if-label-docs - if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i' - .if-label-example_test: &if-label-example_test if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i' @@ -486,29 +470,6 @@ - <<: *if-dev-push changes: *patterns-docker -.rules:build:docs: - rules: - - <<: *if-protected - - <<: *if-label-build_docs - - <<: *if-label-docs - - <<: *if-dev-push - changes: *patterns-docs - -.rules:build:docs-production: - rules: - - <<: *if-protected-no_label - -.rules:build:docs:label-only: - rules: - - <<: *if-protected - - <<: *if-label-build_docs - - <<: *if-label-docs - -.rules:build:docs:label-only-preview: - rules: - - <<: *if-label-build_docs - - <<: *if-label-docs - .rules:build:example_test-esp32: rules: - <<: *if-protected diff --git a/docs/build_docs.py b/docs/build_docs.py index 5e980dbd3d..6841f0bb5c 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -100,6 +100,7 @@ def main(): build_parser = action_parsers.add_parser('build', help='Build documentation') build_parser.add_argument('--check-warnings-only', '-w', action='store_true') + build_parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build') action_parsers.add_parser('linkcheck', help='Check links (a current IDF revision should be uploaded to GitHub)') @@ -124,6 +125,11 @@ def main(): if args.action == 'build' or args.action is None: if args.action is None: args.check_warnings_only = False + args.fast_build = False + + if args.fast_build: + os.environ['DOCS_FAST_BUILD'] = 'y' + sys.exit(action_build(args)) if args.action == 'linkcheck': diff --git a/docs/en/contribute/documenting-code.rst b/docs/en/contribute/documenting-code.rst index 3f3d36d86f..4c496cca99 100644 --- a/docs/en/contribute/documenting-code.rst +++ b/docs/en/contribute/documenting-code.rst @@ -513,6 +513,16 @@ As well as wildcards:: Note that this is a feature intended to simply testing and debugging during writing of documentation. The HTML output won't be perfect, i.e. it will not build a proper index that lists all the documents, and any references to documents that are not built will result in warnings. +Fast build +"""""""""" +Another trick to speed up building is to skip including doxygen generated API documention into the Sphinx build process, skipping this drastically reduces build times. + +This can be achieved by adding the fast-build argument:: + + ./build_docs.py build -f + +or by setting the environment variable `DOCS_FAST_BUILD`. Note that the the `-f` argument is a subargument to `build` and thus must be listed after `build`. + Building PDF """""""""""" diff --git a/docs/idf_extensions/run_doxygen.py b/docs/idf_extensions/run_doxygen.py index 5e96b009b9..8c0b1a0f19 100644 --- a/docs/idf_extensions/run_doxygen.py +++ b/docs/idf_extensions/run_doxygen.py @@ -73,6 +73,8 @@ def convert_api_xml_to_inc(app, doxyfiles): xml_directory_path = '{}/xml'.format(build_dir) inc_directory_path = '{}/inc'.format(build_dir) + fast_build = os.environ.get('DOCS_FAST_BUILD', None) + if not os.path.isdir(xml_directory_path): raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path)) @@ -96,6 +98,13 @@ def convert_api_xml_to_inc(app, doxyfiles): with open(inc_file_path, 'w', encoding='utf-8') as inc_file: inc_file.write(rst_output) + # For fast builds we wipe the doxygen api documention. + # Parsing this output during the sphinx build process is + # what takes 95% of the build time + if fast_build: + with open(inc_file_path, 'w', encoding='utf-8') as inc_file: + inc_file.write('') + def get_doxyfile_input_paths(app, doxyfile_path): """Get contents of Doxyfile's INPUT statement.