ipydrawio/scripts/project.py

""" important project paths

    this should not import anything not in py36+ stdlib, or any local paths
"""

# Copyright 2022 ipydrawio contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import json
import os
import platform
import pprint
import re
import shutil
import subprocess
import sys
from collections import defaultdict
from pathlib import Path

print_ = pprint.pprint
console = None

try:
    import rich.console
    import rich.markdown

    console = rich.console.Console()
    print_ = console.print

except ImportError:
    pass

LITE_PREFIX = None
SOURCE_DATE_EPOCH = None

try:
    __import__("jupyterlite.manager")
    LITE_PREFIX = "demo_"
    SOURCE_DATE_EPOCH = (
        subprocess.check_output(["git", "log", "-1", "--format=%ct"])
        .decode("utf-8")
        .strip()
    )
except (ImportError, AttributeError, subprocess.CalledProcessError) as err:
    print_(err)
    pass


SKIPS = ["checkpoint", "pytest_cache", "patched-environment"]


def _clean(*paths_or_globs):
    cleaned = []
    for p_or_g in paths_or_globs:
        paths = [p_or_g] if isinstance(p_or_g, Path) else [*p_or_g]
        for p in paths:
            str_p = str(p)
            skipped = False
            for skip in SKIPS:
                if skip in str_p:
                    skipped = True
            if not skipped:
                cleaned += [p]
    return cleaned


def delete_some(*paths_or_globs):
    for p_or_g in paths_or_globs:
        paths = [p_or_g] if isinstance(p_or_g, Path) else [*p_or_g]
        for p in paths:
            if p.is_dir():
                shutil.rmtree(p)
            elif p.exists():
                p.unlink()


_SESSION = None


# platform
PLATFORM = os.environ.get("FAKE_PLATFORM", platform.system())
WIN = PLATFORM == "Windows"
OSX = PLATFORM == "Darwin"
UNIX = not WIN
PREFIX = Path(sys.prefix)
ENC = dict(encoding="utf-8")
JSON_FMT = dict(indent=2, sort_keys=True)

BUILDING_IN_CI = bool(json.loads(os.environ.get("BUILDING_IN_CI", "0")))
TESTING_IN_CI = bool(json.loads(os.environ.get("TESTING_IN_CI", "0")))
CI_ARTIFACT = os.environ.get("CI_ARTIFACT", "wheel")
CI = bool(json.loads(os.environ.get("CI", "0")))
RTD = bool(json.loads(os.environ.get("READTHEDOCS", "0").lower()))


# test arg pass-throughs
ATEST_ARGS = json.loads(os.environ.get("ATEST_ARGS", "[]"))
ATEST_RETRIES = int(os.environ.get("ATEST_RETRIES") or "0")
PYTEST_ARGS = json.loads(os.environ.get("PYTEST_ARGS", "[]"))
ATEST_PROCS = int(os.environ.get("ATEST_PROCS", "4"))

# find root
SCRIPTS = Path(__file__).parent.resolve()
ROOT = SCRIPTS.parent
PY_MAJOR = "".join(map(str, sys.version_info[:2]))

# demo
BINDER = ROOT / ".binder"
OVERRIDES = BINDER / "overrides.json"
POSTBUILD_PY = BINDER / "postBuild"
ENV_BINDER = BINDER / "environment.yml"

# top-level stuff
NODE_MODULES = ROOT / "node_modules"
PACKAGE = ROOT / "package.json"
PACKAGES = ROOT / "packages"
YARN_INTEGRITY = NODE_MODULES / ".yarn-integrity"
YARN_LOCK = ROOT / "yarn.lock"
DODO = ROOT / "dodo.py"
BUILD = ROOT / "build"
DIST = ROOT / "dist"
DOCS = ROOT / "docs"
README = ROOT / "README.md"
CHANGELOG = ROOT / "CHANGELOG.md"
SETUP_CFG = ROOT / "setup.cfg"

# external URLs
# archive.org template
CACHE_EPOCH = 0
HTTP_CACHE = BUILD / ".requests-cache"


def A_O(archive_id, url, cache_bust=CACHE_EPOCH):
    return "https://web.archive.org/web/{}/{}#{}".format(archive_id, url, cache_bust)


DIA_FAQ = "https://www.diagrams.net/doc/faq"

FETCHED = BUILD / "fetched"

DIA_URLS = {
    FETCHED
    / "supported-url-parameters.html": (
        A_O(20210425055302, f"{DIA_FAQ}/supported-url-parameters")
    ),
    FETCHED / "embed-mode.html": (A_O(20200924053756, f"{DIA_FAQ}/embed-mode")),
    FETCHED
    / "configure-diagram-editor.html": (
        A_O(20210503071537, f"{DIA_FAQ}/configure-diagram-editor")
    ),
}


# ci
GH = ROOT / ".github"
CI_YML = GH / "workflows/ci.yml"
ENV_GH = GH / "environment.yml"
ENV_GH_CB = GH / "environment-conda-build.yml"
ENV_GH_CB_WIN = GH / "environment-conda-build-win.yml"

# tools
PY = ["python"]
PYM = [*PY, "-m"]
PIP = [*PYM, "pip"]
PIP_CHECK_IGNORE = "^(No broken|pylint) "

NPM = (
    shutil.which("npm")
    or shutil.which("npm.cmd")
    or shutil.which("npm.exe")
    or shutil.which("npm.bat")
)
JLPM = ["jlpm"]
LERNA = [*JLPM, "lerna"]
JLPM_INSTALL = [*JLPM, "--ignore-optional", "--prefer-offline"]
LAB_EXT = ["jupyter", "labextension"]
LAB = ["jupyter", "lab"]
PRETTIER = [str(NODE_MODULES / ".bin" / "prettier")]

# tests
EXAMPLES = ROOT / "notebooks"
EXAMPLE_IPYNB = _clean(EXAMPLES.rglob("*.ipynb"))

DIST_NBHTML = DIST / "nbsmoke"
ATEST = ROOT / "atest"
ATEST_DIO = _clean(ATEST.rglob("*.dio"), ATEST.rglob("*.dio.svg"))
ATEST_OUT = BUILD / "atest"
ATEST_OUT_XML = "output.xml"
ATEST_TEMPLATES = [*ATEST.rglob("*.robot.j2")]

# js packages
JS_NS = "deathbeds"
IPYDIO = PACKAGES / "ipydrawio"
TSCONFIGBASE = PACKAGES / "tsconfigbase.json"
TSCONFIG_TYPEDOC = PACKAGES / "tsconfig.typedoc.json"
TYPEDOC_JSON = PACKAGES / "typedoc.json"
TYPEDOC_CONF = [TSCONFIG_TYPEDOC, TYPEDOC_JSON]
NO_TYPEDOC = ["_meta", "ipydrawio-webpack"]

# so many js packages
JS_PKG_JSON = {p.parent.name: p for p in PACKAGES.glob("*/package.json")}

JS_PKG_DATA = {k: json.loads(v.read_text(**ENC)) for k, v in JS_PKG_JSON.items()}

JS_PKG_JSON_LABEXT = {
    k: v
    for k, v in JS_PKG_JSON.items()
    if JS_PKG_DATA[k].get("jupyterlab", {}).get("extension")
}

JS_LABEXT_PY_HOST = {
    k: JS_PKG_DATA[k]["jupyterlab"]["discovery"]["server"]["base"]["name"]
    for k, v in JS_PKG_JSON.items()
    if JS_PKG_DATA[k].get("jupyterlab", {}).get("discovery")
}

JS_PKG_NOT_META = {k: v for k, v in JS_PKG_JSON.items() if k.startswith("_")}


def _norm_js_version(pkg):
    """undo some package weirdness"""
    v = pkg["version"]
    final = ""
    # alphas, beta use dashes
    for dashed in v.split("-"):
        if final:
            final += "-"
        for dotted in dashed.split("."):
            if final:
                final += "."
            if re.findall(r"^\d+$", dotted):
                final += str(int(dotted))
            else:
                final += dotted
    return final


JS_TARBALL = {
    k: JS_PKG_JSON[k].parent
    / f"""{v["name"].replace('@', '').replace("/", "-")}-{_norm_js_version(v)}.tgz"""
    for k, v in JS_PKG_DATA.items()
    if k not in JS_PKG_NOT_META
}

JS_TSCONFIG = {
    k: v.parent / "src/tsconfig.json"
    for k, v in JS_PKG_JSON.items()
    if (v.parent / "src/tsconfig.json").exists()
}

JS_TSSRC = {
    k: sorted(
        [
            *(v.parent).rglob("*.ts"),
            *(v.parent / "src").rglob("*.tsx"),
        ]
    )
    for k, v in JS_TSCONFIG.items()
}

JS_TSBUILDINFO = {
    k: v.parent.parent / ".src.tsbuildinfo" for k, v in JS_TSCONFIG.items()
}

JS_STYLE = {
    k: sorted((v.parent / "style").glob("*.css"))
    for k, v in JS_PKG_JSON.items()
    if (v.parent / "style").exists()
}

JS_PY_SCRIPTS = {
    k: sorted((v.parent / "scripts").glob("*.py"))
    for k, v in JS_PKG_JSON.items()
    if (v.parent / "scripts").exists()
}

JS_SCHEMAS = {
    k: sorted((v.parent / "schema").glob("*.json"))
    for k, v in JS_PKG_JSON.items()
    if (v.parent / "schema").exists()
}

# special things for ipydrawio-webpack
IPDWP = JS_PKG_JSON["ipydrawio-webpack"].parent
IPDWP_APP = IPDWP / "dio/js/app.min.js"
IPDWP_PY = (IPDWP / "scripts").rglob("*.py")
DRAWIO = IPDWP / "drawio"
IPDWP_LIB = IPDWP / "lib"
IPDWP_IGNORE = IPDWP / ".npmignore"
ALL_IPDWP_JS = IPDWP_LIB.glob("*.js")

IPJT = JS_PKG_JSON["ipydrawio-jupyter-templates"].parent
IPJT_TMPL = IPJT / "tmpl"
IPJT_TMPL_DIO = _clean(
    IPJT_TMPL.rglob("*.dio"),
    IPJT_TMPL.rglob("*.dio.svg"),
)

PY_PACKAGES = ROOT / "py_packages"

PY_SETUP = {p.parent.name: p for p in sorted((ROOT / "py_packages").glob("*/setup.py"))}
PY_SRC = {k: sorted((v.parent / "src").rglob("*.py")) for k, v in PY_SETUP.items()}
PY_SETUP_CFG = {k: v.parent / "setup.cfg" for k, v in PY_SETUP.items()}

PY_VERSION = {
    "ipydrawio": JS_PKG_DATA["ipydrawio"]["version"],
    "ipydrawio-export": JS_PKG_DATA["ipydrawio-pdf"]["version"],
    "ipydrawio-widgets": JS_PKG_DATA["ipydrawio"]["version"],
}

IPD = PY_SETUP["ipydrawio"].parent
IPDE = PY_SETUP["ipydrawio-export"].parent
IPDW = PY_SETUP["ipydrawio-widgets"].parent

IPDE_VENDOR = IPDE / "src/ipydrawio_export/vendor"
IPDE_DIE2 = IPDE_VENDOR / "draw-image-export2"
IPDE_DIE2_PACKAGE_JSON = IPDE_DIE2 / "package.json"
IPDE_DIE2_YARN_LOCK = IPDE_DIE2 / "yarn.lock"

IPDW_SRC = IPDW / "src/ipydrawio_widgets"
IPDW_DEPS = {
    JS_PKG_JSON["ipydrawio"]: IPDW_SRC / "js/package.json",
    JS_PKG_JSON["ipydrawio"].parent / "schema/plugin.json": IPDW_SRC / "js/plugin.json",
}

IPD_VERSION = PY_VERSION["ipydrawio"]
IPDW_VERSION = PY_VERSION["ipydrawio-widgets"]
IPDE_VERSION = PY_VERSION["ipydrawio-export"]

PY_HAS_EXT = {
    IPD: True,
    IPDE: True,
    IPDW: False,
}

PY_SETUP_DEPS = {
    IPD: lambda: [OK_PYSETUP["ipydrawio-widgets"]],
    IPDE: lambda: [OK_PYSETUP["ipydrawio"]],
    IPDW: lambda: [*IPDW_DEPS.values()],
}

PY_SDIST = {
    IPDE.name: IPDE / "dist" / f"{IPDE.name}-{IPDE_VERSION}.tar.gz",
    IPD.name: IPD / "dist" / f"{IPD.name}-{IPD_VERSION}.tar.gz",
    IPDW.name: IPDW / "dist" / f"{IPDW.name}-{IPDW_VERSION}.tar.gz",
}
PY_WHEEL = {
    IPDE.name: IPDE
    / "dist"
    / f"""{IPDE.name.replace("-", "_")}-{IPDE_VERSION}-py3-none-any.whl""",
    IPD.name: IPD
    / "dist"
    / f"""{IPD.name.replace("-", "_")}-{IPD_VERSION}-py3-none-any.whl""",
    IPDW.name: IPDW
    / "dist"
    / f"""{IPDW.name.replace("-", "_")}-{IPDW_VERSION}-py3-none-any.whl""",
}
PY_TEST_DEP = {}

SERVER_EXT = {
    k: sorted(v.parent.glob("src/*/serverextension.py"))[0]
    for k, v in PY_SETUP.items()
    if sorted(v.parent.glob("src/*/serverextension.py"))
}

# demo
RE_CONDA_FORGE_URL = r"/conda-forge/(.*/)?(noarch|linux-64|win-64|osx-64)/([^/]+)$"
CONDA_FORGE_RELEASE = "https://conda.anaconda.org/conda-forge"
FED_EXT_MARKER = "### FEDERATED EXTENSIONS ###"
DEMO = ROOT / "demo"
DEMO_FILES = DEMO / "files"
DEMO_CONFIG = DEMO / "jupyter_lite_config.json"
DEMO_REQS = DEMO / "requirements.txt"
DEMO_APPS = ["lab"]
DEMO_BUILD = BUILD / "demo"
DEMO_HASHES = DEMO_BUILD / "SHA256SUMS"
DEMO_CONTENTS_API = DEMO_BUILD / "api/contents/all.json"
DEMO_ARCHIVE = (
    DEMO_BUILD / f"""ipydrawio-lite-{JS_PKG_DATA["ipydrawio"]["version"]}.tgz"""
)
BUILD_WHEELS = BUILD / "wheels"
DEMO_WHEELS = DEMO / "pypi"
NOARCH_WHL = "py3-none-any.whl"
IGNORED_WHEELS = ["widgetsnbextension", "nbformat", "ipykernel", "pyolite"]
PYODIDE_URL = "https://cdn.jsdelivr.net/pyodide/v0.18.1/full"
PYODIDE_PACKAGES = BUILD / "pyodide-packages.json"


# docs
SPHINX_ARGS = json.loads(os.environ.get("SPHINX_ARGS", "[]"))
DOCS_CONF = DOCS / "conf.py"
ENV_DOCS = DOCS / "environment.yml"
DOCS_BUILD = BUILD / "docs"
DOCS_BUILDINFO = DOCS_BUILD / ".buildinfo"
DOCS_MD = _clean(
    [
        p
        for p in DOCS.rglob("*.md")
        if not (p.parent.name == "ts" or p.parent.parent.name == "ts")
    ]
)
DOCS_DIO = _clean(DOCS.rglob("*.dio"), DOCS.rglob("*.dio.svg"))
DOCS_RST = _clean(DOCS.rglob("*.rst"))
DOCS_IPYNB = _clean(DOCS.rglob("*.ipynb"))
DOCS_SRC = _clean(DOCS_MD, DOCS_RST, DOCS_IPYNB)
DOCS_STATIC = DOCS / "_static"
DOCS_FAVICON_SVG = DOCS_STATIC / "icon.svg"
DOCS_FAVICON_ICO = DOCS_STATIC / "favicon.ico"
DOCS_TS = DOCS / "api/ts"
DOCS_TS_MYST_INDEX = DOCS_TS / "index.md"
DOCS_TS_MODULES = [
    ROOT / "docs/api/ts" / f"{p.parent.name}.md"
    for p in JS_PKG_JSON.values()
    if p.parent.name not in NO_TYPEDOC
]

DOCS_RAW_TYPEDOC = BUILD / "typedoc"
DOCS_RAW_TYPEDOC_README = DOCS_RAW_TYPEDOC / "README.md"
MD_FOOTER = """
```
Copyright 2022 ipydrawio contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
```
"""

# collections, mostly for linting
ALL_PY = [
    *ATEST.rglob("*.py"),
    *BINDER.glob("*.py"),
    *IPDWP_PY,
    *PY_SETUP.values(),
    *SCRIPTS.glob("*.py"),
    *sum(JS_PY_SCRIPTS.values(), []),
    *sum(PY_SRC.values(), []),
    DODO,
    POSTBUILD_PY,
    DOCS_CONF,
]
ALL_YML = _clean(
    ROOT.glob("*.yml"),
    GH.rglob("*.yml"),
    BINDER.glob("*.yml"),
    DOCS.rglob("*.yml"),
)
ALL_JSON = [
    *ROOT.glob("*.json"),
    *PACKAGES.glob("*/*.json"),
    *PACKAGES.glob("*/schema/*.json"),
    *ATEST.glob("fixtures/*.json"),
    *BINDER.glob("*.json"),
    *[
        p
        for p in DEMO.rglob("*.json")
        if "/_output/" not in str(p)
        and "/.cache/" not in str(p)
        and p not in [DEMO_CONFIG]
    ],
]
ALL_DIO = [*DOCS_DIO, *IPJT_TMPL_DIO, *ATEST_DIO]
ALL_MD = [*ROOT.glob("*.md"), *PACKAGES.glob("*/*.md"), *DOCS_MD]
ALL_SETUP_CFG = [SETUP_CFG, *PY_SETUP_CFG.values()]
ALL_JS = [PACKAGES / ".eslintrc.js"]
ALL_TS = sum(JS_TSSRC.values(), [])
ALL_CSS = [*sum(JS_STYLE.values(), []), *DOCS.rglob("*.css")]
ALL_ROBOT = [*ATEST.rglob("*.robot")]
ALL_PRETTIER = [*ALL_YML, *ALL_JSON, *ALL_MD, *ALL_TS, *ALL_CSS, *ALL_JS]
ALL_HEADERS = _clean(
    ALL_SETUP_CFG,
    ALL_PY,
    ALL_TS,
    ALL_CSS,
    ALL_JS,
    ALL_MD,
    ALL_YML,
    ALL_ROBOT,
)

ALL_DEMO_SOURCES = [
    d
    for d in ALL_DIO
    if "test" not in str(d.relative_to(ROOT)).lower()
    and ".doit" not in d.name
    and " " not in d.name
    and d.name not in ["A.dio"]
] + [*DOCS.glob("*.ipynb")]

ALL_DEMO_CONTENTS = {p: DEMO_FILES / p.name for p in sorted(ALL_DEMO_SOURCES)}

ESLINTRC = PACKAGES / ".eslintrc.js"

RFLINT_OPTS = sum(
    [
        ["--ignore", c]
        for c in [
            "FileTooLong",
            "LineTooLong",
            "RequireKeywordDocumentation",
            "RequireKeywordDocumentation",
            "TooFewKeywordSteps",
            "TooFewTestSteps",
            "TooManyTestSteps",
        ]
    ],
    [],
)

# package: [dependencies, targets]
JS_PKG_PACK = {
    k: [[v.parent / "package.json", *v.parent.glob("schema/*.json")], [v]]
    for k, v in JS_TARBALL.items()
}
[
    JS_PKG_PACK[k][0].append(v)
    for k, v in JS_TSBUILDINFO.items()
    if not k.startswith("_")
]
JS_PKG_PACK[IPDWP.name][0] += [
    IPDWP_IGNORE,
    IPDWP_APP,
    *ALL_IPDWP_JS,
]

# provisioning stuff
IPYDRAWIO_DATA_DIR = Path(sys.prefix) / "share/jupyter/ipydrawio_export"

# built files
OK_PIP_CHECK = BUILD / "pip.check.ok"
OK_INTEGRITY = BUILD / "integrity.ok"
OK_SUBMODULES = BUILD / "submodules.ok"
OK_BLACK = BUILD / "black.ok"
OK_FLAKE8 = BUILD / "flake8.ok"
OK_ISORT = BUILD / "isort.ok"
OK_LINT = BUILD / "lint.ok"
OK_ROBOTIDY = BUILD / "robot.tidy.ok"
OK_PYFLAKES = BUILD / "pyflakes.ok"
OK_PRETTIER = BUILD / "prettier.ok"
OK_ESLINT = BUILD / "eslint.ok"
OK_JS_BUILD_PRE = BUILD / "js.build.pre.ok"
OK_JS_BUILD = BUILD / "js.build.ok"
OK_PYSETUP = {k: BUILD / f"pysetup.{k}.ok" for k, v in PY_SETUP.items()}
OK_PYTEST = {k: BUILD / f"pytest.{k}.ok" for k, v in PY_SETUP.items()}
OK_SERVEREXT = {k: BUILD / f"serverext.{k}.ok" for k, v in SERVER_EXT.items()}
OK_PROVISION = BUILD / "provision.ok"
OK_ROBOT_DRYRUN = BUILD / "robot.dryrun.ok"
OK_RFLINT = BUILD / "robot.rflint.ok"
OK_DIOLINT = BUILD / "dio.lint.ok"
OK_ATEST = BUILD / "atest.ok"
OK_CONDA_TEST = BUILD / "conda-build.test.ok"
OK_LINK_CHECK = BUILD / "pytest-check-links.ok"

OK_EXT_BUILD = {k: BUILD / f"ext.build.{k}.ok" for k in JS_LABEXT_PY_HOST}

PY_TEST_DEP.setdefault("ipydrawio-export", []).append(OK_PROVISION)

HASH_DEPS = [*PY_SDIST.values(), *PY_WHEEL.values(), *JS_TARBALL.values()]
SHA256SUMS = DIST / "SHA256SUMS"

# built artifacts
EXAMPLE_HTML = [DIST_NBHTML / p.name.replace(".ipynb", ".html") for p in EXAMPLE_IPYNB]

CMD_LIST_EXTENSIONS = ["jupyter", "labextension", "list"]

CMD_LAB = ["jupyter", "lab", "--no-browser", "--debug"]

# conda building
RECIPE = ROOT / "conda.recipe/meta.yaml"
CONDA_BLD = BUILD / "conda-bld"
CONDARC = GH / ".condarc"
# could be mambabuild
CONDA_BUILDERER = os.environ.get("CONDA_BUILDERER", "build")
CONDA_BUILD_ARGS = [
    "conda",
    CONDA_BUILDERER,
    "--override-channels",
    "-c",
    "conda-forge",
    "-c",
    "nodefaults",
]
CONDA_PKGS = {
    pkg: CONDA_BLD / f"noarch/{pkg}-{ver}-py_0.tar.bz2"
    for pkg, ver in PY_VERSION.items()
}

# env inheritance
ENV_INHERITS = {
    ENV_BINDER: [ENV_GH, ENV_DOCS],
    ENV_DOCS: [ENV_GH],
    ENV_GH_CB_WIN: [ENV_GH_CB],
}


def get_atest_stem(attempt=1, extra_args=None, browser=None):
    """get the directory in ATEST_OUT for this platform/apps"""
    browser = browser or "headlessfirefox"
    extra_args = extra_args or []

    stem = f"{PLATFORM}_{PY_MAJOR}_{browser}_{attempt}"

    if "--dryrun" in extra_args:
        stem += "_dry_run"

    return stem


def ensure_session():
    global _SESSION

    if _SESSION is None:
        try:
            import requests_cache

            _SESSION = requests_cache.CachedSession(cache_name=str(HTTP_CACHE))
        except ImportError:
            import requests

            _SESSION = requests.Session()


def fetch_one(url, path):
    import doit

    yield dict(
        uptodate=[doit.tools.config_changed({"url": url})],
        name=path.name,
        actions=[
            (doit.tools.create_folder, [HTTP_CACHE]),
            (doit.tools.create_folder, [path.parent]),
            (ensure_session, []),
            lambda: [path.write_bytes(_SESSION.get(url).content), None][-1],
        ],
        targets=[path],
    )


def patch_one_env(source, target):
    source_text = source.read_text(**ENC)
    name = re.findall(r"name: (.*)", source_text)[0]
    comment = f"  ### {name}-deps ###"
    old_target = target.read_text(**ENC).split(comment)
    new_source = source_text.split(comment)
    target.write_text(
        "\n".join(
            [
                old_target[0].strip(),
                comment,
                new_source[1],
                comment.rstrip(),
                old_target[2],
            ]
        ),
        **ENC,
    )


def typedoc_conf():
    typedoc = json.loads(TYPEDOC_JSON.read_text(**ENC))
    original_entry_points = sorted(typedoc["entryPoints"])
    new_entry_points = sorted(
        [
            str(
                (
                    p.parent / "src/index.ts"
                    if (p.parent / "src/index.ts").exists()
                    else p.parent / "lib/index.d.ts"
                )
                .relative_to(ROOT)
                .as_posix()
            )
            for p in JS_PKG_JSON.values()
            if p.parent.name not in NO_TYPEDOC
        ]
    )

    if json.dumps(original_entry_points) != json.dumps(new_entry_points):
        typedoc["entryPoints"] = new_entry_points
        TYPEDOC_JSON.write_text(json.dumps(typedoc, **JSON_FMT), **ENC)

    tsconfig = json.loads(TSCONFIG_TYPEDOC.read_text(**ENC))
    original_references = tsconfig["references"]
    new_references = sum(
        [
            [
                {"path": f"./{p.parent.name}/src"},
                {"path": f"./{p.parent.name}"},
            ]
            for p in JS_PKG_JSON.values()
            if p.parent.name not in NO_TYPEDOC
        ],
        [],
    )

    if json.dumps(original_references) != json.dumps(new_references):
        tsconfig["references"] = new_references
        TSCONFIG_TYPEDOC.write_text(json.dumps(tsconfig, **JSON_FMT), **ENC)


def mystify():
    """unwrap monorepo docs into per-module docs"""
    mods = defaultdict(lambda: defaultdict(list))
    if DOCS_TS.exists():
        shutil.rmtree(DOCS_TS)

    def mod_md_name(mod):
        return mod.replace("@jupyterlite/", "") + ".md"

    for doc in sorted(DOCS_RAW_TYPEDOC.rglob("*.md")):
        if doc.parent == DOCS_RAW_TYPEDOC:
            continue
        if doc.name == "README.md":
            continue
        doc_text = doc.read_text(**ENC)
        doc_lines = doc_text.splitlines()
        mod_chunks = doc_lines[0].split(" / ")
        src = mod_chunks[1]
        if src.startswith("["):
            src = re.findall(r"\[(.*)/src\]", src)[0]
        else:
            src = src.replace("/src", "")
        pkg = f"""@jupyterlite/{src.replace("/src", "")}"""
        mods[pkg][doc.parent.name] += [
            str(doc.relative_to(DOCS_RAW_TYPEDOC).as_posix())[:-3]
        ]

        # rewrite doc and write back out
        out_doc = DOCS_TS / doc.relative_to(DOCS_RAW_TYPEDOC)
        if not out_doc.parent.exists():
            out_doc.parent.mkdir(parents=True)

        out_text = "\n".join([*doc_lines[1:], ""]).replace("README.md", "index.md")
        out_text = re.sub(
            r"## Table of contents(.*?)\n## ",
            "\n## ",
            out_text,
            flags=re.M | re.S,
        )
        out_text = out_text.replace("/src]", "]")
        out_text = re.sub("/src$", "", out_text, flags=re.M)
        out_text = re.sub(
            r"^((Implementation of|Overrides|Inherited from):)",
            "_\\1_",
            out_text,
            flags=re.M | re.S,
        )
        out_text = re.sub(
            r"^Defined in: ([^\n]+)$",
            "_Defined in:_ `\\1`",
            out_text,
            flags=re.M | re.S,
        )

        out_text += MD_FOOTER

        out_doc.write_text(out_text, **ENC)

    for mod, sections in mods.items():
        out_doc = DOCS_TS / mod_md_name(mod)
        mod_lines = [f"""# `{mod.replace("@jupyterlite/", "")}`\n"""]
        for label, contents in sections.items():
            mod_lines += [
                f"## {label.title()}\n",
                "```{toctree}",
                ":maxdepth: 1",
                *contents,
                "```\n",
                MD_FOOTER,
            ]
        out_doc.write_text("\n".join(mod_lines), **ENC)

    DOCS_TS_MYST_INDEX.write_text(
        "\n".join(
            [
                "# `@deathbeds/ipydrawio`\n",
                "```{toctree}",
                ":maxdepth: 1",
                *[mod_md_name(mod) for mod in sorted(mods)],
                "```",
                MD_FOOTER,
            ]
        ),
        **ENC,
    )


def pip_check():
    proc = subprocess.Popen([*PIP, "check"], stdout=subprocess.PIPE)
    proc.wait()
    out = proc.stdout.read().decode("utf-8")
    print(out)
    lines = [
        line
        for line in out.splitlines()
        if line.strip() and not re.findall(PIP_CHECK_IGNORE, line)
    ]
    return not len(lines)


# utilities
def _echo_ok(msg):
    def _echo():
        print(msg, flush=True)
        return True

    return _echo


def _ok(task, ok):
    task.setdefault("targets", []).append(ok)
    task["actions"] = [
        lambda: [ok.exists() and ok.unlink(), True][-1],
        *task["actions"],
        lambda: [ok.parent.mkdir(exist_ok=True), ok.write_text("ok", **ENC), True][-1],
    ]
    return task


def _show(*args, **kwargs):
    import rich.markdown

    for arg in args:
        print_(arg()) if callable(arg) else print_(arg)
    for kw, kwarg in kwargs.items():
        print_(rich.markdown.Markdown(f"# {kw}") if console else kw)
        print_(kwarg()) if callable(kwarg) else print_(kwarg)


def _copy_one(src, dest):
    if not src.exists():
        return False
    if not dest.parent.exists():
        dest.parent.mkdir(parents=True)
    if dest.exists():
        if dest.is_dir():
            shutil.rmtree(dest)
        else:
            dest.unlink()
    if src.is_dir():
        shutil.copytree(src, dest)
    else:
        shutil.copy2(src, dest)


def _lite(lite_actions, extra_args=[]):
    lite = ["jupyter", "lite"]
    args = ["--source-date-epoch", SOURCE_DATE_EPOCH]

    try:
        from jupyter_server_mathjax.app import STATIC_ASSETS_PATH as MATHJAX_DIR
    except Exception:
        MATHJAX_DIR = None

    if MATHJAX_DIR:
        args += ["--mathjax-dir", str(MATHJAX_DIR)]

    for act in lite_actions:
        act_args = list(map(str, [*lite, act, *args, *extra_args]))
        if subprocess.call(act_args, cwd=DEMO) != 0:
            print("FAILED", *act_args)
            return False


def _sync_lite_config(from_env, to_json, marker, extra_federated, extra_pyolite):
    """use conda list to derive tarball names"""
    raw_lock = subprocess.check_output(["conda", "list", "--explicit"])
    ext_packages = [
        p.strip().split(" ")[0]
        for p in from_env.read_text(**ENC).split(marker)[1].split(" - ")
        if p.strip()
    ]

    federated_extensions = sorted(
        ["../" + str(extra.relative_to(ROOT).as_posix()) for extra in extra_federated]
    )
    piplite_urls = [
        "../" + str(extra.relative_to(ROOT).as_posix()) for extra in extra_pyolite
    ]

    for raw_url in sorted(raw_lock.decode("utf-8").splitlines()):
        try:
            label, subdir, pkg = re.findall(RE_CONDA_FORGE_URL, raw_url)[0]
        except IndexError:
            continue

        if label:
            # TODO: haven't looked into this
            continue

        for ext in ext_packages:
            if pkg.startswith(ext):
                federated_extensions += ["/".join([CONDA_FORGE_RELEASE, subdir, pkg])]

    config = json.loads(to_json.read_text(**ENC))

    config["LiteBuildConfig"].update(
        federated_extensions=federated_extensions,
        piplite_urls=piplite_urls,
    )

    to_json.write_text(json.dumps(config, **JSON_FMT), **ENC)

    subprocess.call([*PRETTIER, "--write", to_json])


def fetch_pyodide_packages():
    import urllib.request

    url = f"{PYODIDE_URL}/packages.json"
    print("fetching pyodide packages from", url)
    with urllib.request.urlopen(url) as response:
        packages = json.loads(response.read().decode("utf-8"))
    PYODIDE_PACKAGES.parent.mkdir(exist_ok=True, parents=True)
    PYODIDE_PACKAGES.write_text(json.dumps(packages, **JSON_FMT), **ENC)


def fetch_wheels():
    BUILD_WHEELS.mkdir(exist_ok=True, parents=True)
    DEMO_WHEELS.mkdir(exist_ok=True, parents=True)
    pyodide_pkgs = json.loads(PYODIDE_PACKAGES.read_text(encoding="utf-8"))
    pyodide_norm_names = [k.lower().replace("-", "_") for k in pyodide_pkgs["packages"]]
    subprocess.check_call(
        ["pip", "download", "-r", str(DEMO_REQS), "--prefer-binary"],
        cwd=str(BUILD_WHEELS),
    )
    for pkg in sorted(BUILD_WHEELS.glob("*")):
        norm_name = pkg.name.split("-")[0].lower()
        if not pkg.name.endswith(NOARCH_WHL):
            continue
        if norm_name in pyodide_norm_names or norm_name in IGNORED_WHEELS:
            continue
        dest = DEMO_WHEELS / pkg.name

        if dest.exists():
            if pkg.stat().st_mtime > dest.stat().st_mtime():
                dest.unlink()
            else:
                continue

        shutil.copy2(pkg, dest)


def template_one(src, dest):
    import jinja2

    template = jinja2.Template(src.read_text(**ENC))
    if not dest.parent.exists():
        dest.parent.mkdir()
    if dest.exists():
        dest.unlink()
    dest.write_text(template.render(P=globals()))


# Late environment hacks
os.environ.update(
    CONDARC=str(CONDARC),
    IPYDRAWIO_DATA_DIR=str(IPYDRAWIO_DATA_DIR),
    PIP_DISABLE_PIP_VERSION_CHECK="1",
)