2017-12-04 16:34:43 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
"""
|
|
|
|
Freeze the conda environment.yml
|
|
|
|
|
2021-03-03 10:18:27 +00:00
|
|
|
Using conda-lock
|
2018-02-01 13:28:41 +00:00
|
|
|
|
|
|
|
Usage:
|
|
|
|
|
2020-01-28 12:05:46 +00:00
|
|
|
python freeze.py [3.8]
|
2017-12-04 16:34:43 +00:00
|
|
|
"""
|
|
|
|
|
|
|
|
import os
|
|
|
|
import pathlib
|
2018-01-03 15:37:40 +00:00
|
|
|
import shutil
|
2018-02-01 13:28:41 +00:00
|
|
|
import sys
|
2022-10-31 22:32:14 +00:00
|
|
|
from argparse import ArgumentParser
|
|
|
|
from datetime import datetime
|
|
|
|
from subprocess import check_call
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
from ruamel.yaml import YAML
|
|
|
|
|
|
|
|
HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
ENV_FILE = HERE / "environment.yml"
|
2021-03-03 14:35:29 +00:00
|
|
|
FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + ".lock"
|
2018-01-03 15:37:40 +00:00
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
ENV_FILE_T = HERE / "environment.py-{py}.yml"
|
2017-12-04 16:34:43 +00:00
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
yaml = YAML(typ="rt")
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
|
2021-03-03 10:18:27 +00:00
|
|
|
def freeze(env_file, frozen_file, platform="linux-64"):
|
2021-03-03 14:35:29 +00:00
|
|
|
"""Freeze a conda environment
|
2017-12-04 16:34:43 +00:00
|
|
|
|
2021-03-03 14:35:29 +00:00
|
|
|
By running:
|
2017-12-04 16:34:43 +00:00
|
|
|
|
2021-03-03 10:18:27 +00:00
|
|
|
conda-lock --mamba --platform=linux-64 -f environment.yml
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
Result will be stored in frozen_file
|
|
|
|
"""
|
2018-12-11 09:43:01 +00:00
|
|
|
frozen_dest = HERE / frozen_file
|
2019-03-23 17:47:22 +00:00
|
|
|
|
2018-12-11 09:43:01 +00:00
|
|
|
if frozen_dest.exists():
|
2019-05-31 09:10:17 +00:00
|
|
|
with frozen_dest.open("r") as f:
|
2018-12-11 09:43:01 +00:00
|
|
|
line = f.readline()
|
2019-05-31 09:10:17 +00:00
|
|
|
if "GENERATED" not in line:
|
|
|
|
print(
|
|
|
|
f"{frozen_file.relative_to(HERE)} not autogenerated, not refreezing"
|
|
|
|
)
|
2018-12-11 09:43:01 +00:00
|
|
|
return
|
2018-02-01 13:28:41 +00:00
|
|
|
print(f"Freezing {env_file} -> {frozen_file}")
|
2017-12-04 16:34:43 +00:00
|
|
|
|
2021-03-03 10:18:27 +00:00
|
|
|
# FIXME: conda-lock 0.8 requires {platform} in template
|
|
|
|
# https://github.com/conda-incubator/conda-lock/pull/78
|
|
|
|
frozen_template = str(frozen_dest) + ".{platform}"
|
|
|
|
frozen_tempfile = pathlib.Path(frozen_template.format(platform=platform))
|
|
|
|
|
|
|
|
check_call(
|
|
|
|
[
|
|
|
|
"conda-lock",
|
|
|
|
# FIXME: adopt micromamba after ordering is fixed
|
|
|
|
# https://github.com/conda-incubator/conda-lock/issues/79
|
|
|
|
"--mamba",
|
2022-08-01 23:04:11 +00:00
|
|
|
"--kind=explicit",
|
2021-03-03 10:18:27 +00:00
|
|
|
f"--platform={platform}",
|
|
|
|
f"--filename-template={frozen_template}",
|
|
|
|
f"--file={env_file}",
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
with frozen_dest.open("w") as f:
|
|
|
|
f.write(
|
|
|
|
f"# AUTO GENERATED FROM {env_file.relative_to(HERE)}, DO NOT MANUALLY MODIFY\n"
|
|
|
|
)
|
2017-12-04 16:34:43 +00:00
|
|
|
f.write(f"# Frozen on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
|
2021-03-03 10:18:27 +00:00
|
|
|
with frozen_tempfile.open() as temp:
|
|
|
|
f.write(temp.read())
|
2017-12-04 16:34:43 +00:00
|
|
|
|
2021-03-03 10:18:27 +00:00
|
|
|
os.remove(frozen_tempfile)
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
|
2018-01-03 15:37:40 +00:00
|
|
|
def set_python(py_env_file, py):
|
|
|
|
"""Set the Python version in an env file"""
|
|
|
|
if os.path.exists(py_env_file):
|
|
|
|
# only clobber auto-generated files
|
|
|
|
with open(py_env_file) as f:
|
2018-12-11 09:43:01 +00:00
|
|
|
text = f.readline()
|
2019-05-31 09:10:17 +00:00
|
|
|
if "GENERATED" not in text:
|
2018-01-03 15:37:40 +00:00
|
|
|
return
|
2018-02-01 13:28:41 +00:00
|
|
|
|
|
|
|
print(f"Regenerating {py_env_file} from {ENV_FILE}")
|
2018-01-03 15:37:40 +00:00
|
|
|
with open(ENV_FILE) as f:
|
|
|
|
env = yaml.load(f)
|
2019-05-31 09:10:17 +00:00
|
|
|
for idx, dep in enumerate(env["dependencies"]):
|
|
|
|
if dep.split("=")[0] == "python":
|
|
|
|
env["dependencies"][idx] = f"python={py}.*"
|
2018-01-03 15:37:40 +00:00
|
|
|
break
|
|
|
|
else:
|
|
|
|
raise ValueError(f"python dependency not found in {env['dependencies']}")
|
|
|
|
# update python dependency
|
2019-05-31 09:10:17 +00:00
|
|
|
with open(py_env_file, "w") as f:
|
|
|
|
f.write(
|
|
|
|
f"# AUTO GENERATED FROM {ENV_FILE.relative_to(HERE)}, DO NOT MANUALLY MODIFY\n"
|
|
|
|
)
|
2018-01-03 15:37:40 +00:00
|
|
|
f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
|
|
|
|
yaml.dump(env, f)
|
|
|
|
|
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
if __name__ == "__main__":
|
2021-12-23 14:45:41 +00:00
|
|
|
parser = ArgumentParser(
|
|
|
|
description=(
|
|
|
|
"Refreeze conda environments. See "
|
|
|
|
"https://repo2docker.readthedocs.io/en/latest/contributing/tasks.html#update-and-freeze-buildpack-dependencies"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"py",
|
|
|
|
nargs="*",
|
|
|
|
help="Python version(s) to update and freeze",
|
2022-08-22 13:09:02 +00:00
|
|
|
default=("3.7", "3.8", "3.9", "3.10"),
|
2021-12-23 14:45:41 +00:00
|
|
|
)
|
|
|
|
args = parser.parse_args()
|
2020-01-28 12:05:46 +00:00
|
|
|
default_py = "3.7"
|
2021-12-23 14:45:41 +00:00
|
|
|
for py in args.py:
|
2019-03-23 16:46:33 +00:00
|
|
|
env_file = pathlib.Path(str(ENV_FILE_T).format(py=py))
|
2018-01-03 15:37:40 +00:00
|
|
|
set_python(env_file, py)
|
2021-03-03 14:35:29 +00:00
|
|
|
frozen_file = pathlib.Path(os.path.splitext(env_file)[0] + ".lock")
|
2018-01-03 15:37:40 +00:00
|
|
|
freeze(env_file, frozen_file)
|
2020-01-28 12:05:46 +00:00
|
|
|
if py == default_py:
|
|
|
|
shutil.copy(frozen_file, FROZEN_FILE)
|