2017-12-04 16:34:43 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
"""
|
|
|
|
Freeze the conda environment.yml
|
|
|
|
|
2018-02-07 00:54:50 +00:00
|
|
|
It runs the freeze in a continuumio/miniconda3 image to ensure portability
|
2018-02-01 13:28:41 +00:00
|
|
|
|
|
|
|
Usage:
|
|
|
|
|
|
|
|
python freeze.py [3.5]
|
2017-12-04 16:34:43 +00:00
|
|
|
"""
|
|
|
|
|
|
|
|
from datetime import datetime
|
|
|
|
import os
|
|
|
|
import pathlib
|
2018-01-03 15:37:40 +00:00
|
|
|
import shutil
|
2017-12-04 16:34:43 +00:00
|
|
|
from subprocess import check_call
|
2018-02-01 13:28:41 +00:00
|
|
|
import sys
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
from ruamel.yaml import YAML
|
|
|
|
|
|
|
|
|
2018-09-07 16:05:33 +00:00
|
|
|
# Docker image version can be different than conda version,
|
|
|
|
# since miniconda3 docker images seem to lag conda releases.
|
|
|
|
MINICONDA_DOCKER_VERSION = '4.5.4'
|
|
|
|
CONDA_VERSION = '4.5.11'
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
|
|
|
ENV_FILE = 'environment.yml'
|
2018-01-03 15:37:40 +00:00
|
|
|
FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml'
|
|
|
|
|
|
|
|
ENV_FILE_T = 'environment.py-{py}.yml'
|
|
|
|
FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml'
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
yaml = YAML(typ='rt')
|
|
|
|
|
|
|
|
|
|
|
|
def freeze(env_file, frozen_file):
|
|
|
|
"""Freeze a conda environment.yml
|
|
|
|
|
|
|
|
By running in docker:
|
|
|
|
|
|
|
|
conda env create
|
|
|
|
conda env export
|
|
|
|
|
|
|
|
Result will be stored in frozen_file
|
|
|
|
"""
|
2018-02-01 13:28:41 +00:00
|
|
|
print(f"Freezing {env_file} -> {frozen_file}")
|
2017-12-04 16:34:43 +00:00
|
|
|
|
|
|
|
with open(HERE / frozen_file, 'w') as f:
|
|
|
|
f.write(f"# AUTO GENERATED FROM {env_file}, DO NOT MANUALLY MODIFY\n")
|
|
|
|
f.write(f"# Frozen on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
|
|
|
|
|
|
|
|
check_call([
|
|
|
|
'docker',
|
|
|
|
'run',
|
|
|
|
'--rm',
|
|
|
|
'-v' f"{HERE}:/r2d",
|
|
|
|
'-it',
|
2018-09-07 16:05:33 +00:00
|
|
|
f"continuumio/miniconda3:{MINICONDA_DOCKER_VERSION}",
|
2017-12-04 16:34:43 +00:00
|
|
|
"sh", "-c",
|
|
|
|
'; '.join([
|
2018-09-07 23:01:51 +00:00
|
|
|
'set -e',
|
2018-09-07 16:05:33 +00:00
|
|
|
f"conda install -yq conda={CONDA_VERSION}",
|
2017-12-04 16:34:43 +00:00
|
|
|
'conda config --add channels conda-forge',
|
|
|
|
'conda config --system --set auto_update_conda false',
|
|
|
|
f"conda env create -v -f /r2d/{env_file} -n r2d",
|
2018-01-08 16:07:46 +00:00
|
|
|
# add conda-forge broken channel as lowest priority in case
|
|
|
|
# any of our frozen packages are marked as broken after freezing
|
|
|
|
'conda config --append channels conda-forge/label/broken',
|
2018-01-03 15:37:40 +00:00
|
|
|
f"conda env export -n r2d >> /r2d/{frozen_file}",
|
2017-12-04 16:34:43 +00:00
|
|
|
])
|
|
|
|
])
|
|
|
|
|
|
|
|
|
2018-01-03 15:37:40 +00:00
|
|
|
def set_python(py_env_file, py):
|
|
|
|
"""Set the Python version in an env file"""
|
|
|
|
if os.path.exists(py_env_file):
|
|
|
|
# only clobber auto-generated files
|
|
|
|
with open(py_env_file) as f:
|
|
|
|
text = f.read()
|
|
|
|
if text and 'GENERATED' not in text:
|
|
|
|
return
|
2018-02-01 13:28:41 +00:00
|
|
|
|
|
|
|
print(f"Regenerating {py_env_file} from {ENV_FILE}")
|
2018-01-03 15:37:40 +00:00
|
|
|
with open(ENV_FILE) as f:
|
|
|
|
env = yaml.load(f)
|
|
|
|
for idx, dep in enumerate(env['dependencies']):
|
|
|
|
if dep.split('=')[0] == 'python':
|
|
|
|
env['dependencies'][idx] = f'python={py}.*'
|
|
|
|
break
|
|
|
|
else:
|
|
|
|
raise ValueError(f"python dependency not found in {env['dependencies']}")
|
|
|
|
# update python dependency
|
|
|
|
with open(py_env_file, 'w') as f:
|
|
|
|
f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n")
|
|
|
|
f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
|
|
|
|
yaml.dump(env, f)
|
|
|
|
|
|
|
|
|
2017-12-04 16:34:43 +00:00
|
|
|
if __name__ == '__main__':
|
2018-02-01 13:28:41 +00:00
|
|
|
# allow specifying which Pythons to update on argv
|
2018-09-17 08:29:59 +00:00
|
|
|
pys = sys.argv[1:] or ('2.7', '3.7', '3.5', '3.6')
|
2018-02-01 13:28:41 +00:00
|
|
|
for py in pys:
|
2018-01-03 15:37:40 +00:00
|
|
|
env_file = ENV_FILE_T.format(py=py)
|
|
|
|
set_python(env_file, py)
|
|
|
|
frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml'
|
|
|
|
freeze(env_file, frozen_file)
|
2018-01-08 16:07:46 +00:00
|
|
|
|
2018-01-03 15:37:40 +00:00
|
|
|
# use last version as default
|
|
|
|
shutil.copy(frozen_file, FROZEN_FILE)
|