From 3a6ab0b46d6471bee00ae815444c624709dd4cdd Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Thu, 27 May 2021 16:57:06 +1000 Subject: [PATCH] top: Remove upip-related scripts. Signed-off-by: Jim Mussared --- Makefile | 16 ----- make_metadata.py | 182 ----------------------------------------------- optimize_upip.py | 123 -------------------------------- sdist_upip.py | 141 ------------------------------------ 4 files changed, 462 deletions(-) delete mode 100644 Makefile delete mode 100755 make_metadata.py delete mode 100644 optimize_upip.py delete mode 100644 sdist_upip.py diff --git a/Makefile b/Makefile deleted file mode 100644 index 7ae08835..00000000 --- a/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -PREFIX = ~/.micropython/lib - -all: - -# Installs all modules to a lib location, for development testing -CMD="find . -maxdepth 1 -mindepth 1 \( -name '*.py' -not -name 'test_*' -not -name 'setup.py' \) -or \( -type d -not -name 'dist' -not -name '*.egg-info' -not -name '__pycache__' \)| xargs --no-run-if-empty cp -r -t $(PREFIX)" -install: - @mkdir -p $(PREFIX) - @if [ -n "$(MOD)" ]; then \ - (cd $(MOD); sh -c $(CMD)); \ - else \ - for d in $$(find -maxdepth 1 -type d ! -name ".*"); do \ - echo $$d; \ - (cd $$d; sh -c $(CMD)); \ - done \ - fi diff --git a/make_metadata.py b/make_metadata.py deleted file mode 100755 index 455de7de..00000000 --- a/make_metadata.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 -# MicroPython will pick up glob from the current dir otherwise. -import sys - -sys.path.pop(0) - -import glob - - -TEMPLATE = """\ -import sys -# Remove current dir from sys.path, otherwise setuptools will peek up our -# module instead of system's. -sys.path.pop(0) -from setuptools import setup -sys.path.append("..") -import sdist_upip - -setup(name='micropython-%(dist_name)s', - version='%(version)s', - description=%(desc)r, - long_description=%(long_desc)s, - url='https://github.com/micropython/micropython-lib', - author=%(author)r, - author_email=%(author_email)r, - maintainer=%(maintainer)r, - maintainer_email='micro-python@googlegroups.com', - license=%(license)r, - cmdclass={'sdist': sdist_upip.sdist}, - %(_what_)s=[%(modules)s]%(_inst_req_)s) -""" - -DUMMY_DESC = """\ -This is a dummy implementation of a module for MicroPython standard library. -It contains zero or very little functionality, and primarily intended to -avoid import errors (using idea that even if an application imports a -module, it may be not using it onevery code path, so may work at least -partially). It is expected that more complete implementation of the module -will be provided later. Please help with the development if you are -interested in this module.""" - -CPYTHON_DESC = """\ -This is a module ported from CPython standard library to be compatible with -MicroPython interpreter. Usually, this means applying small patches for -features not supported (yet, or at all) in MicroPython. Sometimes, heavier -changes are required. Note that CPython modules are written with availability -of vast resources in mind, and may not work for MicroPython ports with -limited heap. If you are affected by such a case, please help reimplement -the module from scratch.""" - -PYPY_DESC = """\ -This is a module ported from PyPy standard library to be compatible with -MicroPython interpreter. Usually, this means applying small patches for -features not supported (yet, or at all) in MicroPython. Sometimes, heavier -changes are required. Note that CPython modules are written with availability -of vast resources in mind, and may not work for MicroPython ports with -limited heap. If you are affected by such a case, please help reimplement -the module from scratch.""" - -MICROPYTHON_LIB_DESC = """\ -This is a module reimplemented specifically for MicroPython standard library, -with efficient and lean design in mind. Note that this module is likely work -in progress and likely supports just a subset of CPython's corresponding -module. Please help with the development if you are interested in this -module.""" - -BACKPORT_DESC = """\ -This is MicroPython compatibility module, allowing applications using -MicroPython-specific features to run on CPython. -""" - -MICROPYTHON_DEVELS = "micropython-lib Developers" -MICROPYTHON_DEVELS_EMAIL = "micro-python@googlegroups.com" -CPYTHON_DEVELS = "CPython Developers" -CPYTHON_DEVELS_EMAIL = "python-dev@python.org" -PYPY_DEVELS = "PyPy Developers" -PYPY_DEVELS_EMAIL = "pypy-dev@python.org" - - -def parse_metadata(f): - data = {} - for l in f: - l = l.strip() - if l[0] == "#": - continue - k, v = l.split("=", 1) - data[k.strip()] = v.strip() - return data - - -def write_setup(fname, substs): - with open(fname, "w") as f: - f.write(TEMPLATE % substs) - - -def main(): - for fname in glob.iglob("*/metadata.txt"): - print(fname) - with open(fname) as f: - data = parse_metadata(f) - - dirname = fname.split("/")[0] - module = dirname - if data["type"] == "module": - data["_what_"] = "py_modules" - elif data["type"] == "package": - data["_what_"] = "packages" - else: - raise ValueError - - if data["srctype"] == "dummy": - data["author"] = MICROPYTHON_DEVELS - data["author_email"] = MICROPYTHON_DEVELS_EMAIL - data["maintainer"] = MICROPYTHON_DEVELS - data["license"] = "MIT" - data["desc"] = "Dummy %s module for MicroPython" % module - data["long_desc"] = DUMMY_DESC - elif data["srctype"] == "cpython": - data["author"] = CPYTHON_DEVELS - data["author_email"] = CPYTHON_DEVELS_EMAIL - data["maintainer"] = MICROPYTHON_DEVELS - data["license"] = "Python" - data["desc"] = "CPython %s module ported to MicroPython" % module - data["long_desc"] = CPYTHON_DESC - elif data["srctype"] == "pypy": - data["author"] = PYPY_DEVELS - data["author_email"] = PYPY_DEVELS_EMAIL - data["maintainer"] = MICROPYTHON_DEVELS - data["license"] = "MIT" - data["desc"] = "PyPy %s module ported to MicroPython" % module - data["long_desc"] = PYPY_DESC - elif data["srctype"] == "micropython-lib": - if "author" not in data: - data["author"] = MICROPYTHON_DEVELS - if "author_email" not in data: - data["author_email"] = MICROPYTHON_DEVELS_EMAIL - if "maintainer" not in data: - data["maintainer"] = MICROPYTHON_DEVELS - if "desc" not in data: - data["desc"] = "%s module for MicroPython" % module - if "long_desc" not in data: - data["long_desc"] = MICROPYTHON_LIB_DESC - if "license" not in data: - data["license"] = "MIT" - elif data["srctype"] == "cpython-backport": - assert module.startswith("cpython-") - module = module[len("cpython-") :] - data["author"] = MICROPYTHON_DEVELS - data["author_email"] = MICROPYTHON_DEVELS_EMAIL - data["maintainer"] = MICROPYTHON_DEVELS - data["license"] = "Python" - data["desc"] = "MicroPython module %s ported to CPython" % module - data["long_desc"] = BACKPORT_DESC - else: - raise ValueError - - if "dist_name" not in data: - data["dist_name"] = dirname - if "name" not in data: - data["name"] = module - if data["long_desc"] in ("README", "README.rst"): - data["long_desc"] = "open(%r).read()" % data["long_desc"] - else: - data["long_desc"] = repr(data["long_desc"]) - - data["modules"] = "'" + data["name"].rsplit(".", 1)[0] + "'" - if "extra_modules" in data: - data["modules"] += ", " + ", ".join( - ["'" + x.strip() + "'" for x in data["extra_modules"].split(",")] - ) - - if "depends" in data: - deps = ["micropython-" + x.strip() for x in data["depends"].split(",")] - data["_inst_req_"] = ",\n install_requires=['" + "', '".join(deps) + "']" - else: - data["_inst_req_"] = "" - - write_setup(dirname + "/setup.py", data) - - -if __name__ == "__main__": - main() diff --git a/optimize_upip.py b/optimize_upip.py deleted file mode 100644 index 156b220f..00000000 --- a/optimize_upip.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# This script optimizes a Python source distribution tarball as produced by -# "python3 setup.py sdist" command for MicroPython's native package manager, -# upip. Optimization includes: -# * Removing metadata files not used by upip (this includes setup.py) -# * Recompressing gzip archive with 4K dictionary size so it can be -# installed even on low-heap targets. -# -import sys -import os -import zlib -from subprocess import Popen, PIPE -import glob -import tarfile -import re -import io - - -def gzip_4k(inf, fname): - comp = zlib.compressobj(level=9, wbits=16 + 12) - with open(fname + ".out", "wb") as outf: - while 1: - data = inf.read(1024) - if not data: - break - outf.write(comp.compress(data)) - outf.write(comp.flush()) - os.rename(fname, fname + ".orig") - os.rename(fname + ".out", fname) - - -def recompress(fname): - with Popen(["gzip", "-d", "-c", fname], stdout=PIPE).stdout as inf: - gzip_4k(inf, fname) - - -def find_latest(dir): - res = [] - for fname in glob.glob(dir + "/*.gz"): - st = os.stat(fname) - res.append((st.st_mtime, fname)) - res.sort() - latest = res[-1][1] - return latest - - -def recompress_latest(dir): - latest = find_latest(dir) - print(latest) - recompress(latest) - - -FILTERS = [ - # include, exclude, repeat - (r".+\.egg-info/(PKG-INFO|requires\.txt)", r"setup.py$"), - (r".+\.py$", r"[^/]+$"), - (None, r".+\.egg-info/.+"), -] - - -outbuf = io.BytesIO() - - -def filter_tar(name): - fin = tarfile.open(name, "r:gz") - fout = tarfile.open(fileobj=outbuf, mode="w") - for info in fin: - # print(info) - if not "/" in info.name: - continue - fname = info.name.split("/", 1)[1] - include = None - - for inc_re, exc_re in FILTERS: - if include is None and inc_re: - if re.match(inc_re, fname): - include = True - - if include is None and exc_re: - if re.match(exc_re, fname): - include = False - - if include is None: - include = True - - if include: - print("Including:", fname) - else: - print("Excluding:", fname) - continue - - farch = fin.extractfile(info) - fout.addfile(info, farch) - fout.close() - fin.close() - - -from setuptools import Command - - -class OptimizeUpip(Command): - - user_options = [] - - def run(self): - latest = find_latest("dist") - filter_tar(latest) - outbuf.seek(0) - gzip_4k(outbuf, latest) - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - -# For testing only -if __name__ == "__main__": - # recompress_latest(sys.argv[1]) - filter_tar(sys.argv[1]) - outbuf.seek(0) - gzip_4k(outbuf, sys.argv[1]) diff --git a/sdist_upip.py b/sdist_upip.py deleted file mode 100644 index 74e14e0a..00000000 --- a/sdist_upip.py +++ /dev/null @@ -1,141 +0,0 @@ -# -# This module overrides distutils (also compatible with setuptools) "sdist" -# command to perform pre- and post-processing as required for MicroPython's -# upip package manager. -# -# Preprocessing steps: -# * Creation of Python resource module (R.py) from each top-level package's -# resources. -# Postprocessing steps: -# * Removing metadata files not used by upip (this includes setup.py) -# * Recompressing gzip archive with 4K dictionary size so it can be -# installed even on low-heap targets. -# -import sys -import os -import zlib -from subprocess import Popen, PIPE -import glob -import tarfile -import re -import io - -from distutils.filelist import FileList -from setuptools.command.sdist import sdist as _sdist - - -def gzip_4k(inf, fname): - comp = zlib.compressobj(level=9, wbits=16 + 12) - with open(fname + ".out", "wb") as outf: - while 1: - data = inf.read(1024) - if not data: - break - outf.write(comp.compress(data)) - outf.write(comp.flush()) - os.rename(fname, fname + ".orig") - os.rename(fname + ".out", fname) - - -FILTERS = [ - # include, exclude, repeat - (r".+\.egg-info/(PKG-INFO|requires\.txt)", r"setup.py$"), - (r".+\.py$", r"[^/]+$"), - (None, r".+\.egg-info/.+"), -] - - -outbuf = io.BytesIO() - - -def filter_tar(name): - fin = tarfile.open(name, "r:gz") - fout = tarfile.open(fileobj=outbuf, mode="w") - for info in fin: - # print(info) - if not "/" in info.name: - continue - fname = info.name.split("/", 1)[1] - include = None - - for inc_re, exc_re in FILTERS: - if include is None and inc_re: - if re.match(inc_re, fname): - include = True - - if include is None and exc_re: - if re.match(exc_re, fname): - include = False - - if include is None: - include = True - - if include: - print("including:", fname) - else: - print("excluding:", fname) - continue - - farch = fin.extractfile(info) - fout.addfile(info, farch) - fout.close() - fin.close() - - -def make_resource_module(manifest_files): - resources = [] - # Any non-python file included in manifest is resource - for fname in manifest_files: - ext = fname.rsplit(".", 1)[1] - if ext != "py": - resources.append(fname) - - if resources: - print("creating resource module R.py") - resources.sort() - last_pkg = None - r_file = None - for fname in resources: - try: - pkg, res_name = fname.split("/", 1) - except ValueError: - print("not treating %s as a resource" % fname) - continue - if last_pkg != pkg: - last_pkg = pkg - if r_file: - r_file.write("}\n") - r_file.close() - r_file = open(pkg + "/R.py", "w") - r_file.write("R = {\n") - - with open(fname, "rb") as f: - r_file.write("%r: %r,\n" % (res_name, f.read())) - - if r_file: - r_file.write("}\n") - r_file.close() - - -class sdist(_sdist): - def run(self): - self.filelist = FileList() - self.get_file_list() - make_resource_module(self.filelist.files) - - r = super().run() - - assert len(self.archive_files) == 1 - print("filtering files and recompressing with 4K dictionary") - filter_tar(self.archive_files[0]) - outbuf.seek(0) - gzip_4k(outbuf, self.archive_files[0]) - - return r - - -# For testing only -if __name__ == "__main__": - filter_tar(sys.argv[1]) - outbuf.seek(0) - gzip_4k(outbuf, sys.argv[1])