mip: Add PyPI support

README.md

@@ -115,6 +115,30 @@ mip.install(PACKAGE_NAME, index="https://USERNAME.github.io/micropython-lib/mip/
 (Where `USERNAME`, `BRANCH_NAME` and `PACKAGE_NAME` are replaced with the owner
 of the fork, the branch the packages were built from, and the package name.)
 
+## Installing packages from Python Package Index
+
+It is possible to use the `mpremote mip install` or `mip.install()` methods to
+install packages built from the official
+[PyPI](https://pypi.org/), [Test PyPI](https://test.pypi.org/) or a selfhosted
+Python Package Index.
+
+To install a package and its dependencies from a Python Package Index, use
+commands such as:
+
+```bash
+$ mpremote connect /dev/ttyUSB0 mip install --index PACKAGE_INDEX --pypi PACKAGE_NAME
+```
+
+Or from a networked device:
+
+```py
+import mip
+mip.install(PACKAGE_NAME, index=PACKAGE_INDEX, pypi=True)
+```
+
+(Where `PACKAGE_NAME` and `PACKAGE_INDEX` are replaced with the package name
+and the package index URL, e.g. `https://test.pypi.org/pypi` for Test PyPI)
+
 ## Contributing
 
 We use [GitHub Discussions](https://github.com/micropython/micropython/discussions)

micropython/mip/mip/__init__.py

@@ -1,5 +1,7 @@
 # MicroPython package installer
-# MIT license; Copyright (c) 2022 Jim Mussared
+# MIT license
+# Copyright (c) 2022 Jim Mussared
+# Extended with PyPI support by brainelectronics 2023
 
 import urequests as requests
 import sys
@@ -42,8 +44,6 @@ def _chunk(src, dest):
 
 # Check if the specified path exists and matches the hash.
 def _check_exists(path, short_hash):
-    import os
-
     try:
         import binascii
         import hashlib
@@ -92,16 +92,24 @@ def _download_file(url, dest):
         response.close()
 
 
-def _install_json(package_json_url, index, target, version, mpy):
+def _get_package_json(package_json_url, version):
+    package_json = {}
     response = requests.get(_rewrite_url(package_json_url, version))
     try:
         if response.status_code != 200:
             print("Package not found:", package_json_url)
-            return False
+            return package_json
 
         package_json = response.json()
     finally:
         response.close()
+
+    return package_json
+
+
+def _install_json(package_json_url, index, target, version, mpy):
+    package_json = _get_package_json(package_json_url, version)
+
     for target_path, short_hash in package_json.get("hashes", ()):
         fs_target_path = target + "/" + target_path
         if _check_exists(fs_target_path, short_hash):
@@ -122,11 +130,124 @@ def _install_json(package_json_url, index, target, version, mpy):
     return True
 
 
-def _install_package(package, index, target, version, mpy):
+def _install_tar(package_json_url, index, target, version):
+    import gc
+
+    package_json = _get_package_json(package_json_url, version)
+    meta = {}
+
+    if not version:
+        version = package_json.get("info", {}).get("version", "")
+
+    if version not in package_json.get("releases", ()):
+        print("Version {} not found".format(version))
+        return False
+
+    package_url = package_json["releases"][version][0]["url"]
+    # save some memory, the large dict is no longer required
+    del package_json
+    gc.collect()
+
+    fs_target_path = target + "/" + package_url.rsplit("/", 1)[1]
+
+    if not _download_file(package_url, fs_target_path):
+        print("Failed to download {} to {}".format(package_url, fs_target_path))
+        return False
+
+    try:
+        from uzlib import DecompIO
+        from utarfile import TarFile
+
+        gzdict_sz = 16 + 15
+        sz = gc.mem_free() + gc.mem_alloc()
+        if sz <= 65536:
+            gzdict_sz = 16 + 12
+
+        zipped_file = open(fs_target_path, "rb")
+        decompressed_file = DecompIO(zipped_file, gzdict_sz)
+        tar_file = TarFile(fileobj=decompressed_file)
+
+        meta = _install_tar_file(tar_file, target)
+
+        zipped_file.close()
+        del zipped_file
+        del decompressed_file
+        del tar_file
+    except Exception as e:
+        print("Failed to decompress downloaded file due to {}".format(e))
+        return False
+
+    # cleanup downloaded file
+    try:
+        from os import unlink
+
+        unlink(fs_target_path)
+    except Exception as e:
+        print("Error during cleanup of {}".format(fs_target_path), e)
+
+    gc.collect()
+
+    deps = meta.get("deps", "").rstrip()
+    if deps:
+        deps = deps.decode("utf-8").split("\n")
+        print("Install additional deps: {}".format(deps))
+        results = []
+
+        for ele in deps:
+            res = _install_package(
+                package=ele, index=index, target=target, version=None, mpy=False, pypi=True
+            )
+            if not res:
+                print("Package may be partially installed")
+            results.append(res)
+
+        return all(results)
+
+    return True
+
+
+def _install_tar_file(f, target):
+    from utarfile import DIRTYPE
+    from shutil import copyfileobj
+
+    meta = {}
+
+    for info in f:
+        if "PaxHeader" in info.name:
+            continue
+
+        print("Processing: {}".format(info))
+        fname = info.name
+        try:
+            fname = fname[fname.index("/") + 1 :]
+        except ValueError:
+            fname = ""
+
+        save = True
+        for p in ("setup.", "PKG-INFO", "README"):
+            if fname.startswith(p) or ".egg-info" in fname:
+                if fname.endswith("/requires.txt"):
+                    meta["deps"] = f.extractfile(info).read()
+                save = False
+                break
+
+        if save:
+            outfname = target + "/" + fname
+            _ensure_path_exists(outfname)
+
+            if info.type != DIRTYPE:
+                this_file = f.extractfile(info)
+                copyfileobj(this_file, open(outfname, "wb"))
+
+    return meta
+
+
+def _install_package(package, index, target, version, mpy, pypi):
     if (
         package.startswith("http://")
         or package.startswith("https://")
         or package.startswith("github:")
+        or pypi
     ):
         if package.endswith(".py") or package.endswith(".mpy"):
             print("Downloading {} to {}".format(package, target))
@@ -134,11 +255,23 @@ def _install_package(package, index, target, version, mpy):
                 _rewrite_url(package, version), target + "/" + package.rsplit("/")[-1]
             )
         else:
-            if not package.endswith(".json"):
+            if pypi:
-                if not package.endswith("/"):
+                this_version = version
-                    package += "/"
+                if not version:
-                package += "package.json"
+                    this_version = "latest"
-            print("Installing {} to {}".format(package, target))
+                print(
+                    "Installing {} ({}) from {} to {}".format(package, this_version, index, target)
+                )
+                package = "{}/{}/json".format(index, package)
+                install("utarfile")
+                install("shutil")
+                return _install_tar(package, index, target, version)
+            else:
+                if not package.endswith(".json"):
+                    if not package.endswith("/"):
+                        package += "/"
+                    package += "package.json"
+                print("Installing {} to {}".format(package, target))
     else:
         if not version:
             version = "latest"
@@ -153,7 +286,7 @@ def _install_package(package, index, target, version, mpy):
     return _install_json(package, index, target, version, mpy)
 
 
-def install(package, index=None, target=None, version=None, mpy=True):
+def install(package, index=None, target=None, version=None, mpy=True, pypi=False):
     if not target:
         for p in sys.path:
             if p.endswith("/lib"):
@@ -166,7 +299,7 @@ def install(package, index=None, target=None, version=None, mpy=True):
     if not index:
         index = _PACKAGE_INDEX
 
-    if _install_package(package, index.rstrip("/"), target, version, mpy):
+    if _install_package(package, index.rstrip("/"), target, version, mpy, pypi):
         print("Done")
     else:
         print("Package may be partially installed")