tools/makepyproject.py: Add tool to generate PyPI package.

This tool makes a buildable package (including pyproject.toml) from
supported micropython-lib packages, suitable for publishing to PyPI and
using from CPython.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

tools/build.py

@@ -112,7 +112,6 @@
 
 # mip (or other tools) should request /package/{mpy_version}/{package_name}/{version}.json.
 
-import argparse
 import glob
 import hashlib
 import json
@@ -132,7 +131,7 @@ _COLOR_ERROR_OFF = "\033[0m"
 
 
 # Create all directories in the path (such that the file can be created).
-def _ensure_path_exists(file_path):
+def ensure_path_exists(file_path):
     path = os.path.dirname(file_path)
     if not os.path.isdir(path):
         os.makedirs(path)
@@ -155,7 +154,7 @@ def _identical_files(path_a, path_b):
 # Helper to write the object as json to the specified path, creating any
 # directories as required.
 def _write_json(obj, path, minify=False):
-    _ensure_path_exists(path)
+    ensure_path_exists(path)
     with open(path, "w") as f:
         json.dump(
             obj, f, indent=(None if minify else 2), separators=((",", ":") if minify else None)
@@ -173,7 +172,7 @@ def _write_package_json(
 
 
 # Format s with bold red.
-def _error_color(s):
+def error_color(s):
     return _COLOR_ERROR_ON + s + _COLOR_ERROR_OFF
 
 
@@ -191,7 +190,7 @@ def _write_hashed_file(package_name, src, target_path, out_file_dir, hash_prefix
         # that it's actually the same file.
         if not _identical_files(src.name, output_file_path):
             print(
-                _error_color("Hash collision processing:"),
+                error_color("Hash collision processing:"),
                 package_name,
                 file=sys.stderr,
             )
@@ -204,7 +203,7 @@ def _write_hashed_file(package_name, src, target_path, out_file_dir, hash_prefix
             sys.exit(1)
     else:
         # Create new file.
-        _ensure_path_exists(output_file_path)
+        ensure_path_exists(output_file_path)
         shutil.copyfile(src.name, output_file_path)
 
     return short_file_hash
@@ -235,7 +234,7 @@ def _compile_as_mpy(
             )
         except mpy_cross.CrossCompileError as e:
             print(
-                _error_color("Error:"),
+                error_color("Error:"),
                 "Unable to compile",
                 target_path,
                 "in package",
@@ -329,7 +328,7 @@ def build(output_path, hash_prefix_len, mpy_cross_path):
 
             # Append this package to the index.
             if not manifest.metadata().version:
-                print(_error_color("Warning:"), package_name, "doesn't have a version.")
+                print(error_color("Warning:"), package_name, "doesn't have a version.")
 
             # Try to find this package in the previous index.json.
             for p in index_json["packages"]:
@@ -360,11 +359,12 @@ def build(output_path, hash_prefix_len, mpy_cross_path):
             for result in manifest.files():
                 # This isn't allowed in micropython-lib anyway.
                 if result.file_type != manifestfile.FILE_TYPE_LOCAL:
-                    print("Non-local file not supported.", file=sys.stderr)
+                    print(error_color("Error:"), "Non-local file not supported.", file=sys.stderr)
                     sys.exit(1)
 
                 if not result.target_path.endswith(".py"):
                     print(
+                        error_color("Error:"),
                         "Target path isn't a .py file:",
                         result.target_path,
                         file=sys.stderr,

tools/makepyproject.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+#
+# This file is part of the MicroPython project, http://micropython.org/
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2023 Jim Mussared
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script makes a CPython-compatible package from a micropython-lib package
+# with a pyproject.toml that can be built (via hatch) and deployed to PyPI.
+# Requires that the project sets the pypi_publish= kwarg in its metadata().
+
+# Usage:
+# ./tools/makepyproject.py --output /tmp/foo micropython/foo
+# python -m build /tmp/foo
+# python -m twine upload /tmp/foo/dist/*.whl
+
+from email.utils import parseaddr
+import os
+import re
+import shutil
+import sys
+
+from build import error_color, ensure_path_exists
+
+
+DEFAULT_AUTHOR = "micropython-lib <contact@micropython.org>"
+DEFAULT_LICENSE = "MIT"
+
+
+def quoted_escape(s):
+    return s.replace('"', '\\"')
+
+
+def build(manifest_path, output_path):
+    import manifestfile
+
+    if not manifest_path.endswith(".py"):
+        # Allow specifying either the directory or the manifest file explicitly.
+        manifest_path = os.path.join(manifest_path, "manifest.py")
+
+    print("Generating pyproject for {} in {}...".format(manifest_path, output_path))
+
+    toml_path = os.path.join(output_path, "pyproject.toml")
+    ensure_path_exists(toml_path)
+
+    path_vars = {
+        "MPY_LIB_DIR": os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
+    }
+
+    # .../foo/manifest.py -> foo
+    package_name = os.path.basename(os.path.dirname(manifest_path))
+
+    # Compile the manifest.
+    manifest = manifestfile.ManifestFile(manifestfile.MODE_PYPROJECT, path_vars)
+    manifest.execute(manifest_path)
+
+    # If a package doesn't have a pypi name, then assume it isn't intended to
+    # be publishable.
+    if not manifest.metadata().pypi_publish:
+        print(error_color("Error:"), package_name, "doesn't have a pypi_publish name.")
+        sys.exit(1)
+
+    # These should be in all packages eventually.
+    if not manifest.metadata().version:
+        print(error_color("Error:"), package_name, "doesn't have a version.")
+        sys.exit(1)
+    if not manifest.metadata().description:
+        print(error_color("Error:"), package_name, "doesn't have a description.")
+        sys.exit(1)
+
+    # This is the root path of all .py files that are copied. We ensure that
+    # they all match.
+    top_level_package = None
+
+    for result in manifest.files():
+        # This isn't allowed in micropython-lib anyway.
+        if result.file_type != manifestfile.FILE_TYPE_LOCAL:
+            print(error_color("Error:"), "Non-local file not supported.", file=sys.stderr)
+            sys.exit(1)
+
+        # "foo/bar/baz.py" --> "foo"
+        # "baz.py" --> ""
+        result_package = os.path.split(result.target_path)[0]
+
+        if not result_package:
+            # This is a standalone .py file.
+            print(
+                error_color("Error:"),
+                "Unsupported single-file module: {}".format(result.target_path),
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        if top_level_package and result_package != top_level_package:
+            # This likely suggests that something needs to use require(..., pypi="...").
+            print(
+                error_color("Error:"),
+                "More than one top-level package: {}, {}.".format(
+                    result_package, top_level_package
+                ),
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        top_level_package = result_package
+
+        # Tag each file with the package metadata and copy the .py directly.
+        with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path:
+            dest_path = os.path.join(output_path, result.target_path)
+            ensure_path_exists(dest_path)
+            shutil.copyfile(tagged_path, dest_path)
+
+    # Copy README.md if it exists
+    readme_path = os.path.join(os.path.dirname(manifest_path), "README.md")
+    readme_toml = ""
+    if os.path.exists(readme_path):
+        shutil.copyfile(readme_path, os.path.join(output_path, "README.md"))
+        readme_toml = 'readme = "README.md"'
+
+    # Apply default author and license, otherwise use the package metadata.
+    license_toml = 'license = {{ text = "{}" }}'.format(
+        quoted_escape(manifest.metadata().license or DEFAULT_LICENSE)
+    )
+    author_name, author_email = parseaddr(manifest.metadata().author or DEFAULT_AUTHOR)
+    author_toml = 'authors = [ {{ name = "{}", email = "{}"}} ]'.format(
+        quoted_escape(author_name), quoted_escape(author_email)
+    )
+
+    # Write pyproject.toml.
+    with open(toml_path, "w") as toml_file:
+        print("# Generated by makepyproject.py", file=toml_file)
+
+        print(
+            """
+[build-system]
+requires = [
+    "hatchling"
+]
+build-backend = "hatchling.build"
+""",
+            file=toml_file,
+        )
+
+        print(
+            """
+[project]
+name = "{}"
+description = "{}"
+{}
+{}
+version = "{}"
+dependencies = [{}]
+urls = {{ Homepage = "https://github.com/micropython/micropython-lib" }}
+{}
+""".format(
+                quoted_escape(manifest.metadata().pypi_publish),
+                quoted_escape(manifest.metadata().description),
+                author_toml,
+                license_toml,
+                quoted_escape(manifest.metadata().version),
+                ", ".join('"{}"'.format(quoted_escape(r)) for r in manifest.pypi_dependencies()),
+                readme_toml,
+            ),
+            file=toml_file,
+        )
+
+        print(
+            """
+[tool.hatch.build]
+packages = ["{}"]
+""".format(
+                top_level_package
+            ),
+            file=toml_file,
+        )
+
+    print("Done.")
+
+
+def main():
+    import argparse
+
+    cmd_parser = argparse.ArgumentParser(
+        description="Generate a project that can be pushed to PyPI."
+    )
+    cmd_parser.add_argument("--output", required=True, help="output directory")
+    cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo")
+    cmd_parser.add_argument("manifest", help="input package path")
+    args = cmd_parser.parse_args()
+
+    if args.micropython:
+        sys.path.append(os.path.join(args.micropython, "tools"))  # for manifestfile
+
+    build(args.manifest, args.output)
+
+
+if __name__ == "__main__":
+    main()