upip: Get rid of FFI dependency, use builtin _os module instead.

Also, delete as many as possible other dependencies (needed functions
are just copied into codebase). This will allow to work on static
MicroPython builds (including being a first step to support bare-metal
ports).

upip/Makefile

@@ -2,7 +2,7 @@ all:
 
 # This target prepares snapshot of all dependency modules, for
 # self-contained install
-deps: upip_ffilib.py upip_os.py upip_os_path.py upip_errno.py upip_gzip.py upip_stat.py upip_utarfile.py
+deps: upip_errno.py upip_gzip.py upip_utarfile.py
 
 upip_ffilib.py: ../ffilib/ffilib.py
 	cp $^ $@

upip/metadata.txt

@@ -1,7 +1,7 @@
 srctype = micropython-lib
 type = module
-version = 0.5.9
+version = 0.6
 author = Paul Sokolovsky
-extra_modules = upip_ffilib, upip_errno, upip_gzip, upip_os, upip_os_path, upip_stat, upip_utarfile
+extra_modules = upip_errno, upip_gzip, upip_utarfile
 desc = Simple package manager for MicroPython.
 long_desc = Simple package manager for MicroPython, targetting to be self-hosted (but not yet there). Compatible only with packages without custom setup.py code.

upip/setup.py

@@ -6,7 +6,7 @@ from setuptools import setup
 
 
 setup(name='micropython-upip',
-      version='0.5.9',
+      version='0.6',
       description='Simple package manager for MicroPython.',
       long_description='Simple package manager for MicroPython, targetting to be self-hosted (but not yet there). Compatible only with packages without custom setup.py code.',
       url='https://github.com/micropython/micropython/issues/405',
@@ -15,4 +15,4 @@ setup(name='micropython-upip',
       maintainer='MicroPython Developers',
       maintainer_email='micro-python@googlegroups.com',
       license='MIT',
-      py_modules=['upip', 'upip_ffilib', 'upip_errno', 'upip_gzip', 'upip_os', 'upip_os_path', 'upip_stat', 'upip_utarfile'])
+      py_modules=['upip', 'upip_errno', 'upip_gzip', 'upip_utarfile'])

upip/upip.py

@@ -11,9 +11,7 @@ def upip_import(mod, sub=None):
         return m
 
 sys = upip_import("sys")
-os = upip_import("os")
+import _os as os
-#os.path = upip_import("os.path").path
-ospath = upip_import("os", "path")
 
 errno = upip_import("errno")
 gzip = upip_import("gzip")
@@ -35,6 +33,35 @@ cleanup_files = [".pkg.tar"]
 class NotFoundError(Exception):
     pass
 
+def op_split(path):
+    if path == "":
+        return ("", "")
+    r = path.rsplit("/", 1)
+    if len(r) == 1:
+        return ("", path)
+    head = r[0]
+    if not head:
+        head = "/"
+    return (head, r[1])
+
+def op_basename(path):
+    return op_split(path)[1]
+
+def _makedirs(name, mode=0o777):
+    ret = False
+    s = ""
+    for c in name.rstrip("/").split("/"):
+        s += c + "/"
+        try:
+            os.mkdir(s)
+            ret = True
+        except OSError as e:
+            if e.args[0] != errno.EEXIST:
+                raise
+            ret = False
+    return ret
+
+
 def save_file(fname, subf):
     outf = open(fname, "wb")
     while True:
@@ -68,12 +95,8 @@ def install_tar(f, prefix):
         if save:
             outfname = prefix + fname
             if info.type == tarfile.DIRTYPE:
-                try:
+                if _makedirs(outfname):
-                    os.makedirs(outfname)
                     print("Created " + outfname)
-                except OSError as e:
-                    if e.args[0] != errno.EEXIST:
-                        raise
             else:
                 if debug:
                     print("Extracting " + outfname)
@@ -163,7 +186,7 @@ def install_pkg(pkg_spec, install_path):
     assert len(packages) == 1
     package_url = packages[0]["url"]
     print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
-    package_fname = ospath.basename(package_url)
+    package_fname = op_basename(package_url)
     download(package_url, package_fname)
 
     data = gzdecompress(package_fname)

upip/upip_ffilib.py

@@ -1,41 +0,0 @@
-import sys
-import ffi
-
-_cache = {}
-
-def open(name, maxver=10, extra=()):
-    try:
-        return _cache[name]
-    except KeyError:
-        pass
-    def libs():
-        if sys.platform == "linux":
-            yield '%s.so' % name
-            for i in range(maxver, -1, -1):
-                yield '%s.so.%u' % (name, i)
-        else:
-            for ext in ('dylib', 'dll'):
-                yield '%s.%s' % (name, ext)
-        for n in extra:
-            yield n
-    err = None
-    for n in libs():
-        try:
-            l = ffi.open(n)
-            _cache[name] = l
-            return l
-        except OSError as e:
-            err = e
-    raise err
-
-def libc():
-    return open("libc", 6)
-
-# Find out bitness of the platform, even if long ints are not supported
-# TODO: All bitness differences should be removed from micropython-lib, and
-# this snippet too.
-bitness = 1
-v = sys.maxsize
-while v:
-    bitness += 1
-    v >>= 1

upip/upip_os.py

@@ -1,242 +0,0 @@
-import ffi
-import array
-import ustruct as struct
-import upip_errno
-import upip_stat as stat_
-import upip_ffilib
-try:
-    from _os import *
-except:
-    pass
-
-
-libc = upip_ffilib.libc()
-
-try:
-    errno__ = libc.var("i", "errno")
-    def errno_(val=None):
-        if val is None:
-            return errno__.get()
-        errno__.set(val)
-except OSError:
-    __upip_errno = libc.func("p", "__errno", "")
-    def errno_(val=None):
-        if val is None:
-            p = __upip_errno()
-            buf = ffi.as_bytearray(p, 4)
-            return int.from_bytes(buf)
-        raise NotImplementedError
-
-chdir_ = libc.func("i", "chdir", "s")
-mkdir_ = libc.func("i", "mkdir", "si")
-rename_ = libc.func("i", "rename", "ss")
-unlink_ = libc.func("i", "unlink", "s")
-rmdir_ = libc.func("i", "rmdir", "s")
-getcwd_ = libc.func("s", "getcwd", "si")
-opendir_ = libc.func("P", "opendir", "s")
-readdir_ = libc.func("P", "readdir", "P")
-open_ = libc.func("i", "open", "sii")
-read_ = libc.func("i", "read", "ipi")
-write_ = libc.func("i", "write", "iPi")
-close_ = libc.func("i", "close", "i")
-access_ = libc.func("i", "access", "si")
-fork_ = libc.func("i", "fork", "")
-pipe_ = libc.func("i", "pipe", "p")
-_exit_ = libc.func("v", "_exit", "i")
-getpid_ = libc.func("i", "getpid", "")
-waitpid_ = libc.func("i", "waitpid", "ipi")
-system_ = libc.func("i", "system", "s")
-getenv_ = libc.func("s", "getenv", "P")
-
-R_OK = const(4)
-W_OK = const(2)
-X_OK = const(1)
-F_OK = const(0)
-
-O_ACCMODE  = 0o0000003
-O_RDONLY   = 0o0000000
-O_WRONLY   = 0o0000001
-O_RDWR     = 0o0000002
-O_CREAT    = 0o0000100
-O_EXCL     = 0o0000200
-O_NOCTTY   = 0o0000400
-O_TRUNC    = 0o0001000
-O_APPEND   = 0o0002000
-O_NONBLOCK = 0o0004000
-
-error = OSError
-name = "posix"
-sep = "/"
-curdir = "."
-pardir = ".."
-environ = {"WARNING": "NOT_IMPLEMENTED"}
-
-
-def check_error(ret):
-    # Return True is error was EINTR (which usually means that OS call
-    # should be restarted).
-    if ret == -1:
-        e = errno_()
-        if e == upip_errno.EINTR:
-            return True
-        raise OSError(e)
-
-def raise_error():
-    raise OSError(errno_.get())
-
-
-def getcwd():
-    buf = bytearray(512)
-    return getcwd_(buf, 512)
-
-def mkdir(name, mode=0o777):
-    e = mkdir_(name, mode)
-    check_error(e)
-
-def rename(old, new):
-    e = rename_(old, new)
-    check_error(e)
-
-def unlink(name):
-    e = unlink_(name)
-    check_error(e)
-
-def rmdir(name):
-    e = rmdir_(name)
-    check_error(e)
-
-def makedirs(name, mode=0o777, exist_ok=False):
-    exists = access(name, F_OK)
-    if exists:
-        if exist_ok:
-            return
-        raise OSError(upip_errno.EEXIST)
-    s = ""
-    for c in name.split("/"):
-        s += c + "/"
-        try:
-            mkdir(s)
-        except OSError as e:
-            if e.args[0] != upip_errno.EEXIST:
-                raise
-
-def ilistdir_ex(path="."):
-    dir = opendir_(path)
-    if not dir:
-        raise_error()
-    res = []
-    dirent_fmt = "LLHB256s"
-    while True:
-        dirent = readdir_(dir)
-        if not dirent:
-            break
-        dirent = ffi.as_bytearray(dirent, struct.calcsize(dirent_fmt))
-        dirent = struct.unpack(dirent_fmt, dirent)
-        yield dirent
-
-def listdir(path="."):
-    is_str = type(path) is not bytes
-    res = []
-    for dirent in ilistdir_ex(path):
-        fname = dirent[4].split(b'\0', 1)[0]
-        if fname != b"." and fname != b"..":
-            if is_str:
-                fname = fsdecode(fname)
-            res.append(fname)
-    return res
-
-def walk(top, topdown=True):
-    files = []
-    dirs = []
-    for dirent in ilistdir_ex(top):
-        mode = dirent[3] << 12
-        fname = dirent[4].split(b'\0', 1)[0]
-        if stat_.S_ISDIR(mode):
-            if fname != b"." and fname != b"..":
-                dirs.append(fsdecode(fname))
-        else:
-            files.append(fsdecode(fname))
-    if topdown:
-        yield top, dirs, files
-    for d in dirs:
-        yield from walk(top + "/" + d, topdown)
-    if not topdown:
-        yield top, dirs, files
-
-def open(n, flags, mode=0o777):
-    r = open_(n, flags, mode)
-    check_error(r)
-    return r
-
-def read(fd, n):
-    buf = bytearray(n)
-    r = read_(fd, buf, n)
-    check_error(r)
-    return bytes(buf[:r])
-
-def write(fd, buf):
-    r = write_(fd, buf, len(buf))
-    check_error(r)
-    return r
-
-def close(fd):
-    r = close_(fd)
-    check_error(r)
-    return r
-
-def access(path, mode):
-    return access_(path, mode) == 0
-
-def chdir(dir):
-    r = chdir_(dir)
-    check_error(r)
-
-def fork():
-    r = fork_()
-    check_error(r)
-    return r
-
-def pipe():
-    a = array.array('i', [0, 0])
-    r = pipe_(a)
-    check_error(r)
-    return a[0], a[1]
-
-def _exit(n):
-    _exit_(n)
-
-def getpid():
-    return getpid_()
-
-def waitpid(pid, opts):
-    a = array.array('i', [0])
-    r = waitpid_(pid, a, opts)
-    check_error(r)
-    return (r, a[0])
-
-def system(command):
-    r = system_(command)
-    check_error(r)
-    return r
-
-def getenv(var, default=None):
-    var = getenv_(var)
-    if var is None:
-        return default
-    return var
-
-def fsencode(s):
-    if type(s) is bytes:
-        return s
-    return bytes(s, "utf-8")
-
-def fsdecode(s):
-    if type(s) is str:
-        return s
-    return str(s, "utf-8")
-
-
-def urandom(n):
-    import builtins
-    with builtins.open("/dev/urandom", "rb") as f:
-        return f.read(n)

upip/upip_os_path.py

@@ -1,61 +0,0 @@
-import upip_os
-
-
-def normcase(s):
-    return s
-
-def normpath(s):
-    return s
-
-def abspath(s):
-    if s[0] != "/":
-        return upip_os.getcwd() + "/" + s
-    return s
-
-def join(*args):
-    # TODO: this is non-compliant
-    if type(args[0]) is bytes:
-        return b"/".join(args)
-    else:
-        return "/".join(args)
-
-def split(path):
-    if path == "":
-        return ("", "")
-    r = path.rsplit("/", 1)
-    if len(r) == 1:
-        return ("", path)
-    head = r[0] #.rstrip("/")
-    if not head:
-        head = "/"
-    return (head, r[1])
-
-def dirname(path):
-    return split(path)[0]
-
-def basename(path):
-    return split(path)[1]
-
-def exists(path):
-    return upip_os.access(path, os.F_OK)
-
-# TODO
-lexists = exists
-
-def isdir(path):
-    import upip_stat
-    try:
-        mode = upip_os.stat(path)[0]
-        return upip_stat.S_ISDIR(mode)
-    except OSError:
-        return False
-
-
-def expanduser(s):
-    if s == "~" or s.startswith("~/"):
-        h = upip_os.getenv("HOME")
-        return h + s[1:]
-    if s[0] == "~":
-        # Sorry folks, follow conventions
-        return "/home/" + s[1:]
-    return s

upip/upip_stat.py

@@ -1,149 +0,0 @@
-"""Constants/functions for interpreting results of os.stat() and os.lstat().
-
-Suggested usage: from stat import *
-"""
-
-# Indices for stat struct members in the tuple returned by os.stat()
-
-ST_MODE  = 0
-ST_INO   = 1
-ST_DEV   = 2
-ST_NLINK = 3
-ST_UID   = 4
-ST_GID   = 5
-ST_SIZE  = 6
-ST_ATIME = 7
-ST_MTIME = 8
-ST_CTIME = 9
-
-# Extract bits from the mode
-
-def S_IMODE(mode):
-    """Return the portion of the file's mode that can be set by
-    os.chmod().
-    """
-    return mode & 0o7777
-
-def S_IFMT(mode):
-    """Return the portion of the file's mode that describes the
-    file type.
-    """
-    return mode & 0o170000
-
-# Constants used as S_IFMT() for various file types
-# (not all are implemented on all systems)
-
-S_IFDIR  = 0o040000  # directory
-S_IFCHR  = 0o020000  # character device
-S_IFBLK  = 0o060000  # block device
-S_IFREG  = 0o100000  # regular file
-S_IFIFO  = 0o010000  # fifo (named pipe)
-S_IFLNK  = 0o120000  # symbolic link
-S_IFSOCK = 0o140000  # socket file
-
-# Functions to test for each file type
-
-def S_ISDIR(mode):
-    """Return True if mode is from a directory."""
-    return S_IFMT(mode) == S_IFDIR
-
-def S_ISCHR(mode):
-    """Return True if mode is from a character special device file."""
-    return S_IFMT(mode) == S_IFCHR
-
-def S_ISBLK(mode):
-    """Return True if mode is from a block special device file."""
-    return S_IFMT(mode) == S_IFBLK
-
-def S_ISREG(mode):
-    """Return True if mode is from a regular file."""
-    return S_IFMT(mode) == S_IFREG
-
-def S_ISFIFO(mode):
-    """Return True if mode is from a FIFO (named pipe)."""
-    return S_IFMT(mode) == S_IFIFO
-
-def S_ISLNK(mode):
-    """Return True if mode is from a symbolic link."""
-    return S_IFMT(mode) == S_IFLNK
-
-def S_ISSOCK(mode):
-    """Return True if mode is from a socket."""
-    return S_IFMT(mode) == S_IFSOCK
-
-# Names for permission bits
-
-S_ISUID = 0o4000  # set UID bit
-S_ISGID = 0o2000  # set GID bit
-S_ENFMT = S_ISGID # file locking enforcement
-S_ISVTX = 0o1000  # sticky bit
-S_IREAD = 0o0400  # Unix V7 synonym for S_IRUSR
-S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR
-S_IEXEC = 0o0100  # Unix V7 synonym for S_IXUSR
-S_IRWXU = 0o0700  # mask for owner permissions
-S_IRUSR = 0o0400  # read by owner
-S_IWUSR = 0o0200  # write by owner
-S_IXUSR = 0o0100  # execute by owner
-S_IRWXG = 0o0070  # mask for group permissions
-S_IRGRP = 0o0040  # read by group
-S_IWGRP = 0o0020  # write by group
-S_IXGRP = 0o0010  # execute by group
-S_IRWXO = 0o0007  # mask for others (not in group) permissions
-S_IROTH = 0o0004  # read by others
-S_IWOTH = 0o0002  # write by others
-S_IXOTH = 0o0001  # execute by others
-
-# Names for file flags
-
-UF_NODUMP    = 0x00000001  # do not dump file
-UF_IMMUTABLE = 0x00000002  # file may not be changed
-UF_APPEND    = 0x00000004  # file may only be appended to
-UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
-UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
-UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
-UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
-SF_ARCHIVED  = 0x00010000  # file may be archived
-SF_IMMUTABLE = 0x00020000  # file may not be changed
-SF_APPEND    = 0x00040000  # file may only be appended to
-SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
-SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
-
-
-_filemode_table = (
-    ((S_IFLNK,         "l"),
-     (S_IFREG,         "-"),
-     (S_IFBLK,         "b"),
-     (S_IFDIR,         "d"),
-     (S_IFCHR,         "c"),
-     (S_IFIFO,         "p")),
-
-    ((S_IRUSR,         "r"),),
-    ((S_IWUSR,         "w"),),
-    ((S_IXUSR|S_ISUID, "s"),
-     (S_ISUID,         "S"),
-     (S_IXUSR,         "x")),
-
-    ((S_IRGRP,         "r"),),
-    ((S_IWGRP,         "w"),),
-    ((S_IXGRP|S_ISGID, "s"),
-     (S_ISGID,         "S"),
-     (S_IXGRP,         "x")),
-
-    ((S_IROTH,         "r"),),
-    ((S_IWOTH,         "w"),),
-    ((S_IXOTH|S_ISVTX, "t"),
-     (S_ISVTX,         "T"),
-     (S_IXOTH,         "x"))
-)
-
-def filemode(mode):
-    """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
-    perm = []
-    for table in _filemode_table:
-        for bit, char in table:
-            if mode & bit == bit:
-                perm.append(char)
-                break
-        else:
-            perm.append("-")
-    return "".join(perm)