upip: Capture dependency sub-modules.

These modules are just copies of corresponding standalone modules.

upip/upip__libc.py

@@ -0,0 +1,34 @@
+import ffi
+import sys
+
+
+_h = None
+
+names = ('libc.so', 'libc.so.0', 'libc.so.6', 'libc.dylib')
+
+def get():
+    global _h
+    if _h:
+        return _h
+    err = None
+    for n in names:
+        try:
+            _h = ffi.open(n)
+            return _h
+        except OSError as e:
+            err = e
+    raise err
+
+
+def set_names(n):
+    global names
+    names = n
+
+# Find out bitness of the platform, even if long ints are not supported
+# TODO: All bitness differences should be removed from micropython-lib, and
+# this snippet too.
+bitness = 1
+v = sys.maxsize
+while v:
+    bitness += 1
+    v >>= 1

upip/upip_errno.py

@@ -0,0 +1,38 @@
+EPERM   = 1   # Operation not permitted
+ENOENT  = 2   # No such file or directory
+ESRCH   = 3   # No such process
+EINTR   = 4   # Interrupted system call
+EIO     = 5   # I/O error
+ENXIO   = 6   # No such device or address
+E2BIG   = 7   # Argument list too long
+ENOEXEC = 8   # Exec format error
+EBADF   = 9   # Bad file number
+ECHILD  = 10  # No child processes
+EAGAIN  = 11  # Try again
+ENOMEM  = 12  # Out of memory
+EACCES  = 13  # Permission denied
+EFAULT  = 14  # Bad address
+ENOTBLK = 15  # Block device required
+EBUSY   = 16  # Device or resource busy
+EEXIST  = 17  # File exists
+EXDEV   = 18  # Cross-device link
+ENODEV  = 19  # No such device
+ENOTDIR = 20  # Not a directory
+EISDIR  = 21  # Is a directory
+EINVAL  = 22  # Invalid argument
+ENFILE  = 23  # File table overflow
+EMFILE  = 24  # Too many open files
+ENOTTY  = 25  # Not a typewriter
+ETXTBSY = 26  # Text file busy
+EFBIG   = 27  # File too large
+ENOSPC  = 28  # No space left on device
+ESPIPE  = 29  # Illegal seek
+EROFS   = 30  # Read-only file system
+EMLINK  = 31  # Too many links
+EPIPE   = 32  # Broken pipe
+EDOM    = 33  # Math argument out of domain of func
+ERANGE  = 34  # Math result not representable
+EAFNOSUPPORT = 97 # Address family not supported by protocol
+ECONNRESET = 104 # Connection timed out
+ETIMEDOUT = 110 # Connection timed out
+EINPROGRESS = 115 # Operation now in progress

upip/upip_gzip.py

@@ -0,0 +1,28 @@
+#import zlib
+import uzlib as zlib
+
+FTEXT    = 1
+FHCRC    = 2
+FEXTRA   = 4
+FNAME    = 8
+FCOMMENT = 16
+
+def decompress(data):
+    assert data[0] == 0x1f and data[1] == 0x8b
+    assert data[2] == 8
+    flg = data[3]
+    assert flg & 0xe0 == 0
+    i = 10
+    if flg & FEXTRA:
+        i += data[11] << 8 + data[10] + 2
+    if flg & FNAME:
+        while data[i]:
+            i += 1
+        i += 1
+    if flg & FCOMMENT:
+        while data[i]:
+            i += 1
+        i += 1
+    if flg & FHCRC:
+        i += 2
+    return zlib.decompress(memoryview(data)[i:], -15)

upip/upip_os.py

@@ -0,0 +1,227 @@
+import ffi
+import array
+import struct
+import upip_errno
+import upip_stat as stat_
+import upip__libc
+try:
+    from _os import *
+except:
+    pass
+
+
+libc = upip__libc.get()
+
+errno_ = libc.var("i", "errno")
+chdir_ = libc.func("i", "chdir", "s")
+mkdir_ = libc.func("i", "mkdir", "si")
+rename_ = libc.func("i", "rename", "ss")
+unlink_ = libc.func("i", "unlink", "s")
+rmdir_ = libc.func("i", "rmdir", "s")
+getcwd_ = libc.func("s", "getcwd", "si")
+opendir_ = libc.func("P", "opendir", "s")
+readdir_ = libc.func("P", "readdir", "P")
+open_ = libc.func("i", "open", "sii")
+read_ = libc.func("i", "read", "ipi")
+write_ = libc.func("i", "write", "iPi")
+close_ = libc.func("i", "close", "i")
+access_ = libc.func("i", "access", "si")
+fork_ = libc.func("i", "fork", "")
+pipe_ = libc.func("i", "pipe", "p")
+_exit_ = libc.func("v", "_exit", "i")
+getpid_ = libc.func("i", "getpid", "")
+waitpid_ = libc.func("i", "waitpid", "ipi")
+system_ = libc.func("i", "system", "s")
+getenv_ = libc.func("s", "getenv", "P")
+
+R_OK = const(4)
+W_OK = const(2)
+X_OK = const(1)
+F_OK = const(0)
+
+O_ACCMODE  = 0o0000003
+O_RDONLY   = 0o0000000
+O_WRONLY   = 0o0000001
+O_RDWR     = 0o0000002
+O_CREAT    = 0o0000100
+O_EXCL     = 0o0000200
+O_NOCTTY   = 0o0000400
+O_TRUNC    = 0o0001000
+O_APPEND   = 0o0002000
+O_NONBLOCK = 0o0004000
+
+error = OSError
+name = "posix"
+sep = "/"
+curdir = "."
+pardir = ".."
+environ = {"WARNING": "NOT_IMPLEMENTED"}
+
+
+def check_error(ret):
+    # Return True is error was EINTR (which usually means that OS call
+    # should be restarted).
+    if ret == -1:
+        e = errno_.get()
+        if e == upip_errno.EINTR:
+            return True
+        raise OSError(e)
+
+def raise_error():
+    raise OSError(errno_.get())
+
+
+def getcwd():
+    buf = bytearray(512)
+    return getcwd_(buf, 512)
+
+def mkdir(name, mode=0o777):
+    e = mkdir_(name, mode)
+    check_error(e)
+
+def rename(old, new):
+    e = rename_(old, new)
+    check_error(e)
+
+def unlink(name):
+    e = unlink_(name)
+    check_error(e)
+
+def rmdir(name):
+    e = rmdir_(name)
+    check_error(e)
+
+def makedirs(name, mode=0o777, exist_ok=False):
+    exists = access(name, F_OK)
+    if exists:
+        if exist_ok:
+            return
+        raise OSError(upip_errno.EEXIST)
+    s = ""
+    for c in name.split("/"):
+        s += c + "/"
+        try:
+            mkdir(s)
+        except OSError as e:
+            if e.args[0] != upip_errno.EEXIST:
+                raise
+
+def ilistdir_ex(path="."):
+    dir = opendir_(path)
+    if not dir:
+        raise_error()
+    res = []
+    dirent_fmt = "LLHB256s"
+    while True:
+        dirent = readdir_(dir)
+        if not dirent:
+            break
+        dirent = ffi.as_bytearray(dirent, struct.calcsize(dirent_fmt))
+        dirent = struct.unpack(dirent_fmt, dirent)
+        yield dirent
+
+def listdir(path="."):
+    is_str = type(path) is not bytes
+    res = []
+    for dirent in ilistdir_ex(path):
+        fname = dirent[4].split(b'\0', 1)[0]
+        if fname != b"." and fname != b"..":
+            if is_str:
+                fname = fsdecode(fname)
+            res.append(fname)
+    return res
+
+def walk(top, topdown=True):
+    files = []
+    dirs = []
+    for dirent in ilistdir_ex(top):
+        mode = dirent[3] << 12
+        fname = dirent[4].split(b'\0', 1)[0]
+        if stat_.S_ISDIR(mode):
+            if fname != b"." and fname != b"..":
+                dirs.append(fsdecode(fname))
+        else:
+            files.append(fsdecode(fname))
+    if topdown:
+        yield top, dirs, files
+    for d in dirs:
+        yield from walk(top + "/" + d, topdown)
+    if not topdown:
+        yield top, dirs, files
+
+def open(n, flags, mode=0o777):
+    r = open_(n, flags, mode)
+    check_error(r)
+    return r
+
+def read(fd, n):
+    buf = bytearray(n)
+    r = read_(fd, buf, n)
+    check_error(r)
+    return bytes(buf[:r])
+
+def write(fd, buf):
+    r = write_(fd, buf, len(buf))
+    check_error(r)
+    return r
+
+def close(fd):
+    r = close_(fd)
+    check_error(r)
+    return r
+
+def access(path, mode):
+    return access_(path, mode) == 0
+
+def chdir(dir):
+    r = chdir_(dir)
+    check_error(r)
+
+def fork():
+    r = fork_()
+    check_error(r)
+    return r
+
+def pipe():
+    a = array.array('i', [0, 0])
+    r = pipe_(a)
+    check_error(r)
+    return a[0], a[1]
+
+def _exit(n):
+    _exit_(n)
+
+def getpid():
+    return getpid_()
+
+def waitpid(pid, opts):
+    a = array.array('i', [0])
+    r = waitpid_(pid, a, opts)
+    check_error(r)
+    return (r, a[0])
+
+def system(command):
+    r = system_(command)
+    check_error(r)
+    return r
+
+def getenv(var, default=None):
+    var = getenv_(var)
+    if var is None:
+        return default
+    return var
+
+def fsencode(s):
+    if type(s) is bytes:
+        return s
+    return bytes(s, "utf-8")
+
+def fsdecode(s):
+    if type(s) is str:
+        return s
+    return str(s, "utf-8")
+
+
+def urandom(n):
+    with open("/dev/urandom", "rb") as f:
+        return f.read(n)

upip/upip_os_path.py

@@ -0,0 +1,49 @@
+import upip_os
+
+
+def normcase(s):
+    return s
+
+def normpath(s):
+    return s
+
+def abspath(s):
+    return upip_os.getcwd() + "/" + s
+
+def join(*args):
+    # TODO: this is non-compliant
+    if type(args[0]) is bytes:
+        return b"/".join(args)
+    else:
+        return "/".join(args)
+
+def split(path):
+    if path == "":
+        return ("", "")
+    r = path.rsplit("/", 1)
+    if len(r) == 1:
+        return ("", path)
+    head = r[0] #.rstrip("/")
+    if not head:
+        head = "/"
+    return (head, r[1])
+
+def dirname(path):
+    return split(path)[0]
+
+def basename(path):
+    return split(path)[1]
+
+def exists(path):
+    return upip_os.access(path, os.F_OK)
+
+# TODO
+lexists = exists
+
+def isdir(path):
+    import upip_stat
+    try:
+        mode = upip_os.stat(path)[0]
+        return upip_stat.S_ISDIR(mode)
+    except OSError:
+        return False

upip/upip_stat.py

@@ -0,0 +1,149 @@
+"""Constants/functions for interpreting results of os.stat() and os.lstat().
+
+Suggested usage: from stat import *
+"""
+
+# Indices for stat struct members in the tuple returned by os.stat()
+
+ST_MODE  = 0
+ST_INO   = 1
+ST_DEV   = 2
+ST_NLINK = 3
+ST_UID   = 4
+ST_GID   = 5
+ST_SIZE  = 6
+ST_ATIME = 7
+ST_MTIME = 8
+ST_CTIME = 9
+
+# Extract bits from the mode
+
+def S_IMODE(mode):
+    """Return the portion of the file's mode that can be set by
+    os.chmod().
+    """
+    return mode & 0o7777
+
+def S_IFMT(mode):
+    """Return the portion of the file's mode that describes the
+    file type.
+    """
+    return mode & 0o170000
+
+# Constants used as S_IFMT() for various file types
+# (not all are implemented on all systems)
+
+S_IFDIR  = 0o040000  # directory
+S_IFCHR  = 0o020000  # character device
+S_IFBLK  = 0o060000  # block device
+S_IFREG  = 0o100000  # regular file
+S_IFIFO  = 0o010000  # fifo (named pipe)
+S_IFLNK  = 0o120000  # symbolic link
+S_IFSOCK = 0o140000  # socket file
+
+# Functions to test for each file type
+
+def S_ISDIR(mode):
+    """Return True if mode is from a directory."""
+    return S_IFMT(mode) == S_IFDIR
+
+def S_ISCHR(mode):
+    """Return True if mode is from a character special device file."""
+    return S_IFMT(mode) == S_IFCHR
+
+def S_ISBLK(mode):
+    """Return True if mode is from a block special device file."""
+    return S_IFMT(mode) == S_IFBLK
+
+def S_ISREG(mode):
+    """Return True if mode is from a regular file."""
+    return S_IFMT(mode) == S_IFREG
+
+def S_ISFIFO(mode):
+    """Return True if mode is from a FIFO (named pipe)."""
+    return S_IFMT(mode) == S_IFIFO
+
+def S_ISLNK(mode):
+    """Return True if mode is from a symbolic link."""
+    return S_IFMT(mode) == S_IFLNK
+
+def S_ISSOCK(mode):
+    """Return True if mode is from a socket."""
+    return S_IFMT(mode) == S_IFSOCK
+
+# Names for permission bits
+
+S_ISUID = 0o4000  # set UID bit
+S_ISGID = 0o2000  # set GID bit
+S_ENFMT = S_ISGID # file locking enforcement
+S_ISVTX = 0o1000  # sticky bit
+S_IREAD = 0o0400  # Unix V7 synonym for S_IRUSR
+S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR
+S_IEXEC = 0o0100  # Unix V7 synonym for S_IXUSR
+S_IRWXU = 0o0700  # mask for owner permissions
+S_IRUSR = 0o0400  # read by owner
+S_IWUSR = 0o0200  # write by owner
+S_IXUSR = 0o0100  # execute by owner
+S_IRWXG = 0o0070  # mask for group permissions
+S_IRGRP = 0o0040  # read by group
+S_IWGRP = 0o0020  # write by group
+S_IXGRP = 0o0010  # execute by group
+S_IRWXO = 0o0007  # mask for others (not in group) permissions
+S_IROTH = 0o0004  # read by others
+S_IWOTH = 0o0002  # write by others
+S_IXOTH = 0o0001  # execute by others
+
+# Names for file flags
+
+UF_NODUMP    = 0x00000001  # do not dump file
+UF_IMMUTABLE = 0x00000002  # file may not be changed
+UF_APPEND    = 0x00000004  # file may only be appended to
+UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
+UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
+UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
+UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
+SF_ARCHIVED  = 0x00010000  # file may be archived
+SF_IMMUTABLE = 0x00020000  # file may not be changed
+SF_APPEND    = 0x00040000  # file may only be appended to
+SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
+SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
+
+
+_filemode_table = (
+    ((S_IFLNK,         "l"),
+     (S_IFREG,         "-"),
+     (S_IFBLK,         "b"),
+     (S_IFDIR,         "d"),
+     (S_IFCHR,         "c"),
+     (S_IFIFO,         "p")),
+
+    ((S_IRUSR,         "r"),),
+    ((S_IWUSR,         "w"),),
+    ((S_IXUSR|S_ISUID, "s"),
+     (S_ISUID,         "S"),
+     (S_IXUSR,         "x")),
+
+    ((S_IRGRP,         "r"),),
+    ((S_IWGRP,         "w"),),
+    ((S_IXGRP|S_ISGID, "s"),
+     (S_ISGID,         "S"),
+     (S_IXGRP,         "x")),
+
+    ((S_IROTH,         "r"),),
+    ((S_IWOTH,         "w"),),
+    ((S_IXOTH|S_ISVTX, "t"),
+     (S_ISVTX,         "T"),
+     (S_IXOTH,         "x"))
+)
+
+def filemode(mode):
+    """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
+    perm = []
+    for table in _filemode_table:
+        for bit, char in table:
+            if mode & bit == bit:
+                perm.append(char)
+                break
+        else:
+            perm.append("-")
+    return "".join(perm)

upip/upip_utarfile.py

@@ -0,0 +1,82 @@
+import uctypes
+
+# http://www.gnu.org/software/tar/manual/html_node/Standard.html
+TAR_HEADER = {
+    "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100),
+    "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12),
+}
+
+DIRTYPE = "dir"
+REGTYPE = "file"
+
+def roundup(val, align):
+    return (val + align - 1) & ~(align - 1)
+
+def skip(f, size):
+    assert size % 512 == 0
+    buf = bytearray(512)
+    while size:
+        size -= f.readinto(buf)
+
+class FileSection:
+
+    def __init__(self, f, content_len, aligned_len):
+        self.f = f
+        self.content_len = content_len
+        self.align = aligned_len - content_len
+
+    def read(self, sz=65536):
+        if self.content_len == 0:
+            return b""
+        if sz > self.content_len:
+            sz = self.content_len
+        data = self.f.read(sz)
+        sz = len(data)
+        self.content_len -= sz
+        return data
+
+    def skip(self):
+        self.f.read(self.content_len + self.align)
+
+class TarInfo:
+
+    def __str__(self):
+        return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size)
+
+class TarFile:
+
+    def __init__(self, name):
+        self.f = open(name, "rb")
+        self.subf = None
+
+    def next(self):
+            if self.subf:
+                self.subf.skip()
+            buf = self.f.read(512)
+            if not buf:
+                return None
+
+            h = uctypes.struct(TAR_HEADER, uctypes.addressof(buf), uctypes.LITTLE_ENDIAN)
+
+            # Empty block means end of archive
+            if h.name[0] == 0:
+                return None
+
+            d = TarInfo()
+            d.name = str(h.name, "utf-8").rstrip()
+            d.size = int(bytes(h.size).rstrip(), 8)
+            d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"]
+            self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512))
+            return d
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        v = self.next()
+        if v is None:
+            raise StopIteration
+        return v
+
+    def extractfile(self, tarinfo):
+        return tarinfo.subf