diff --git a/upip/upip__libc.py b/upip/upip__libc.py new file mode 100644 index 00000000..a930cbf7 --- /dev/null +++ b/upip/upip__libc.py @@ -0,0 +1,34 @@ +import ffi +import sys + + +_h = None + +names = ('libc.so', 'libc.so.0', 'libc.so.6', 'libc.dylib') + +def get(): + global _h + if _h: + return _h + err = None + for n in names: + try: + _h = ffi.open(n) + return _h + except OSError as e: + err = e + raise err + + +def set_names(n): + global names + names = n + +# Find out bitness of the platform, even if long ints are not supported +# TODO: All bitness differences should be removed from micropython-lib, and +# this snippet too. +bitness = 1 +v = sys.maxsize +while v: + bitness += 1 + v >>= 1 diff --git a/upip/upip_errno.py b/upip/upip_errno.py new file mode 100644 index 00000000..7b7935ef --- /dev/null +++ b/upip/upip_errno.py @@ -0,0 +1,38 @@ +EPERM = 1 # Operation not permitted +ENOENT = 2 # No such file or directory +ESRCH = 3 # No such process +EINTR = 4 # Interrupted system call +EIO = 5 # I/O error +ENXIO = 6 # No such device or address +E2BIG = 7 # Argument list too long +ENOEXEC = 8 # Exec format error +EBADF = 9 # Bad file number +ECHILD = 10 # No child processes +EAGAIN = 11 # Try again +ENOMEM = 12 # Out of memory +EACCES = 13 # Permission denied +EFAULT = 14 # Bad address +ENOTBLK = 15 # Block device required +EBUSY = 16 # Device or resource busy +EEXIST = 17 # File exists +EXDEV = 18 # Cross-device link +ENODEV = 19 # No such device +ENOTDIR = 20 # Not a directory +EISDIR = 21 # Is a directory +EINVAL = 22 # Invalid argument +ENFILE = 23 # File table overflow +EMFILE = 24 # Too many open files +ENOTTY = 25 # Not a typewriter +ETXTBSY = 26 # Text file busy +EFBIG = 27 # File too large +ENOSPC = 28 # No space left on device +ESPIPE = 29 # Illegal seek +EROFS = 30 # Read-only file system +EMLINK = 31 # Too many links +EPIPE = 32 # Broken pipe +EDOM = 33 # Math argument out of domain of func +ERANGE = 34 # Math result not representable +EAFNOSUPPORT = 97 # Address family not supported by protocol +ECONNRESET = 104 # Connection timed out +ETIMEDOUT = 110 # Connection timed out +EINPROGRESS = 115 # Operation now in progress diff --git a/upip/upip_gzip.py b/upip/upip_gzip.py new file mode 100644 index 00000000..be4e8f4a --- /dev/null +++ b/upip/upip_gzip.py @@ -0,0 +1,28 @@ +#import zlib +import uzlib as zlib + +FTEXT = 1 +FHCRC = 2 +FEXTRA = 4 +FNAME = 8 +FCOMMENT = 16 + +def decompress(data): + assert data[0] == 0x1f and data[1] == 0x8b + assert data[2] == 8 + flg = data[3] + assert flg & 0xe0 == 0 + i = 10 + if flg & FEXTRA: + i += data[11] << 8 + data[10] + 2 + if flg & FNAME: + while data[i]: + i += 1 + i += 1 + if flg & FCOMMENT: + while data[i]: + i += 1 + i += 1 + if flg & FHCRC: + i += 2 + return zlib.decompress(memoryview(data)[i:], -15) diff --git a/upip/upip_os.py b/upip/upip_os.py new file mode 100644 index 00000000..3d1239dd --- /dev/null +++ b/upip/upip_os.py @@ -0,0 +1,227 @@ +import ffi +import array +import struct +import upip_errno +import upip_stat as stat_ +import upip__libc +try: + from _os import * +except: + pass + + +libc = upip__libc.get() + +errno_ = libc.var("i", "errno") +chdir_ = libc.func("i", "chdir", "s") +mkdir_ = libc.func("i", "mkdir", "si") +rename_ = libc.func("i", "rename", "ss") +unlink_ = libc.func("i", "unlink", "s") +rmdir_ = libc.func("i", "rmdir", "s") +getcwd_ = libc.func("s", "getcwd", "si") +opendir_ = libc.func("P", "opendir", "s") +readdir_ = libc.func("P", "readdir", "P") +open_ = libc.func("i", "open", "sii") +read_ = libc.func("i", "read", "ipi") +write_ = libc.func("i", "write", "iPi") +close_ = libc.func("i", "close", "i") +access_ = libc.func("i", "access", "si") +fork_ = libc.func("i", "fork", "") +pipe_ = libc.func("i", "pipe", "p") +_exit_ = libc.func("v", "_exit", "i") +getpid_ = libc.func("i", "getpid", "") +waitpid_ = libc.func("i", "waitpid", "ipi") +system_ = libc.func("i", "system", "s") +getenv_ = libc.func("s", "getenv", "P") + +R_OK = const(4) +W_OK = const(2) +X_OK = const(1) +F_OK = const(0) + +O_ACCMODE = 0o0000003 +O_RDONLY = 0o0000000 +O_WRONLY = 0o0000001 +O_RDWR = 0o0000002 +O_CREAT = 0o0000100 +O_EXCL = 0o0000200 +O_NOCTTY = 0o0000400 +O_TRUNC = 0o0001000 +O_APPEND = 0o0002000 +O_NONBLOCK = 0o0004000 + +error = OSError +name = "posix" +sep = "/" +curdir = "." +pardir = ".." +environ = {"WARNING": "NOT_IMPLEMENTED"} + + +def check_error(ret): + # Return True is error was EINTR (which usually means that OS call + # should be restarted). + if ret == -1: + e = errno_.get() + if e == upip_errno.EINTR: + return True + raise OSError(e) + +def raise_error(): + raise OSError(errno_.get()) + + +def getcwd(): + buf = bytearray(512) + return getcwd_(buf, 512) + +def mkdir(name, mode=0o777): + e = mkdir_(name, mode) + check_error(e) + +def rename(old, new): + e = rename_(old, new) + check_error(e) + +def unlink(name): + e = unlink_(name) + check_error(e) + +def rmdir(name): + e = rmdir_(name) + check_error(e) + +def makedirs(name, mode=0o777, exist_ok=False): + exists = access(name, F_OK) + if exists: + if exist_ok: + return + raise OSError(upip_errno.EEXIST) + s = "" + for c in name.split("/"): + s += c + "/" + try: + mkdir(s) + except OSError as e: + if e.args[0] != upip_errno.EEXIST: + raise + +def ilistdir_ex(path="."): + dir = opendir_(path) + if not dir: + raise_error() + res = [] + dirent_fmt = "LLHB256s" + while True: + dirent = readdir_(dir) + if not dirent: + break + dirent = ffi.as_bytearray(dirent, struct.calcsize(dirent_fmt)) + dirent = struct.unpack(dirent_fmt, dirent) + yield dirent + +def listdir(path="."): + is_str = type(path) is not bytes + res = [] + for dirent in ilistdir_ex(path): + fname = dirent[4].split(b'\0', 1)[0] + if fname != b"." and fname != b"..": + if is_str: + fname = fsdecode(fname) + res.append(fname) + return res + +def walk(top, topdown=True): + files = [] + dirs = [] + for dirent in ilistdir_ex(top): + mode = dirent[3] << 12 + fname = dirent[4].split(b'\0', 1)[0] + if stat_.S_ISDIR(mode): + if fname != b"." and fname != b"..": + dirs.append(fsdecode(fname)) + else: + files.append(fsdecode(fname)) + if topdown: + yield top, dirs, files + for d in dirs: + yield from walk(top + "/" + d, topdown) + if not topdown: + yield top, dirs, files + +def open(n, flags, mode=0o777): + r = open_(n, flags, mode) + check_error(r) + return r + +def read(fd, n): + buf = bytearray(n) + r = read_(fd, buf, n) + check_error(r) + return bytes(buf[:r]) + +def write(fd, buf): + r = write_(fd, buf, len(buf)) + check_error(r) + return r + +def close(fd): + r = close_(fd) + check_error(r) + return r + +def access(path, mode): + return access_(path, mode) == 0 + +def chdir(dir): + r = chdir_(dir) + check_error(r) + +def fork(): + r = fork_() + check_error(r) + return r + +def pipe(): + a = array.array('i', [0, 0]) + r = pipe_(a) + check_error(r) + return a[0], a[1] + +def _exit(n): + _exit_(n) + +def getpid(): + return getpid_() + +def waitpid(pid, opts): + a = array.array('i', [0]) + r = waitpid_(pid, a, opts) + check_error(r) + return (r, a[0]) + +def system(command): + r = system_(command) + check_error(r) + return r + +def getenv(var, default=None): + var = getenv_(var) + if var is None: + return default + return var + +def fsencode(s): + if type(s) is bytes: + return s + return bytes(s, "utf-8") + +def fsdecode(s): + if type(s) is str: + return s + return str(s, "utf-8") + + +def urandom(n): + with open("/dev/urandom", "rb") as f: + return f.read(n) diff --git a/upip/upip_os_path.py b/upip/upip_os_path.py new file mode 100644 index 00000000..28fb97d0 --- /dev/null +++ b/upip/upip_os_path.py @@ -0,0 +1,49 @@ +import upip_os + + +def normcase(s): + return s + +def normpath(s): + return s + +def abspath(s): + return upip_os.getcwd() + "/" + s + +def join(*args): + # TODO: this is non-compliant + if type(args[0]) is bytes: + return b"/".join(args) + else: + return "/".join(args) + +def split(path): + if path == "": + return ("", "") + r = path.rsplit("/", 1) + if len(r) == 1: + return ("", path) + head = r[0] #.rstrip("/") + if not head: + head = "/" + return (head, r[1]) + +def dirname(path): + return split(path)[0] + +def basename(path): + return split(path)[1] + +def exists(path): + return upip_os.access(path, os.F_OK) + +# TODO +lexists = exists + +def isdir(path): + import upip_stat + try: + mode = upip_os.stat(path)[0] + return upip_stat.S_ISDIR(mode) + except OSError: + return False diff --git a/upip/upip_stat.py b/upip/upip_stat.py new file mode 100644 index 00000000..704adfe2 --- /dev/null +++ b/upip/upip_stat.py @@ -0,0 +1,149 @@ +"""Constants/functions for interpreting results of os.stat() and os.lstat(). + +Suggested usage: from stat import * +""" + +# Indices for stat struct members in the tuple returned by os.stat() + +ST_MODE = 0 +ST_INO = 1 +ST_DEV = 2 +ST_NLINK = 3 +ST_UID = 4 +ST_GID = 5 +ST_SIZE = 6 +ST_ATIME = 7 +ST_MTIME = 8 +ST_CTIME = 9 + +# Extract bits from the mode + +def S_IMODE(mode): + """Return the portion of the file's mode that can be set by + os.chmod(). + """ + return mode & 0o7777 + +def S_IFMT(mode): + """Return the portion of the file's mode that describes the + file type. + """ + return mode & 0o170000 + +# Constants used as S_IFMT() for various file types +# (not all are implemented on all systems) + +S_IFDIR = 0o040000 # directory +S_IFCHR = 0o020000 # character device +S_IFBLK = 0o060000 # block device +S_IFREG = 0o100000 # regular file +S_IFIFO = 0o010000 # fifo (named pipe) +S_IFLNK = 0o120000 # symbolic link +S_IFSOCK = 0o140000 # socket file + +# Functions to test for each file type + +def S_ISDIR(mode): + """Return True if mode is from a directory.""" + return S_IFMT(mode) == S_IFDIR + +def S_ISCHR(mode): + """Return True if mode is from a character special device file.""" + return S_IFMT(mode) == S_IFCHR + +def S_ISBLK(mode): + """Return True if mode is from a block special device file.""" + return S_IFMT(mode) == S_IFBLK + +def S_ISREG(mode): + """Return True if mode is from a regular file.""" + return S_IFMT(mode) == S_IFREG + +def S_ISFIFO(mode): + """Return True if mode is from a FIFO (named pipe).""" + return S_IFMT(mode) == S_IFIFO + +def S_ISLNK(mode): + """Return True if mode is from a symbolic link.""" + return S_IFMT(mode) == S_IFLNK + +def S_ISSOCK(mode): + """Return True if mode is from a socket.""" + return S_IFMT(mode) == S_IFSOCK + +# Names for permission bits + +S_ISUID = 0o4000 # set UID bit +S_ISGID = 0o2000 # set GID bit +S_ENFMT = S_ISGID # file locking enforcement +S_ISVTX = 0o1000 # sticky bit +S_IREAD = 0o0400 # Unix V7 synonym for S_IRUSR +S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR +S_IEXEC = 0o0100 # Unix V7 synonym for S_IXUSR +S_IRWXU = 0o0700 # mask for owner permissions +S_IRUSR = 0o0400 # read by owner +S_IWUSR = 0o0200 # write by owner +S_IXUSR = 0o0100 # execute by owner +S_IRWXG = 0o0070 # mask for group permissions +S_IRGRP = 0o0040 # read by group +S_IWGRP = 0o0020 # write by group +S_IXGRP = 0o0010 # execute by group +S_IRWXO = 0o0007 # mask for others (not in group) permissions +S_IROTH = 0o0004 # read by others +S_IWOTH = 0o0002 # write by others +S_IXOTH = 0o0001 # execute by others + +# Names for file flags + +UF_NODUMP = 0x00000001 # do not dump file +UF_IMMUTABLE = 0x00000002 # file may not be changed +UF_APPEND = 0x00000004 # file may only be appended to +UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack +UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted +UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed +UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +SF_ARCHIVED = 0x00010000 # file may be archived +SF_IMMUTABLE = 0x00020000 # file may not be changed +SF_APPEND = 0x00040000 # file may only be appended to +SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted +SF_SNAPSHOT = 0x00200000 # file is a snapshot file + + +_filemode_table = ( + ((S_IFLNK, "l"), + (S_IFREG, "-"), + (S_IFBLK, "b"), + (S_IFDIR, "d"), + (S_IFCHR, "c"), + (S_IFIFO, "p")), + + ((S_IRUSR, "r"),), + ((S_IWUSR, "w"),), + ((S_IXUSR|S_ISUID, "s"), + (S_ISUID, "S"), + (S_IXUSR, "x")), + + ((S_IRGRP, "r"),), + ((S_IWGRP, "w"),), + ((S_IXGRP|S_ISGID, "s"), + (S_ISGID, "S"), + (S_IXGRP, "x")), + + ((S_IROTH, "r"),), + ((S_IWOTH, "w"),), + ((S_IXOTH|S_ISVTX, "t"), + (S_ISVTX, "T"), + (S_IXOTH, "x")) +) + +def filemode(mode): + """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" + perm = [] + for table in _filemode_table: + for bit, char in table: + if mode & bit == bit: + perm.append(char) + break + else: + perm.append("-") + return "".join(perm) diff --git a/upip/upip_utarfile.py b/upip/upip_utarfile.py new file mode 100644 index 00000000..e9ebde18 --- /dev/null +++ b/upip/upip_utarfile.py @@ -0,0 +1,82 @@ +import uctypes + +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12), +} + +DIRTYPE = "dir" +REGTYPE = "file" + +def roundup(val, align): + return (val + align - 1) & ~(align - 1) + +def skip(f, size): + assert size % 512 == 0 + buf = bytearray(512) + while size: + size -= f.readinto(buf) + +class FileSection: + + def __init__(self, f, content_len, aligned_len): + self.f = f + self.content_len = content_len + self.align = aligned_len - content_len + + def read(self, sz=65536): + if self.content_len == 0: + return b"" + if sz > self.content_len: + sz = self.content_len + data = self.f.read(sz) + sz = len(data) + self.content_len -= sz + return data + + def skip(self): + self.f.read(self.content_len + self.align) + +class TarInfo: + + def __str__(self): + return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) + +class TarFile: + + def __init__(self, name): + self.f = open(name, "rb") + self.subf = None + + def next(self): + if self.subf: + self.subf.skip() + buf = self.f.read(512) + if not buf: + return None + + h = uctypes.struct(TAR_HEADER, uctypes.addressof(buf), uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + d = TarInfo() + d.name = str(h.name, "utf-8").rstrip() + d.size = int(bytes(h.size).rstrip(), 8) + d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] + self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) + return d + + def __iter__(self): + return self + + def __next__(self): + v = self.next() + if v is None: + raise StopIteration + return v + + def extractfile(self, tarinfo): + return tarinfo.subf