From bd37778efb7464c1aed3d062ff5ca2c58930984d Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Nov 2015 00:02:10 +0200 Subject: [PATCH] upip: Get rid of FFI dependency, use builtin _os module instead. Also, delete as many as possible other dependencies (needed functions are just copied into codebase). This will allow to work on static MicroPython builds (including being a first step to support bare-metal ports). --- upip/Makefile | 2 +- upip/metadata.txt | 4 +- upip/setup.py | 4 +- upip/upip.py | 41 ++++++-- upip/upip_ffilib.py | 41 -------- upip/upip_os.py | 242 ------------------------------------------- upip/upip_os_path.py | 61 ----------- upip/upip_stat.py | 149 -------------------------- 8 files changed, 37 insertions(+), 507 deletions(-) delete mode 100644 upip/upip_ffilib.py delete mode 100644 upip/upip_os.py delete mode 100644 upip/upip_os_path.py delete mode 100644 upip/upip_stat.py diff --git a/upip/Makefile b/upip/Makefile index 802f8cbf..15d0f9ed 100644 --- a/upip/Makefile +++ b/upip/Makefile @@ -2,7 +2,7 @@ all: # This target prepares snapshot of all dependency modules, for # self-contained install -deps: upip_ffilib.py upip_os.py upip_os_path.py upip_errno.py upip_gzip.py upip_stat.py upip_utarfile.py +deps: upip_errno.py upip_gzip.py upip_utarfile.py upip_ffilib.py: ../ffilib/ffilib.py cp $^ $@ diff --git a/upip/metadata.txt b/upip/metadata.txt index 2b1936f3..4043647d 100644 --- a/upip/metadata.txt +++ b/upip/metadata.txt @@ -1,7 +1,7 @@ srctype = micropython-lib type = module -version = 0.5.9 +version = 0.6 author = Paul Sokolovsky -extra_modules = upip_ffilib, upip_errno, upip_gzip, upip_os, upip_os_path, upip_stat, upip_utarfile +extra_modules = upip_errno, upip_gzip, upip_utarfile desc = Simple package manager for MicroPython. long_desc = Simple package manager for MicroPython, targetting to be self-hosted (but not yet there). Compatible only with packages without custom setup.py code. diff --git a/upip/setup.py b/upip/setup.py index 3c94a3aa..25a6fac7 100644 --- a/upip/setup.py +++ b/upip/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup(name='micropython-upip', - version='0.5.9', + version='0.6', description='Simple package manager for MicroPython.', long_description='Simple package manager for MicroPython, targetting to be self-hosted (but not yet there). Compatible only with packages without custom setup.py code.', url='https://github.com/micropython/micropython/issues/405', @@ -15,4 +15,4 @@ setup(name='micropython-upip', maintainer='MicroPython Developers', maintainer_email='micro-python@googlegroups.com', license='MIT', - py_modules=['upip', 'upip_ffilib', 'upip_errno', 'upip_gzip', 'upip_os', 'upip_os_path', 'upip_stat', 'upip_utarfile']) + py_modules=['upip', 'upip_errno', 'upip_gzip', 'upip_utarfile']) diff --git a/upip/upip.py b/upip/upip.py index 8e7f1153..66fc5a6e 100644 --- a/upip/upip.py +++ b/upip/upip.py @@ -11,9 +11,7 @@ def upip_import(mod, sub=None): return m sys = upip_import("sys") -os = upip_import("os") -#os.path = upip_import("os.path").path -ospath = upip_import("os", "path") +import _os as os errno = upip_import("errno") gzip = upip_import("gzip") @@ -35,6 +33,35 @@ cleanup_files = [".pkg.tar"] class NotFoundError(Exception): pass +def op_split(path): + if path == "": + return ("", "") + r = path.rsplit("/", 1) + if len(r) == 1: + return ("", path) + head = r[0] + if not head: + head = "/" + return (head, r[1]) + +def op_basename(path): + return op_split(path)[1] + +def _makedirs(name, mode=0o777): + ret = False + s = "" + for c in name.rstrip("/").split("/"): + s += c + "/" + try: + os.mkdir(s) + ret = True + except OSError as e: + if e.args[0] != errno.EEXIST: + raise + ret = False + return ret + + def save_file(fname, subf): outf = open(fname, "wb") while True: @@ -68,12 +95,8 @@ def install_tar(f, prefix): if save: outfname = prefix + fname if info.type == tarfile.DIRTYPE: - try: - os.makedirs(outfname) + if _makedirs(outfname): print("Created " + outfname) - except OSError as e: - if e.args[0] != errno.EEXIST: - raise else: if debug: print("Extracting " + outfname) @@ -163,7 +186,7 @@ def install_pkg(pkg_spec, install_path): assert len(packages) == 1 package_url = packages[0]["url"] print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url)) - package_fname = ospath.basename(package_url) + package_fname = op_basename(package_url) download(package_url, package_fname) data = gzdecompress(package_fname) diff --git a/upip/upip_ffilib.py b/upip/upip_ffilib.py deleted file mode 100644 index 72ceedaf..00000000 --- a/upip/upip_ffilib.py +++ /dev/null @@ -1,41 +0,0 @@ -import sys -import ffi - -_cache = {} - -def open(name, maxver=10, extra=()): - try: - return _cache[name] - except KeyError: - pass - def libs(): - if sys.platform == "linux": - yield '%s.so' % name - for i in range(maxver, -1, -1): - yield '%s.so.%u' % (name, i) - else: - for ext in ('dylib', 'dll'): - yield '%s.%s' % (name, ext) - for n in extra: - yield n - err = None - for n in libs(): - try: - l = ffi.open(n) - _cache[name] = l - return l - except OSError as e: - err = e - raise err - -def libc(): - return open("libc", 6) - -# Find out bitness of the platform, even if long ints are not supported -# TODO: All bitness differences should be removed from micropython-lib, and -# this snippet too. -bitness = 1 -v = sys.maxsize -while v: - bitness += 1 - v >>= 1 diff --git a/upip/upip_os.py b/upip/upip_os.py deleted file mode 100644 index c3f49d34..00000000 --- a/upip/upip_os.py +++ /dev/null @@ -1,242 +0,0 @@ -import ffi -import array -import ustruct as struct -import upip_errno -import upip_stat as stat_ -import upip_ffilib -try: - from _os import * -except: - pass - - -libc = upip_ffilib.libc() - -try: - errno__ = libc.var("i", "errno") - def errno_(val=None): - if val is None: - return errno__.get() - errno__.set(val) -except OSError: - __upip_errno = libc.func("p", "__errno", "") - def errno_(val=None): - if val is None: - p = __upip_errno() - buf = ffi.as_bytearray(p, 4) - return int.from_bytes(buf) - raise NotImplementedError - -chdir_ = libc.func("i", "chdir", "s") -mkdir_ = libc.func("i", "mkdir", "si") -rename_ = libc.func("i", "rename", "ss") -unlink_ = libc.func("i", "unlink", "s") -rmdir_ = libc.func("i", "rmdir", "s") -getcwd_ = libc.func("s", "getcwd", "si") -opendir_ = libc.func("P", "opendir", "s") -readdir_ = libc.func("P", "readdir", "P") -open_ = libc.func("i", "open", "sii") -read_ = libc.func("i", "read", "ipi") -write_ = libc.func("i", "write", "iPi") -close_ = libc.func("i", "close", "i") -access_ = libc.func("i", "access", "si") -fork_ = libc.func("i", "fork", "") -pipe_ = libc.func("i", "pipe", "p") -_exit_ = libc.func("v", "_exit", "i") -getpid_ = libc.func("i", "getpid", "") -waitpid_ = libc.func("i", "waitpid", "ipi") -system_ = libc.func("i", "system", "s") -getenv_ = libc.func("s", "getenv", "P") - -R_OK = const(4) -W_OK = const(2) -X_OK = const(1) -F_OK = const(0) - -O_ACCMODE = 0o0000003 -O_RDONLY = 0o0000000 -O_WRONLY = 0o0000001 -O_RDWR = 0o0000002 -O_CREAT = 0o0000100 -O_EXCL = 0o0000200 -O_NOCTTY = 0o0000400 -O_TRUNC = 0o0001000 -O_APPEND = 0o0002000 -O_NONBLOCK = 0o0004000 - -error = OSError -name = "posix" -sep = "/" -curdir = "." -pardir = ".." -environ = {"WARNING": "NOT_IMPLEMENTED"} - - -def check_error(ret): - # Return True is error was EINTR (which usually means that OS call - # should be restarted). - if ret == -1: - e = errno_() - if e == upip_errno.EINTR: - return True - raise OSError(e) - -def raise_error(): - raise OSError(errno_.get()) - - -def getcwd(): - buf = bytearray(512) - return getcwd_(buf, 512) - -def mkdir(name, mode=0o777): - e = mkdir_(name, mode) - check_error(e) - -def rename(old, new): - e = rename_(old, new) - check_error(e) - -def unlink(name): - e = unlink_(name) - check_error(e) - -def rmdir(name): - e = rmdir_(name) - check_error(e) - -def makedirs(name, mode=0o777, exist_ok=False): - exists = access(name, F_OK) - if exists: - if exist_ok: - return - raise OSError(upip_errno.EEXIST) - s = "" - for c in name.split("/"): - s += c + "/" - try: - mkdir(s) - except OSError as e: - if e.args[0] != upip_errno.EEXIST: - raise - -def ilistdir_ex(path="."): - dir = opendir_(path) - if not dir: - raise_error() - res = [] - dirent_fmt = "LLHB256s" - while True: - dirent = readdir_(dir) - if not dirent: - break - dirent = ffi.as_bytearray(dirent, struct.calcsize(dirent_fmt)) - dirent = struct.unpack(dirent_fmt, dirent) - yield dirent - -def listdir(path="."): - is_str = type(path) is not bytes - res = [] - for dirent in ilistdir_ex(path): - fname = dirent[4].split(b'\0', 1)[0] - if fname != b"." and fname != b"..": - if is_str: - fname = fsdecode(fname) - res.append(fname) - return res - -def walk(top, topdown=True): - files = [] - dirs = [] - for dirent in ilistdir_ex(top): - mode = dirent[3] << 12 - fname = dirent[4].split(b'\0', 1)[0] - if stat_.S_ISDIR(mode): - if fname != b"." and fname != b"..": - dirs.append(fsdecode(fname)) - else: - files.append(fsdecode(fname)) - if topdown: - yield top, dirs, files - for d in dirs: - yield from walk(top + "/" + d, topdown) - if not topdown: - yield top, dirs, files - -def open(n, flags, mode=0o777): - r = open_(n, flags, mode) - check_error(r) - return r - -def read(fd, n): - buf = bytearray(n) - r = read_(fd, buf, n) - check_error(r) - return bytes(buf[:r]) - -def write(fd, buf): - r = write_(fd, buf, len(buf)) - check_error(r) - return r - -def close(fd): - r = close_(fd) - check_error(r) - return r - -def access(path, mode): - return access_(path, mode) == 0 - -def chdir(dir): - r = chdir_(dir) - check_error(r) - -def fork(): - r = fork_() - check_error(r) - return r - -def pipe(): - a = array.array('i', [0, 0]) - r = pipe_(a) - check_error(r) - return a[0], a[1] - -def _exit(n): - _exit_(n) - -def getpid(): - return getpid_() - -def waitpid(pid, opts): - a = array.array('i', [0]) - r = waitpid_(pid, a, opts) - check_error(r) - return (r, a[0]) - -def system(command): - r = system_(command) - check_error(r) - return r - -def getenv(var, default=None): - var = getenv_(var) - if var is None: - return default - return var - -def fsencode(s): - if type(s) is bytes: - return s - return bytes(s, "utf-8") - -def fsdecode(s): - if type(s) is str: - return s - return str(s, "utf-8") - - -def urandom(n): - import builtins - with builtins.open("/dev/urandom", "rb") as f: - return f.read(n) diff --git a/upip/upip_os_path.py b/upip/upip_os_path.py deleted file mode 100644 index df574c6b..00000000 --- a/upip/upip_os_path.py +++ /dev/null @@ -1,61 +0,0 @@ -import upip_os - - -def normcase(s): - return s - -def normpath(s): - return s - -def abspath(s): - if s[0] != "/": - return upip_os.getcwd() + "/" + s - return s - -def join(*args): - # TODO: this is non-compliant - if type(args[0]) is bytes: - return b"/".join(args) - else: - return "/".join(args) - -def split(path): - if path == "": - return ("", "") - r = path.rsplit("/", 1) - if len(r) == 1: - return ("", path) - head = r[0] #.rstrip("/") - if not head: - head = "/" - return (head, r[1]) - -def dirname(path): - return split(path)[0] - -def basename(path): - return split(path)[1] - -def exists(path): - return upip_os.access(path, os.F_OK) - -# TODO -lexists = exists - -def isdir(path): - import upip_stat - try: - mode = upip_os.stat(path)[0] - return upip_stat.S_ISDIR(mode) - except OSError: - return False - - -def expanduser(s): - if s == "~" or s.startswith("~/"): - h = upip_os.getenv("HOME") - return h + s[1:] - if s[0] == "~": - # Sorry folks, follow conventions - return "/home/" + s[1:] - return s diff --git a/upip/upip_stat.py b/upip/upip_stat.py deleted file mode 100644 index 704adfe2..00000000 --- a/upip/upip_stat.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Constants/functions for interpreting results of os.stat() and os.lstat(). - -Suggested usage: from stat import * -""" - -# Indices for stat struct members in the tuple returned by os.stat() - -ST_MODE = 0 -ST_INO = 1 -ST_DEV = 2 -ST_NLINK = 3 -ST_UID = 4 -ST_GID = 5 -ST_SIZE = 6 -ST_ATIME = 7 -ST_MTIME = 8 -ST_CTIME = 9 - -# Extract bits from the mode - -def S_IMODE(mode): - """Return the portion of the file's mode that can be set by - os.chmod(). - """ - return mode & 0o7777 - -def S_IFMT(mode): - """Return the portion of the file's mode that describes the - file type. - """ - return mode & 0o170000 - -# Constants used as S_IFMT() for various file types -# (not all are implemented on all systems) - -S_IFDIR = 0o040000 # directory -S_IFCHR = 0o020000 # character device -S_IFBLK = 0o060000 # block device -S_IFREG = 0o100000 # regular file -S_IFIFO = 0o010000 # fifo (named pipe) -S_IFLNK = 0o120000 # symbolic link -S_IFSOCK = 0o140000 # socket file - -# Functions to test for each file type - -def S_ISDIR(mode): - """Return True if mode is from a directory.""" - return S_IFMT(mode) == S_IFDIR - -def S_ISCHR(mode): - """Return True if mode is from a character special device file.""" - return S_IFMT(mode) == S_IFCHR - -def S_ISBLK(mode): - """Return True if mode is from a block special device file.""" - return S_IFMT(mode) == S_IFBLK - -def S_ISREG(mode): - """Return True if mode is from a regular file.""" - return S_IFMT(mode) == S_IFREG - -def S_ISFIFO(mode): - """Return True if mode is from a FIFO (named pipe).""" - return S_IFMT(mode) == S_IFIFO - -def S_ISLNK(mode): - """Return True if mode is from a symbolic link.""" - return S_IFMT(mode) == S_IFLNK - -def S_ISSOCK(mode): - """Return True if mode is from a socket.""" - return S_IFMT(mode) == S_IFSOCK - -# Names for permission bits - -S_ISUID = 0o4000 # set UID bit -S_ISGID = 0o2000 # set GID bit -S_ENFMT = S_ISGID # file locking enforcement -S_ISVTX = 0o1000 # sticky bit -S_IREAD = 0o0400 # Unix V7 synonym for S_IRUSR -S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR -S_IEXEC = 0o0100 # Unix V7 synonym for S_IXUSR -S_IRWXU = 0o0700 # mask for owner permissions -S_IRUSR = 0o0400 # read by owner -S_IWUSR = 0o0200 # write by owner -S_IXUSR = 0o0100 # execute by owner -S_IRWXG = 0o0070 # mask for group permissions -S_IRGRP = 0o0040 # read by group -S_IWGRP = 0o0020 # write by group -S_IXGRP = 0o0010 # execute by group -S_IRWXO = 0o0007 # mask for others (not in group) permissions -S_IROTH = 0o0004 # read by others -S_IWOTH = 0o0002 # write by others -S_IXOTH = 0o0001 # execute by others - -# Names for file flags - -UF_NODUMP = 0x00000001 # do not dump file -UF_IMMUTABLE = 0x00000002 # file may not be changed -UF_APPEND = 0x00000004 # file may only be appended to -UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack -UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted -UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed -UF_HIDDEN = 0x00008000 # OS X: file should not be displayed -SF_ARCHIVED = 0x00010000 # file may be archived -SF_IMMUTABLE = 0x00020000 # file may not be changed -SF_APPEND = 0x00040000 # file may only be appended to -SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted -SF_SNAPSHOT = 0x00200000 # file is a snapshot file - - -_filemode_table = ( - ((S_IFLNK, "l"), - (S_IFREG, "-"), - (S_IFBLK, "b"), - (S_IFDIR, "d"), - (S_IFCHR, "c"), - (S_IFIFO, "p")), - - ((S_IRUSR, "r"),), - ((S_IWUSR, "w"),), - ((S_IXUSR|S_ISUID, "s"), - (S_ISUID, "S"), - (S_IXUSR, "x")), - - ((S_IRGRP, "r"),), - ((S_IWGRP, "w"),), - ((S_IXGRP|S_ISGID, "s"), - (S_ISGID, "S"), - (S_IXGRP, "x")), - - ((S_IROTH, "r"),), - ((S_IWOTH, "w"),), - ((S_IXOTH|S_ISVTX, "t"), - (S_ISVTX, "T"), - (S_IXOTH, "x")) -) - -def filemode(mode): - """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" - perm = [] - for table in _filemode_table: - for bit, char in table: - if mode & bit == bit: - perm.append(char) - break - else: - perm.append("-") - return "".join(perm)