From 8bab858be6699bf393694dfe05c2c49f01e5c27a Mon Sep 17 00:00:00 2001 From: Lex Neva Date: Sat, 13 Jan 2018 20:18:50 -0500 Subject: [PATCH] pyinstaller release method (#16) pyinstaller packages up all of a python script's dependencies and builds them into standalone executables. It can either do a directory (containing a single executable and a bunch of shared libraries) or a self-contained executable that effectively just contains a compressed version of the directory. The problem is, if you have several scripts like we do, you get several large directories or standalone binaries, and there's a ton of duplication between them. Fortunately it looks like using the directory method and just combining the directories works fine (for this project). This PR runs the above build on any tagged commit and publishes a release in github containing the pyinstall-ified tarball. If the tag is named like "v1.2.3" _and_ the tag is on the master branch, then the github release will be marked as "production". Otherwise, it will be marked as a "pre-release". This means that we can build testable tarballs of the extension in a pull request by tagging a commit. --- .gitignore | 7 +++- .travis.yml | 103 ++++++++++++++++++++++++++++++++++++++++------- Makefile | 17 ++++++++ bin/build-dist | 39 ++++++++++++++++++ makefile | 3 -- reorder.inx | 16 -------- reorder.py | 39 ------------------ requirements.txt | 1 + stub.py | 30 ++++++++++++++ 9 files changed, 181 insertions(+), 74 deletions(-) create mode 100644 Makefile create mode 100755 bin/build-dist delete mode 100644 makefile delete mode 100644 reorder.inx delete mode 100644 reorder.py create mode 100644 stub.py diff --git a/.gitignore b/.gitignore index 9f526dcb0..c0a09e7ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ .*.swp -*.pyc \ No newline at end of file +*.pyc +*.spec +*.zip +*.tar.gz +dist/ +build/ diff --git a/.travis.yml b/.travis.yml index 2a6e9669e..4e8ddd04d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,94 @@ language: python -cache: pip -python: - - 2.7 - - 3.6 +virtualenv: + # We need this for PyGObject. + system_site_packages: true matrix: - allow_failures: - - python: 3.6 + include: + # always lint on every commit + - python: 2.7 + env: LINT=true + sudo: false + + # only bother to build if we're going to deploy a release + - python: 2.7 + sudo: required + env: BUILD=true + if: tag is present +cache: pip install: - # - pip install -r requirements.txt - - pip install flake8 # pytest # add another testing frameworks later + - | + if [ -n "$BUILD" ]; then + # For some bizarre reason, this build has been failing due to the + # key for the mongodb repo expiring. Maybe Travis includes the + # mongodb repo by default...? + sudo rm /etc/apt/sources.list.d/mongodb*.list + + # Need inkscape >=0.92 for inkex.py and friends + sudo add-apt-repository --yes ppa:inkscape.dev/stable + sudo apt-get update + sudo apt-get install inkscape + + # for shapely + sudo apt-get install libgeos-dev + + # for wxPython + sudo apt-get install glib-networking + + # This is the same as the pypi module PyGObject. We can't just do + # "pip install PyGObject" because it depends on a version of + # libgirepository1.0-dev that doesn't exist in Trusty. + sudo apt-get install python-gi python-gi-cairo libgirepository1.0-dev + + # wxPython doen't publish linux wheels in pypi + wget https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-14.04/wxPython-4.0.0b2-cp27-cp27mu-linux_x86_64.whl + pip install wxPython-4.0.0b2-cp27-cp27mu-linux_x86_64.whl + + # We can't use the shapely wheel because it includes the geos + # library but with a weird file name. Details: + # https://github.com/pyinstaller/pyinstaller/blob/61b1c75c2b0469b32d114298a63bf60b8d597e37/PyInstaller/hooks/hook-shapely.py#L34 + pip install --no-binary shapely -r requirements.txt + + pip install pyinstaller + else + pip install flake8 + fi before_script: - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - "echo LINT: $LINT BUILD: $BUILD" script: - - true # pytest --capture=sys # add other tests here + - | + if [ -n "$LINT" ]; then + flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + elif [ -n "$BUILD" ]; then + make dist + else + true + fi notifications: - on_success: change - on_failure: change # `always` will be the setting once code changes slow down + on_success: never + on_failure: never +deploy: + - provider: releases + api_key: + secure: pYORXHcn0hPcMIo6+brVE+wYce272H4COp1iXmPvBUz64MAX0Bdm5UX6cTAvzwNd9Hhi2nnWebaoS5AiPelbZgQoZJXsy4whrp7+ZrkQZkhGcqsSqXN6j5k5xdGeFX4k37T7eGkFyajTAdIWB3locHcikKN6N6PnyCPxGD/xuxiD1fJSVKGqBOptBYsqFtMddKy3aT0nmRG/2pMElq2Fcxozo+rR00j2/3npVoh2VTRt2L0S/DrX3zKT4vi2+AQ1MmKEAfF/YxCPybJGPe+wHz4egs0+PIJYb7pSZL8Ja81IS4v5cmmy/r2la815amyRyXwZXbJwBX8h9wPa7dwGStMvJdUidlqaXjKMsWba3QbD6YHOi0+UOFvWeubCDqXKhqxVAvZyCvXCx2WPlBfGWsJDHK/j2pU5Iul5Jz2Zsa3PLYA3UeGWVy081SZNuklNdccKGTokntFnR3pGM/jDN/JK7RkvuPM5qQqn3gludQnrdo/Kw0I77hAEDasgUyO8cweSfyqOXBN0dkbLjfBVRslRRRuC5fV9MIqFvLclaPfMlxsSTdDO4MGJRsF7VvFySBdh0xK7Rm+Vb9jYjCR3FV+b8TRsnY0eD2eFM+rmQt/OYdNIs6emVrTXCcAIzq4JoKZdFDIDDRsiztGSjIBB0+rSXmiUMtCiUo5GS/7zxGs= + file: + - '*.tar.gz' + file_glob: true + skip_cleanup: true + on: + tag: true + branch: master + condition: "$TRAVIS_TAG =~ ^v[0-9.]+$" + - provider: releases + api_key: + secure: pYORXHcn0hPcMIo6+brVE+wYce272H4COp1iXmPvBUz64MAX0Bdm5UX6cTAvzwNd9Hhi2nnWebaoS5AiPelbZgQoZJXsy4whrp7+ZrkQZkhGcqsSqXN6j5k5xdGeFX4k37T7eGkFyajTAdIWB3locHcikKN6N6PnyCPxGD/xuxiD1fJSVKGqBOptBYsqFtMddKy3aT0nmRG/2pMElq2Fcxozo+rR00j2/3npVoh2VTRt2L0S/DrX3zKT4vi2+AQ1MmKEAfF/YxCPybJGPe+wHz4egs0+PIJYb7pSZL8Ja81IS4v5cmmy/r2la815amyRyXwZXbJwBX8h9wPa7dwGStMvJdUidlqaXjKMsWba3QbD6YHOi0+UOFvWeubCDqXKhqxVAvZyCvXCx2WPlBfGWsJDHK/j2pU5Iul5Jz2Zsa3PLYA3UeGWVy081SZNuklNdccKGTokntFnR3pGM/jDN/JK7RkvuPM5qQqn3gludQnrdo/Kw0I77hAEDasgUyO8cweSfyqOXBN0dkbLjfBVRslRRRuC5fV9MIqFvLclaPfMlxsSTdDO4MGJRsF7VvFySBdh0xK7Rm+Vb9jYjCR3FV+b8TRsnY0eD2eFM+rmQt/OYdNIs6emVrTXCcAIzq4JoKZdFDIDDRsiztGSjIBB0+rSXmiUMtCiUo5GS/7zxGs= + file: + - '*.tar.gz' + file_glob: true + skip_cleanup: true + prerelease: true + name: "development build $TRAVIS_TAG" + on: + tag: true + all_branches: true + condition: '! "$TRAVIS_TAG" =~ ^v[0-9.]+$' diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..0803e30a4 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +EXTENSIONS:=embroider embroider_params embroider_simulate embroider_update + +# This gets the branch name or the name of the tag +VERSION:=$(TRAVIS_TAG) +OS:=$(shell uname) +ARCH:=$(shell uname -m) + +dist: distclean + bin/build-dist $(EXTENSIONS) + cp *.inx dist + cd dist; tar zcf ../inkstitch-$(VERSION)-$(OS)-$(ARCH).tar.gz * + + # This is only here for debugging the build. + tar zcf build.tar.gz build + +distclean: + rm -rf build dist *.spec *.tar.gz diff --git a/bin/build-dist b/bin/build-dist new file mode 100755 index 000000000..f30901784 --- /dev/null +++ b/bin/build-dist @@ -0,0 +1,39 @@ +#!/bin/bash + +site_packages="$(python -c "import os; print(os.path.dirname(os.__file__) + '/site-packages')")" + +# pyinstaller misses these two +pyinstaller_args+="--add-binary /usr/lib/x86_64-linux-gnu/gio/modules/libgiolibproxy.so:. " +pyinstaller_args+="--add-binary /usr/lib/x86_64-linux-gnu/libproxy.so.1:. " + +# This one's tricky. ink/stitch doesn't actually _use_ gi.repository.Gtk, +# but it does use GTK (through wxPython). pyinstaller has some special +# logic to handle GTK apps that is engaged when you import +# gi.repository.Gtk that pulls in things like themes, icons, etc. Without +# that, the Params dialog is unthemed and barely usable. This hidden +# import option is actually the only reason we had to install python-gi +# above! +pyinstaller_args+="--hidden-import gi.repository.Gtk " + +# This lets pyinstaller see inkex.py, etc. +pyinstaller_args+="-p /usr/share/inkscape/extensions " + +mkdir -p dist/inkstitch/bin +for extension in "$@"; do + # without the LD_LIBRARY_PATH, it seems that pyinstaller can't find all of + # wxpython's shared libraries + LD_LIBRARY_PATH="${site_packages}/wx" pyinstaller $pyinstaller_args ${extension}.py; + + # By default, pyinstaller will treat each of ink/stitch's extensions + # separately. This means it packages a lot of the same shared libraries (like + # wxPython) multiple times. Turns out that we can just copy the contents of + # the directories pyinstaller creates into one and it works fine, eliminating + # the duplication. This significantly decreases the size of the inkstitch + # tarball/zip. + cp -a dist/${extension}/* dist/inkstitch/bin + rm -rf dist/${extension} + + # Inkscape doesn't let us run native binaries as extensions(?!). Instead we + # add this stub script which executes the binaries that pyinstaller creates. + cp stub.py dist/${extension}.py +done diff --git a/makefile b/makefile deleted file mode 100644 index 4c306e116..000000000 --- a/makefile +++ /dev/null @@ -1,3 +0,0 @@ -embroider.tgz: makefile index.html embroider.py embroider.inx images/draft1.jpg images/draft2.jpg images/shirt.jpg PyEmb.py - ln -fs embroider . - tar czf $@ $^ diff --git a/reorder.inx b/reorder.inx deleted file mode 100644 index 77bf59d70..000000000 --- a/reorder.inx +++ /dev/null @@ -1,16 +0,0 @@ - - - <_name>Reorder - lexelby.embroider.reorder - reorder.py - inkex.py - - all - - - - - - diff --git a/reorder.py b/reorder.py deleted file mode 100644 index a30a0a47a..000000000 --- a/reorder.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python -# -# Remove selected objects from the document and readd them in the order they -# were selected. - -import sys -sys.path.append("/usr/share/inkscape/extensions") -import os -import inkex - - -class Reorder(inkex.Effect): - - def get_selected_in_order(self): - selected = [] - - for i in self.options.ids: - path = '//*[@id="%s"]' % i - for node in self.document.xpath(path, namespaces=inkex.NSS): - selected.append(node) - - return selected - - def effect(self): - objects = self.get_selected_in_order() - - for obj in objects[1:]: - obj.getparent().remove(obj) - - insert_parent = objects[0].getparent() - insert_pos = insert_parent.index(objects[0]) - - insert_parent.remove(objects[0]) - - insert_parent[insert_pos:insert_pos] = objects - -if __name__ == '__main__': - e = Reorder() - e.affect() diff --git a/requirements.txt b/requirements.txt index c7132aa26..c029a9ed5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ networkx shapely lxml appdirs +numpy diff --git a/stub.py b/stub.py new file mode 100644 index 000000000..9fa331607 --- /dev/null +++ b/stub.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import sys +import os + +# ink/stitch +# +# stub.py: pyinstaller execution stub +# +# pyinstaller packages the inkstitch extensions into nice tidy executables. +# That's great, but Inkscape can't execute a plain binary as an extension(!). +# +# This Python script exists only to execute the actual extension binary. It +# can be copied to, e.g., "embroider_params.py", in which case it will look +# for a binary at inkstitch/bin/embroider_params. + +script_name = os.path.basename(__file__) + +if script_name.endswith('.py'): + binary_name = script_name[:-3] +else: + # Probably not right, but we can at least try. + binary_name = script_name + +binary_path = os.path.join("inkstitch", "bin", binary_name) + +args = sys.argv[:] +args[0] = binary_path + +os.execv(binary_path, args)