ipydrawio/notebooks/Data-Driven Decks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo of Data-Driven Decks with Drawio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import base64, html, urllib.parse, pathlib as P, tempfile, asyncio, IPython, json\n",
    "import IPython.display as D, jinja2, babel.support as B\n",
    "import PyPDF2, traitlets as T, ipywidgets as W, tornado.ioloop, lxml.etree as E\n",
    "from io import BytesIO\n",
    "import urllib.parse\n",
    "from PIL import Image\n",
    "from nbconvert.filters.markdown_mistune import markdown2html_mistune\n",
    "from tornado.concurrent import run_on_executor\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "import requests, requests_cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "requests_cache.install_cache(\"ddddd\", allowable_methods=[\"GET\", \"POST\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Slides(W.HTML):\n",
    "    \"\"\" some number of slides, as PDF\n",
    "    \"\"\"\n",
    "    pdf = T.Unicode()\n",
    "    error = T.Unicode()\n",
    "    \n",
    "    def __init__(self, *args, **kwargs):\n",
    "        if not kwargs.get(\"layout\"):\n",
    "            kwargs[\"layout\"] = dict(\n",
    "                display = \"flex\",\n",
    "                flex_flow = \"column wrap\",\n",
    "                flex = \"1\",\n",
    "                height = \"100%\",\n",
    "            )\n",
    "        super().__init__(*args, **kwargs)\n",
    "    \n",
    "    @T.observe(\"pdf\", \"error\")\n",
    "    def _on_pdf(self, change):\n",
    "        if self.error:\n",
    "            self.value = f\"<code>{self.error}</code>\"\n",
    "        elif self.pdf:\n",
    "            url = f\"data:application/pdf;base64,{self.pdf}\"\n",
    "            self.value = f\"\"\"\n",
    "                <a href=\"{url}\">Preview</a>\n",
    "                <iframe \n",
    "                    src=\"{url}\" \n",
    "                    style=\"border: 0; min-width: 400px; min-height: 400px; width: 100%; height: 100%;\">\n",
    "                </iframe>\n",
    "            \"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DrawioSlides"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Before we begin\n",
    "\n",
    "> ## This is **NOT READY** for prime-time\n",
    "> Start the **demo** export server. It will try to install its dependencies with `jlpm`, and requires `nodejs`.\n",
    "> ```bash\n",
    "> !python scripts/drawio_export_demo.py\n",
    "> ```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DrawioSlides(Slides):\n",
    "    \"\"\" Slides built with drawio-export from drawio XML\n",
    "    \"\"\"\n",
    "    executor = ThreadPoolExecutor(max_workers=1)\n",
    "    ipynb = T.Unicode()\n",
    "    xml = T.Union([T.Unicode(), T.Bytes()])\n",
    "    params = T.Dict()\n",
    "    url = T.Unicode(default_value=\"http://localhost:8000\")\n",
    "    \n",
    "    CORE_PARAMS = dict(\n",
    "        format=\"pdf\",\n",
    "        base64=\"1\"\n",
    "    )\n",
    "    \n",
    "    @run_on_executor\n",
    "    def update_pdf(self):\n",
    "        # this really needs to be a queue\n",
    "        self.error = \"\"\n",
    "        self.pdf = \"\"\n",
    "        self.value = \"<blockquote>rendering...</blockqoute>\"\n",
    "        if isinstance(self.xml, str):\n",
    "            xml = self.xml\n",
    "        elif isinstance(self.xml, bytes):\n",
    "            xml = base64.b64encode(self.xml).decode(\"utf-8\")\n",
    "        try:\n",
    "            r = requests.post(\n",
    "                self.url, \n",
    "                timeout=None,\n",
    "                data=dict(\n",
    "                    xml=xml, \n",
    "                    **self.params, \n",
    "                    **self.CORE_PARAMS\n",
    "                ))\n",
    "            if r.status_code != 200:\n",
    "                self.error = r.text\n",
    "            else:\n",
    "                self.pdf = r.text\n",
    "        except Exception as err:\n",
    "            self.error = str(err)\n",
    "            \n",
    "    @T.observe(\"ipynb\")\n",
    "    def _on_ipynb(self, change=None):\n",
    "        self.error = \"\"\n",
    "        try:\n",
    "            self.xml = json.loads(self.ipynb)[\"metadata\"][\"jupyterlab-drawio\"][\"xml\"]\n",
    "        except Exception as err:\n",
    "            self.error = str(err)\n",
    "       \n",
    "    \n",
    "    @T.observe(\"xml\")\n",
    "    def _on_xml(self, change=None):\n",
    "        tornado.ioloop.IOLoop.current().add_callback(self.update_pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a1637093703405681ef262d7fba7bdf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DrawioSlides(value='', layout=Layout(display='flex', flex='1', flex_flow='column wrap', height='100%'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "HOW_IT_WORKS = P.Path(\"testfiles/How it works.dio\")\n",
    "how_it_works = DrawioSlides(xml=HOW_IT_WORKS.read_text())\n",
    "how_it_works"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other formats, like `.dio.ipynb` can also be rendered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b33a191a4c684951ad10a436fa353c25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DrawioSlides(value='', layout=Layout(display='flex', flex='1', flex_flow='column wrap', height='100%'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "DIAGRAM_NOTEBOOK = P.Path(\"Diagram Notebook.dio.ipynb\")\n",
    "diagram_notebook = DrawioSlides(ipynb=DIAGRAM_NOTEBOOK.read_text())\n",
    "diagram_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f0fe3b39aa34b808d5efa456ac15885",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DrawioSlides(value='', layout=Layout(display='flex', flex='1', flex_flow='column wrap', height='100%'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "DIAGRAM_SVG = P.Path(\"testfiles/A.dio.svg\")\n",
    "diagram_svg = DrawioSlides(xml=DIAGRAM_SVG.read_text())\n",
    "diagram_svg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7d8a91d3b0b4ec1beb611a012e5f799",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DrawioSlides(value='', layout=Layout(display='flex', flex='1', flex_flow='column wrap', height='100%'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "DIAGRAM_PNG = P.Path(\"testfiles/B.dio.png\")\n",
    "diagram_png = DrawioSlides(xml=DIAGRAM_PNG.read_bytes())\n",
    "diagram_png"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TemplatedDrawioSlides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TemplatedDrawioSlides(DrawioSlides):\n",
    "    template = T.Unicode()\n",
    "    context = T.Dict()\n",
    "    \n",
    "    AMP = \"&\"\n",
    "    _AMP_ = \"_____AMP_____\"\n",
    "\n",
    "    @T.observe(\"context\", \"template\")\n",
    "    def _on_context(self, change):\n",
    "        env = jinja2.Environment(\n",
    "          extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape'],\n",
    "          autoescape=jinja2.select_autoescape(['html', 'xml'])\n",
    "        )\n",
    "\n",
    "        self.xml = self.smudge(env.from_string(self.clean(self.template)).render(**{\n",
    "            key: self.markdown(value)\n",
    "            for key, value in (self.context or {}).items()\n",
    "        }))\n",
    "    \n",
    "    def clean(self, txt):\n",
    "        return txt.replace(self.AMP, self._AMP_)\n",
    "\n",
    "    def smudge(self, txt):\n",
    "        return txt.replace(self._AMP_, self.AMP)\n",
    "    \n",
    "    def markdown(self, md):\n",
    "        return markdown2html_mistune(md).replace(self.AMP, self._AMP_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82ca1a2997c2488bbfa6826c4164acc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "TemplatedDrawioSlides(value='', layout=Layout(display='flex', flex='1', flex_flow='column wrap', height='100%'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "TEMPLATE = P.Path(\"testfiles/template deck.dio\") \n",
    "title = TemplatedDrawioSlides(template=TEMPLATE.read_text(), context=dict(\n",
    "    hero=\"<h1>???</h1>\",\n",
    "    title=\"_No title here yet..._\"\n",
    "))\n",
    "title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "logo = \"\"\"<img src=\"https://upload.wikimedia.org/wikipedia/commons/3/38/Jupyter_logo.svg\" width=\"200\" height=\"200\"></image>\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "title.context = {\n",
    "    \"title\": \"The _title_ can contain `Markdown`\",\n",
    "    \"hero\": logo\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "ideas = []\n",
    "for i in range(4):\n",
    "    ideas += [\n",
    "        TemplatedDrawioSlides(\n",
    "            template=TEMPLATE.read_text(), \n",
    "            context=dict(\n",
    "                title=f\"# Idea {i + 1}\",\n",
    "                abstract=f\"This is idea {i + 1}. It's better than [idea {i}](#idea-{i})\",\n",
    "                hero=(logo + \"\\n\\n\") * (1 + 1) \n",
    "            ))\n",
    "    ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deck"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deck MK1 Prototype\n",
    "\n",
    "The simplest deck prototype is just a box."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "deck = W.HBox([title, how_it_works, *ideas, diagram_notebook, diagram_png, diagram_svg], \n",
    "              layout=dict(display=\"flex\", flex_flow=\"row wrap\"))\n",
    "# deck"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deck MK2 Prototype\n",
    "\n",
    "This builds a composite deck."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Deck(W.HBox):\n",
    "    composite = T.Unicode()\n",
    "    preview = T.Instance(Slides)\n",
    "    \n",
    "    def __init__(self, *args, **kwargs):\n",
    "        if \"layout\" not in kwargs:\n",
    "            kwargs[\"layout\"] = dict(display=\"flex\", flex_flow=\"row wrap\")\n",
    "        super().__init__(*args, **kwargs)\n",
    "    \n",
    "    @T.default(\"preview\")\n",
    "    def _default_preview(self):\n",
    "        slides = Slides()\n",
    "        T.dlink((self, \"composite\"), (slides, \"pdf\"))\n",
    "        self.update_composite()\n",
    "        return slides\n",
    "    \n",
    "    @T.observe(\"children\")\n",
    "    def _on_children(self, change):\n",
    "        self.update_composite()\n",
    "    \n",
    "    def extract_diagrams(self, child):\n",
    "        if isinstance(child.xml, str):\n",
    "            node = E.fromstring(child.xml)\n",
    "        elif isinstance(child.xml, bytes):\n",
    "            img = Image.open(BytesIO(child.xml))\n",
    "            node = E.fromstring(urllib.parse.unquote(img.info[\"mxfile\"]))\n",
    "        \n",
    "        tag = node.tag\n",
    "        \n",
    "        if tag == \"mxfile\":\n",
    "            for diagram in node.xpath(\"//diagram\"):\n",
    "                yield diagram\n",
    "        elif tag == \"mxGraphModel\":\n",
    "            diagram = E.Element(\"diagram\")\n",
    "            diagram.append(node)\n",
    "            yield diagram\n",
    "        elif tag == \"{http://www.w3.org/2000/svg}svg\":\n",
    "            diagrams = E.fromstring(node.attrib[\"content\"]).xpath(\"//diagram\")\n",
    "            for diagram in diagrams:\n",
    "                yield diagram\n",
    "    \n",
    "    def update_composite(self):\n",
    "        tree = E.fromstring(\"\"\"<mxfile version=\"13.3.6\"></mxfile>\"\"\")\n",
    "        with tempfile.TemporaryDirectory() as td:\n",
    "            tdp = P.Path(td)\n",
    "            merger = PyPDF2.PdfFileMerger()\n",
    "            for i, child in enumerate(self.children):\n",
    "                for diagram in self.extract_diagrams(child):\n",
    "                    tree.append(diagram)\n",
    "                next_pdf = (tdp / f\"doc-{i}.pdf\")\n",
    "                wrote = next_pdf.write_bytes(base64.b64decode(child.pdf.encode(\"utf-8\")))\n",
    "                if wrote:\n",
    "                    merger.append(PyPDF2.PdfFileReader(str(next_pdf)))\n",
    "            output_pdf = tdp / \"output.pdf\"\n",
    "            final_pdf = tdp / \"final.pdf\"\n",
    "            merger.write(str(output_pdf))\n",
    "            self.composite_xml = E.tostring(tree).decode(\"utf-8\")\n",
    "            final = PyPDF2.PdfFileWriter()\n",
    "            final.appendPagesFromReader(PyPDF2.PdfFileReader(str(output_pdf), \"rb\"))\n",
    "            final.addAttachment(\"drawing.drawio\", self.composite_xml.encode(\"utf-8\"))\n",
    "            with final_pdf.open(\"wb\") as fpt:\n",
    "                final.write(fpt)\n",
    "            self.composite = base64.b64encode(final_pdf.read_bytes()).decode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_deck = Deck(deck.children)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30e626b251464502ad0f6f36755f2bd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Slides(value='\\n                <a href=\"data:application/pdf;base64,JVBERi0xLjMKMSAwIG9iago8PAovVHlwZSAvUGFnZ…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_deck.preview"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}