From 03f0cbe9051b62192be97b59f84f63f9216668bf Mon Sep 17 00:00:00 2001
From: Chris Angelico <rosuav@gmail.com>
Date: Sat, 7 Jun 2014 10:24:35 +1000
Subject: [PATCH] Retain characters as UTF-8 encoded Unicode

---
 py/lexer.c |  8 +++++++-
 py/vstr.c  | 38 +++++++++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/py/lexer.c b/py/lexer.c
index c1c2146ce9..3aaa8fe8e5 100644
--- a/py/lexer.c
+++ b/py/lexer.c
@@ -546,7 +546,13 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
                         }
                     }
                     if (c != MP_LEXER_CHAR_EOF) {
-                        vstr_add_char(&lex->vstr, c);
+			if (c < 0x110000 && !is_bytes) {
+			    vstr_add_char(&lex->vstr, c);
+			} else if (c < 0x100 && is_bytes) {
+			    vstr_add_byte(&lex->vstr, c);
+			} else {
+			    assert(!"TODO: Throw an error, invalid escape code probably");
+			}
                     }
                 } else {
                     vstr_add_char(&lex->vstr, CUR_CHAR(lex));
diff --git a/py/vstr.c b/py/vstr.c
index ea6a1c937f..0b96a11e55 100644
--- a/py/vstr.c
+++ b/py/vstr.c
@@ -199,12 +199,40 @@ void vstr_add_byte(vstr_t *vstr, byte b) {
 }
 
 void vstr_add_char(vstr_t *vstr, unichar c) {
-    // TODO UNICODE
-    byte *buf = (byte*)vstr_add_len(vstr, 1);
-    if (buf == NULL) {
-        return;
+    // TODO: Can this be simplified and deduplicated?
+    // Is it worth just calling vstr_add_len(vstr, 4)?
+    if (c < 0x80) {
+        byte *buf = (byte*)vstr_add_len(vstr, 1);
+        if (buf == NULL) {
+            return;
+	}
+        *buf = (byte)c;
+    } else if (c < 0x800) {
+        byte *buf = (byte*)vstr_add_len(vstr, 2);
+        if (buf == NULL) {
+            return;
+	}
+        buf[0] = (c >> 6) | 0xC0;
+        buf[1] = (c & 0x3F) | 0x80;
+    } else if (c < 0x10000) {
+        byte *buf = (byte*)vstr_add_len(vstr, 3);
+        if (buf == NULL) {
+            return;
+	}
+        buf[0] = (c >> 12) | 0xE0;
+        buf[1] = ((c >> 6) & 0x3F) | 0x80;
+        buf[2] = (c & 0x3F) | 0x80;
+    } else {
+        assert(c < 0x110000);
+        byte *buf = (byte*)vstr_add_len(vstr, 4);
+        if (buf == NULL) {
+            return;
+	}
+        buf[0] = (c >> 18) | 0xF0;
+        buf[1] = ((c >> 12) & 0x3F) | 0x80;
+        buf[2] = ((c >> 6) & 0x3F) | 0x80;
+        buf[3] = (c & 0x3F) | 0x80;
     }
-    buf[0] = c;
 }
 
 void vstr_add_str(vstr_t *vstr, const char *str) {