From d0fd108ec73a6524234fe91f5100ba124ab66ce1 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Fri, 30 Dec 2016 01:01:23 +0100 Subject: [PATCH] Allow `=` in parameter values. Some parameter values (e.g., BASE64 encoded binary data often ends with one or two equal signs) may contain an equal sign (`=`). The current implementation splits key-value pairs at all equal signs, which leads to errors. Especially icalendar files generated by Apple's software often feature BASE64 encoded binary data in parameter values. This patch introduces a new parameter `maxsplit` to icalendar.parser.q_split() which works similar as python's string.split(sep, maxsplit) which we then use to split parameter key-value pairs only at the first equal sign. This patch fixes #197. --- CHANGES.rst | 3 ++- src/icalendar/parser.py | 9 ++++--- src/icalendar/tests/apple_xlocation_test.py | 20 +++++--------- src/icalendar/tests/test_icalendar.py | 30 +++++++++++++++++++++ 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index df7de4c..260f0ff 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,7 +14,8 @@ New features: Bug fixes: -- *add item here* +- Don't break on parameter values which contain equal signs, e.g. base64 encoded + binary data [geier] 3.11.3 (2017-02-15) diff --git a/src/icalendar/parser.py b/src/icalendar/parser.py index 4a5fc13..7bc52eb 100644 --- a/src/icalendar/parser.py +++ b/src/icalendar/parser.py @@ -149,13 +149,14 @@ def dquote(val): # parsing helper -def q_split(st, sep=','): +def q_split(st, sep=',', maxsplit=-1): """Splits a string on char, taking double (q)uotes into considderation. """ result = [] cursor = 0 length = len(st) inquote = 0 + splits = 0 for i in range(length): ch = st[i] if ch == '"': @@ -163,8 +164,10 @@ def q_split(st, sep=','): if not inquote and ch == sep: result.append(st[cursor:i]) cursor = i + 1 - if i + 1 == length: + splits += 1 + if i + 1 == length or splits == maxsplit: result.append(st[cursor:]) + break return result @@ -227,7 +230,7 @@ class Parameters(CaselessDict): result = cls() for param in q_split(st, ';'): try: - key, val = q_split(param, '=') + key, val = q_split(param, '=', maxsplit=1) validate_token(key) # Property parameter values that are not in quoted # strings are case insensitive. diff --git a/src/icalendar/tests/apple_xlocation_test.py b/src/icalendar/tests/apple_xlocation_test.py index 12c0abb..327afbb 100644 --- a/src/icalendar/tests/apple_xlocation_test.py +++ b/src/icalendar/tests/apple_xlocation_test.py @@ -10,17 +10,11 @@ class TestEncoding(unittest.TestCase): def test_apple_xlocation(self): """ - Test if error messages are encode properly. + Test if we support base64 encoded binary data in parameter values. """ - try: - directory = os.path.dirname(__file__) - with open(os.path.join(directory, 'x_location.ics'), 'rb') as fp: - data = fp.read() - cal = icalendar.Calendar.from_ical(data) - for event in cal.walk('vevent'): - self.assertEqual(len(event.errors), 1, 'Got too many errors') - error = event.errors[0][1] - self.assertTrue(error.startswith(u'Content line could not be parsed into parts')) - - except UnicodeEncodeError as e: - self.fail("There is something wrong with encoding in the collected error messages") + directory = os.path.dirname(__file__) + with open(os.path.join(directory, 'x_location.ics'), 'rb') as fp: + data = fp.read() + cal = icalendar.Calendar.from_ical(data) + for event in cal.walk('vevent'): + self.assertEqual(len(event.errors), 0, 'Got too many errors') diff --git a/src/icalendar/tests/test_icalendar.py b/src/icalendar/tests/test_icalendar.py index 333a632..475093f 100644 --- a/src/icalendar/tests/test_icalendar.py +++ b/src/icalendar/tests/test_icalendar.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import textwrap + from icalendar.tests import unittest @@ -207,6 +209,30 @@ class IcalendarTestCase (unittest.TestCase): ('key', Parameters({'PARAM': 'pValue'}), 'value') ) + contains_base64 = ( + 'X-APPLE-STRUCTURED-LOCATION;' + 'VALUE=URI;X-ADDRESS="Kaiserliche Hofburg, 1010 Wien";' + 'X-APPLE-MAPKIT-HANDLE=CAESxQEZgr3QZXJyZWljaA==;' + 'X-APPLE-RADIUS=328.7978217977285;X-APPLE-REFERENCEFRAME=1;' + 'X-TITLE=Heldenplatz:geo:48.206686,16.363235' + ).encode('utf-8') + + self.assertEqual( + Contentline(contains_base64, strict=True).parts(), + ('X-APPLE-STRUCTURED-LOCATION', + Parameters({ + 'X-APPLE-RADIUS': '328.7978217977285', + 'X-ADDRESS': 'Kaiserliche Hofburg, 1010 Wien', + 'X-APPLE-REFERENCEFRAME': '1', + 'X-TITLE': u'HELDENPLATZ', + 'X-APPLE-MAPKIT-HANDLE': + 'CAESXQEZGR3QZXJYZWLJAA==', + 'VALUE': 'URI', + }), + 'geo:48.206686,16.363235' + ) + ) + def test_fold_line(self): from ..parser import foldline @@ -247,6 +273,10 @@ class IcalendarTestCase (unittest.TestCase): self.assertEqual(q_split('Max,Moller,"Rasmussen, Max"'), ['Max', 'Moller', '"Rasmussen, Max"']) + def test_q_split_bin(self): + from ..parser import q_split + self.assertEqual(q_split('X-SOMETHING=ABCDE==', '=', maxsplit=1), ['X-SOMETHING', 'ABCDE==']) + def test_q_join(self): from ..parser import q_join self.assertEqual(q_join(['Max', 'Moller', 'Rasmussen, Max']),