Added 'embed finders'. Cleaned up wagtail embeds

pull/37/head
Karl Hobley 2014-02-13 23:15:07 +00:00
rodzic a819527cab
commit f2ca7426ec
12 zmienionych plików z 215 dodań i 396 usunięć

Wyświetl plik

@ -1,2 +1,2 @@
from .models import Embed
from .embeds.embed import get_embed
from .embeds import get_embed

Wyświetl plik

@ -1,29 +1,53 @@
from datetime import datetime
import sys
from importlib import import_module
import requests
from django.conf import settings
from .models import Embed
import os
module_dir = os.path.dirname(__file__) # get current directory
file_path = os.path.join(module_dir, 'endpoints.json')
print file_path
print open(file_path).read()
from datetime import datetime
from django.utils import six
from wagtail.wagtailembeds.oembed_providers import get_oembed_provider
from wagtail.wagtailembeds.models import Embed
def get_embed_embedly(url, max_width=None):
# Check database
class EmbedNotFoundException(Exception): pass
class EmbedlyException(Exception): pass
class AccessDeniedEmbedlyException(EmbedlyException): pass
# Pinched from django 1.7 source code.
# TODO: Replace this with "from django.utils.module_loading import import_string" when django 1.7 is released
def import_string(dotted_path):
"""
Import a dotted module path and return the attribute/class designated by the
last name in the path. Raise ImportError if the import failed.
"""
try:
return Embed.objects.get(url=url, max_width=max_width)
except Embed.DoesNotExist:
pass
module_path, class_name = dotted_path.rsplit('.', 1)
except ValueError:
msg = "%s doesn't look like a module path" % dotted_path
six.reraise(ImportError, ImportError(msg), sys.exc_info()[2])
module = import_module(module_path)
try:
# Call embedly API
client = Embedly(key=settings.EMBEDLY_KEY)
return getattr(module, class_name)
except AttributeError:
return None
msg = 'Module "%s" does not define a "%s" attribute/class' % (
dotted_path, class_name)
six.reraise(ImportError, ImportError(msg), sys.exc_info()[2])
def embedly(url, max_width=None, key=None):
from embedly import Embedly
# Get embedly key
if key is None:
key = settings.EMBEDLY_KEY
# Get embedly client
client = Embedly(key=settings.EMBEDLY_KEY)
# Call embedly
if max_width is not None:
oembed = client.oembed(url, maxwidth=max_width, better=False)
else:
@ -31,45 +55,98 @@ def get_embed_embedly(url, max_width=None):
# Check for error
if oembed.get('error'):
return None
# Save result to database
row, created = Embed.objects.get_or_create(
url=url,
max_width=max_width,
defaults={
'type': oembed['type'],
'title': oembed['title'],
'thumbnail_url': oembed.get('thumbnail_url'),
'width': oembed.get('width'),
'height': oembed.get('height')
}
)
if oembed['error_code'] in [401, 403]:
raise AccessDeniedEmbedlyException
elif oembed['error_code'] == 404:
raise EmbedNotFoundException
else:
raise EmbedlyException
# Convert photos into HTML
if oembed['type'] == 'photo':
html = '<img src="%s" />' % (oembed['url'], )
else:
html = oembed.get('html')
if html:
row.html = html
row.last_updated = datetime.now()
row.save()
# Return embed as a dict
return {
'title': oembed['title'],
'type': oembed['type'],
'thumbnail_url': oembed.get('thumbnail_url'),
'width': oembed.get('width'),
'height': oembed.get('height'),
'html': html,
}
# Return new embed
return row
def get_embed_oembed(url, max_width=None):
pass
get_embed = get_embed_oembed
try:
from embedly import Embedly
if hasattr(settings,'EMBEDLY_KEY'):
get_embed = get_embed_embedly
except:
pass
print get_embed
def oembed(url, max_width=None):
# Find provider
provider = get_oembed_provider(url)
if provider is None:
raise EmbedNotFoundException
# Work out params
params = {'url': url, 'format': 'json', }
if max_width:
params['maxwidth'] = max_width
# Perform request
r = requests.get(provider, params=params)
if r.status_code != 200:
raise EmbedNotFoundException
oembed = r.json()
# Convert photos into HTML
if oembed['type'] == 'photo':
html = '<img src="%s" />' % (oembed['url'], )
else:
html = oembed.get('html')
# Return embed as a dict
return {
'title': oembed['title'],
'type': oembed['type'],
'thumbnail_url': oembed.get('thumbnail_url'),
'width': oembed.get('width'),
'height': oembed.get('height'),
'html': html,
}
def get_default_finder():
# Check if the user has set the embed finder manually
if hasattr(settings, 'WAGTAILEMBEDS_EMBED_FINDER'):
return import_string(settings.WAGTAILEMBEDS_EMBED_FINDER)
# Use embedly if the embedly key is set
if hasattr(settings, 'EMBEDLY_KEY'):
return embedly
# Fall back to oembed
return oembed
def get_embed(url, max_width=None, finder=None):
# Check database
try:
return Embed.objects.get(url=url, max_width=max_width)
except Embed.DoesNotExist:
pass
# Get/Call finder
if not finder:
finder = get_default_finder()
embed_dict = finder(url, max_width)
# Create database record
embed, created = Embed.objects.get_or_create(
url=url,
max_width=max_width,
defaults=embed_dict,
)
# Save
embed.last_updated = datetime.now()
embed.save()
return embed

Wyświetl plik

@ -1,80 +0,0 @@
from datetime import datetime
from django.conf import settings
from ..models import Embed
import oembed_api
class EmbedlyException(Exception): pass
class AccessDeniedEmbedlyException(Exception): pass
class NotFoundEmbedlyException(Exception): pass
def get_embed_embedly(url, max_width=None):
# Check database
try:
return Embed.objects.get(url=url, max_width=max_width)
except Embed.DoesNotExist:
pass
client = Embedly(key=settings.EMBEDLY_KEY)
if max_width is not None:
oembed = client.oembed(url, maxwidth=max_width, better=False)
else:
oembed = client.oembed(url, better=False)
# Check for error
if oembed.get('error'):
if oembed['error_code'] in [401,403]:
raise AccessDeniedEmbedlyException
elif oembed['error_code'] == 404:
raise NotFoundEmbedlyException
else:
raise EmbedlyException
return save_embed(url, max_width, oembed)
def get_embed_oembed(url, max_width=None):
# Check database
try:
return Embed.objects.get(url=url, max_width=max_width)
except Embed.DoesNotExist:
pass
oembed = oembed_api.get_embed_oembed(url, max_width)
return save_embed(url, max_width, oembed)
def save_embed(url, max_width, oembed):
row, created = Embed.objects.get_or_create(
url=url,
max_width=max_width,
defaults={
'type': oembed['type'],
'title': oembed['title'],
'thumbnail_url': oembed.get('thumbnail_url'),
'width': oembed.get('width'),
'height': oembed.get('height')
}
)
if oembed['type'] == 'photo':
html = '<img src="%s" />' % (oembed['url'], )
else:
html = oembed.get('html')
if html:
row.html = html
row.last_updated = datetime.now()
row.save()
return row
# As a default use oembed
get_embed = get_embed_oembed
try:
from embedly import Embedly
# if EMBEDLY_KEY is set and embedly library found the use embedly
if hasattr(settings,'EMBEDLY_KEY'):
get_embed = get_embed_embedly
except:
pass

Wyświetl plik

@ -1,52 +0,0 @@
import os, re
import urllib2, urllib
from datetime import datetime
import json
class NotImplementedOembedException(Exception):
pass
ENDPOINTS = {}
def get_embed_oembed(url, max_width=None):
provider = None
for endpoint in ENDPOINTS.keys():
for pattern in ENDPOINTS[endpoint]:
if re.match(pattern, url):
provider = endpoint
break
if not provider:
raise NotImplementedOembedException
params = {'url': url, 'format': 'json', }
if max_width:
params['maxwidth'] = max_width
req = provider+'?' +urllib.urlencode(params)
request = urllib2.Request(req)
opener = urllib2.build_opener()
# Some provicers were not working without a user agent
request.add_header('User-Agent','Mozilla/5.0')
return json.loads(opener.open(request).read())
# Uses the public domain collection of oembed endpoints by Mathias Panzenbpeck (panzi)
# at https://github.com/panzi/oembedendpoints/blob/master/endpoints-regexp.json
def load_oembed_endpoints():
module_dir = os.path.dirname(__file__)
endpoints_path = os.path.join(module_dir, 'endpoints.json')
with open( endpoints_path) as f:
endpoints = json.loads(f.read())
for endpoint in endpoints.keys():
endpoint_key = endpoint.replace('{format}', 'json')
ENDPOINTS[endpoint_key]=[]
for pattern in endpoints[endpoint]:
ENDPOINTS[endpoint_key].append(re.compile(pattern))
load_oembed_endpoints()

Wyświetl plik

@ -1,66 +0,0 @@
import unittest
import oembed
# Test that a bunch of oembed examples is working
# If any of these is removed or changed then the unit test will fail
# This is a unittest TestCase (and not a django.test one) since django
# database is not actually needed for these tests
TEST_DATA = [
{
'url':'http://www.youtube.com/watch?v=S3xAeTmsJfg',
'title':'Animation: Ferret dance (A series of tubes)'
},
{
'url':'http://vimeo.com/86036070',
'title':'Wagtail: A new Django CMS'
},
{
'url':'https://speakerdeck.com/harmstyler/an-introduction-to-django',
'title':'An Introduction to Django'
},
{
'url':'https://ifttt.com/recipes/144705-new-twitter-followers-in-a-google-spreadsheet',
'title':'New Twitter followers in a Google spreadsheet'
},
{
'url':'http://www.hulu.com/watch/20807/late-night-with-conan-obrien-wed-may-21-2008',
'title':'Wed, May 21, 2008 (Late Night With Conan O\'Brien)'
},
{
'url':'http://www.flickr.com/photos/dfluke/5995957175/',
'title':'Django pony!?'
},
{
'url':'http://www.slideshare.net/simon/the-django-web-application-framework',
'title':'The Django Web Application Framework'
},
{
'url':'http://www.rdio.com/artist/The_Black_Keys/album/Brothers/',
'title':'Brothers'
},
{
'url':'http://instagram.com/p/kFKCcEKmBq/',
'title':'Family holidays in #Greece!'
},
{
'url':'https://www.kickstarter.com/projects/noujaimfilms/the-square-a-film-about-the-egyptian-revolution',
'title':'Sundance Award Winning Film on the Egyptian Revolution'
},
{
'url':'http://www.dailymotion.com/video/xoxulz_babysitter_animals',
'title':'Babysitter!'
}
]
class TestEmbeds(unittest.TestCase):
def test_get_embed_oembed(self):
for td in TEST_DATA:
embed = oembed.get_embed_oembed_low(td['url'])
self.assertEqual(embed['title'], td['title'] )
self.assertIsNotNone(embed['type'] )
self.assertIsNotNone(embed['width'] )
self.assertIsNotNone(embed['height'] )
if __name__ == '__main__':
unittest.main()

Wyświetl plik

@ -1,114 +0,0 @@
[
{
"url": "http://*.blip.tv/*",
"url_re": "blip\\.tv/.+",
"example_url": "http://pycon.blip.tv/file/2058801/",
"endpoint_url": "http://blip.tv/oembed/",
"title": "blip.tv"
},
{
"url": "http://*.dailymotion.com/*",
"url_re": "dailymotion\\.com/.+",
"example_url": "http://www.dailymotion.com/video/x5ioet_phoenix-mars-lander_tech",
"endpoint_url": "http://www.dailymotion.com/api/oembed/",
"title": "Dailymotion"
},
{
"url": "http://*.flickr.com/photos/*",
"url_re": "flickr\\.com/photos/[-.\\w@]+/\\d+/?",
"example_url": "http://www.flickr.com/photos/fuffer2005/2435339994/",
"endpoint_url": "http://www.flickr.com/services/oembed/",
"title": "Flickr Photos"
},
{
"url": "http://www.hulu.com/watch/*",
"url_re": "hulu\\.com/watch/.*",
"example_url": "http://www.hulu.com/watch/20807/late-night-with-conan",
"endpoint_url": "http://www.hulu.com/api/oembed.json",
"title": "Hulu"
},
{
"url": "http://*.nfb.ca/film/*",
"url_re": "nfb\\.ca/film/[-\\w]+/?",
"example_url": "http://www.nfb.ca/film/blackfly/",
"endpoint_url": "http://www.nfb.ca/remote/services/oembed/",
"title": "National Film Board of Canada"
},
{
"url": "http://qik.com/*",
"url_re": "qik\\.com/\\w+",
"example_url": "http://qik.com/video/86776",
"endpoint_url": "http://qik.com/api/oembed.json",
"title": "Qik Video"
},
{
"url": "http://*.revision3.com/*",
"url_re": "revision3\\.com/.+",
"example_url": "http://revision3.com/diggnation/2008-04-17xsanned/",
"endpoint_url": "http://revision3.com/api/oembed/",
"title": "Revision3"
},
{
"url": "http://*.scribd.com/*",
"url_re": "scribd\\.com/.+",
"example_url": "http://www.scribd.com/doc/17896323/Indian-Automobile-industryPEST",
"endpoint_url": "http://www.scribd.com/services/oembed",
"title": "Scribd"
},
{
"url": "http://*.viddler.com/explore/*",
"url_re": "viddler\\.com/explore/.*/videos/\\w+/?",
"example_url": "http://www.viddler.com/explore/engadget/videos/14/",
"endpoint_url": "http://lab.viddler.com/services/oembed/",
"title": "Viddler Video"
},
{
"url": "http://www.vimeo.com/* and http://www.vimeo.com/groups/*/videos/*",
"url_re": "vimeo\\.com/.*",
"example_url": "http://www.vimeo.com/1211060",
"endpoint_url": "http://www.vimeo.com/api/oembed.json",
"title": "Vimeo"
},
{
"url": "http://*.youtube.com/watch*",
"url_re": "youtube\\.com/watch.+v=[\\w-]+&?",
"example_url": "http://www.youtube.com/watch?v=vk1HvP7NO5w",
"endpoint_url": "http://www.youtube.com/oembed",
"title": "YouTube"
},
{
"url": "http://dotsub.com/view/*",
"url_re": "dotsub\\.com/view/[-\\da-zA-Z]+$",
"example_url": "http://dotsub.com/view/10e3cb5e-96c7-4cfb-bcea-8ab11e04e090",
"endpoint_url": "http://dotsub.com/services/oembed",
"title": "dotSUB.com"
},
{
"url": "http://yfrog.(com|ru|com.tr|it|fr|co.il|co.uk|com.pl|pl|eu|us)/*",
"url_re": "yfrog\\.(com|ru|com\\.tr|it|fr|co\\.il|co\\.uk|com\\.pl|pl|eu|us)/[a-zA-Z0-9]+$",
"example_url": "http://yfrog.com/0wgvcpj",
"endpoint_url": "http://www.yfrog.com/api/oembed",
"title": "YFrog"
},
{
"url": "http://*.clikthrough.com/theater/video/*",
"url_re": "clikthrough\\.com/theater/video/\\d+$",
"example_url": "http://www.clikthrough.com/theater/video/55",
"endpoint_url": "http://clikthrough.com/services/oembed",
"title": "Clikthrough"
},
{
"url": "http://*.kinomap.com/*",
"url_re": "kinomap\\.com/.+",
"example_url": "http://www.kinomap.com/kms-vzkpc7",
"endpoint_url": "http://www.kinomap.com/oembed",
"title": "Kinomap"
},
{
"url": "http://*.photobucket.com/albums/*|http://*.photobucket.com/groups/*",
"url_re": "photobucket\\.com/(albums|groups)/.+$",
"example_url": "http://img.photobucket.com/albums/v211/JAV123/Michael%20Holland%20Candle%20Burning/_MG_5661.jpg",
"endpoint_url": "http://photobucket.com/oembed",
"title": "Photobucket"
}
]

Wyświetl plik

@ -2,7 +2,7 @@ from __future__ import division # Use true division
from django.utils.html import escape
from .embeds.embed import get_embed
from wagtail.wagtailembeds import get_embed
def embed_to_frontend_html(url):
@ -24,8 +24,8 @@ def embed_to_frontend_html(url):
def embed_to_editor_html(url):
# Check that the embed exists
embed = get_embed(url)
if embed is None:
return ''
return
return '<div class="embed-placeholder" contenteditable="false" data-embedtype="media" data-url="%s"><h3>%s</h3><p>%s</p><img src="%s"></div>' % (url, escape(embed.title), url, embed.thumbnail_url)

Wyświetl plik

@ -1,4 +1,4 @@
{
OEMBED_ENDPOINTS = {
"https://speakerdeck.com/oembed.{format}": [
"^http(?:s)?://speakerdeck\\.com/.+$"
],
@ -292,4 +292,30 @@
"http://www.ifttt.com/oembed/": [
"^http(?:s)?://ifttt\\.com/recipes/.+$"
]
}
}
# Compile endpoints into regular expression objects
import re
def compile_endpoints():
endpoints = {}
for endpoint in OEMBED_ENDPOINTS.keys():
endpoint_key = endpoint.replace('{format}', 'json')
endpoints[endpoint_key] = []
for pattern in OEMBED_ENDPOINTS[endpoint]:
endpoints[endpoint_key].append(re.compile(pattern))
return endpoints
OEMBED_ENDPOINTS_COMPILED = compile_endpoints()
def get_oembed_provider(url):
for endpoint in OEMBED_ENDPOINTS_COMPILED.keys():
for pattern in OEMBED_ENDPOINTS_COMPILED[endpoint]:
if re.match(pattern, url):
return endpoint
return

Wyświetl plik

@ -1,7 +1,7 @@
from django import template
from django.utils.safestring import mark_safe
from wagtail.wagtailembeds.embeds.embed import get_embed
from wagtail.wagtailembeds import get_embed
register = template.Library()

Wyświetl plik

@ -1,13 +1,45 @@
from django.test import TestCase
#from .embeds import get_embed
from unittest import skip
from wagtail.wagtailembeds import get_embed
class TestEmbeds(TestCase):
# FIXME: test currently depends on a valid EMBEDLY_KEY being set - we don't particularly
# want to put one in runtests.py. See https://github.com/torchbox/wagtail/issues/26 for
# progress on eliminating Embedly as a dependency
def DISABLEDtest_get_embed(self):
# This test will fail if the video is removed or the title is changed
embed = get_embed('http://www.youtube.com/watch?v=S3xAeTmsJfg')
self.assertEqual(embed.title, 'Animation: Ferret dance (A series of tubes)')
def setUp(self):
self.hit_count = 0
def test_get_embed(self):
embed = get_embed('www.test.com/1234', max_width=400, finder=self.dummy_finder)
# Check that the embed is correct
self.assertEqual(embed.title, "Test: www.test.com/1234")
self.assertEqual(embed.type, 'video')
self.assertEqual(embed.width, 400)
# Check that there has only been one hit to the backend
self.assertEqual(self.hit_count, 1)
# Look for the same embed again and check the hit count hasn't increased
embed = get_embed('www.test.com/1234', max_width=400, finder=self.dummy_finder)
self.assertEqual(self.hit_count, 1)
# Look for a different embed, hit count should increase
embed = get_embed('www.test.com/4321', max_width=400, finder=self.dummy_finder)
self.assertEqual(self.hit_count, 2)
# Look for the same embed with a different width, this should also increase hit count
embed = get_embed('www.test.com/4321', finder=self.dummy_finder)
self.assertEqual(self.hit_count, 3)
def dummy_finder(self, url, max_width=None):
# Up hit count
self.hit_count += 1
# Return a pretend record
return {
'title': "Test: " + url,
'type': 'video',
'thumbnail_url': '',
'width': max_width if max_width else 640,
'height': 480,
'html': "<p>Blah blah blah</p>",
}

Wyświetl plik

@ -5,8 +5,7 @@ from wagtail.wagtailadmin.modal_workflow import render_modal_workflow
from wagtail.wagtailembeds.forms import EmbedForm
from wagtail.wagtailembeds.format import embed_to_editor_html
from wagtail.wagtailembeds.embeds.oembed_api import NotImplementedOembedException
from wagtail.wagtailembeds.embeds.embed import EmbedlyException, AccessDeniedEmbedlyException, NotFoundEmbedlyException
from wagtail.wagtailembeds.embeds import EmbedNotFoundException, EmbedlyException, AccessDeniedEmbedlyException
@ -23,27 +22,24 @@ def chooser_upload(request):
form = EmbedForm(request.POST, request.FILES)
if form.is_valid():
error = None
try:
embed_html = embed_to_editor_html(form.cleaned_data['url'])
print embed_html
return render_modal_workflow(
request, None, 'wagtailembeds/chooser/embed_chosen.js',
{'embed_html': embed_html}
)
except Exception as e :
#print e
#import traceback
#traceback.print_exc()
except AccessDeniedEmbedlyException:
error = "There seems to be a problem with your embedly API key. Please check your settings."
except EmbedNotFoundException:
error = "Cannot find an embed for this URL."
except EmbedlyException:
error = "There seems to be an error with Embedly while trying to embed this URL. Please try again later."
if error:
errors = form._errors.setdefault('url', ErrorList())
if type(e) == NotImplementedOembedException:
errors.append("This URL is not supported by an oembed provider. You may try embedding it using Embedly by setting a propery EMBEDLY_KEY in your settings.")
elif type(e) == AccessDeniedEmbedlyException:
errors.append("There seems to be a problem with your embedly API key. Please check your settings.")
elif type(e) == NotFoundEmbedlyException:
errors.append("The URL you are trying to embed cannot be found.")
elif type(e) == EmbedlyException:
errors.append("There seems to be an error with Embedly while trying to embed this URL. Please try again later.")
else:
errors.append(str(e) )
errors.append(error)
return render_modal_workflow(request, 'wagtailembeds/chooser/chooser.html', 'wagtailembeds/chooser/chooser.js', {
'form': form,
})