From 369a759acc9d12590355c6d9f96ef7852153570f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Oct 2013 16:54:38 +0100
Subject: [PATCH 001/132] setup.py: Make sure the setuptools_available variable
is set
Otherwise it would crash if it can't import setuptools.
---
setup.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/setup.py b/setup.py
index f14f96377..aa7cfca08 100644
--- a/setup.py
+++ b/setup.py
@@ -11,6 +11,7 @@ try:
setuptools_available = True
except ImportError:
from distutils.core import setup
+ setuptools_available = False
try:
# This will create an exe that needs Microsoft Visual C++ 2008
From 32a35e441874ad9daba10c29a6a33f13a4953fbb Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Mon, 28 Oct 2013 17:35:01 +0100
Subject: [PATCH 002/132] Add support for http://www.extremetube.com
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/extremetube.py | 52 +++++++++++++++++++++++++++++
2 files changed, 53 insertions(+)
create mode 100644 youtube_dl/extractor/extremetube.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d933986f..5eed1eebd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -39,6 +39,7 @@ from .ehow import EHowIE
from .eighttracks import EightTracksIE
from .escapist import EscapistIE
from .exfm import ExfmIE
+from .extremetube import ExtremeTubeIE
from .facebook import FacebookIE
from .faz import FazIE
from .fktv import (
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
new file mode 100644
index 000000000..981de430d
--- /dev/null
+++ b/youtube_dl/extractor/extremetube.py
@@ -0,0 +1,52 @@
+import os
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_parse_urlparse,
+ compat_urllib_request,
+ compat_urllib_parse,
+)
+
+class ExtremeTubeIE(InfoExtractor):
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
+ _TEST = {
+ u'url': u'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
+ u'file': u'652431.mp4',
+ u'md5': u'1fb9228f5e3332ec8c057d6ac36f33e0',
+ u'info_dict': {
+ u"title": u"Music Video 14 british euro brit european cumshots swallow",
+ u"uploader": u"unknown",
+ u"age_limit": 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('videoid')
+ url = 'http://www.' + mobj.group('url')
+
+ req = compat_urllib_request.Request(url)
+ req.add_header('Cookie', 'age_verified=1')
+ webpage = self._download_webpage(req, video_id)
+
+ video_title = self._html_search_regex(r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, u'title')
+ uploader = self._html_search_regex(r'>Posted by:(?=<)(\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
+ video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, u'video_url'))
+ path = compat_urllib_parse_urlparse( video_url ).path
+ extension = os.path.splitext( path )[1][1:]
+ format = path.split('/')[5].split('_')[:2]
+ format = "-".join( format )
+
+ age_limit = self._rta_search(webpage)
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'uploader': uploader,
+ 'url': video_url,
+ 'ext': extension,
+ 'format': format,
+ 'format_id': format,
+ 'age_limit': age_limit,
+ }
From 77ae65877e7b4b71d446ea928fd14f973826f07b Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Mon, 28 Oct 2013 18:18:58 +0100
Subject: [PATCH 003/132] Add support for http://www.mofosex.com
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/mofosex.py | 49 ++++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+)
create mode 100644 youtube_dl/extractor/mofosex.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d933986f..045d4447a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -81,6 +81,7 @@ from .metacafe import MetacafeIE
from .metacritic import MetacriticIE
from .mit import TechTVMITIE, MITIE
from .mixcloud import MixcloudIE
+from .mofosex import MofosexIE
from .mtv import MTVIE
from .muzu import MuzuTVIE
from .myspass import MySpassIE
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
new file mode 100644
index 000000000..a0c926cd1
--- /dev/null
+++ b/youtube_dl/extractor/mofosex.py
@@ -0,0 +1,49 @@
+import os
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_parse_urlparse,
+ compat_urllib_request,
+ compat_urllib_parse,
+)
+
+class MofosexIE(InfoExtractor):
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
+ _TEST = {
+ u'url': u'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
+ u'file': u'5018.mp4',
+ u'md5': u'1b2eb47ac33cc75d4a80e3026b613c5a',
+ u'info_dict': {
+ u"title": u"Japanese Teen Music Video",
+ u"age_limit": 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('videoid')
+ url = 'http://www.' + mobj.group('url')
+
+ req = compat_urllib_request.Request(url)
+ req.add_header('Cookie', 'age_verified=1')
+ webpage = self._download_webpage(req, video_id)
+
+ video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, u'title')
+ video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, u'video_url'))
+ path = compat_urllib_parse_urlparse( video_url ).path
+ extension = os.path.splitext( path )[1][1:]
+ format = path.split('/')[5].split('_')[:2]
+ format = "-".join( format )
+
+ age_limit = self._rta_search(webpage)
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'url': video_url,
+ 'ext': extension,
+ 'format': format,
+ 'format_id': format,
+ 'age_limit': age_limit,
+ }
From 2bc67c35acece68a75284b88fcb03d69f267a63c Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Mon, 28 Oct 2013 18:22:55 +0100
Subject: [PATCH 004/132] [KeezMoviesIE] Detect URLs with numbers in the SEO
part correct
---
youtube_dl/extractor/keezmovies.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 5e05900da..786924445 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -12,7 +12,7 @@ from ..aes import (
)
class KeezMoviesIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>keezmovies\.com/video/.+?(?P<videoid>[0-9]+))'
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>keezmovies\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
_TEST = {
u'url': u'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
u'file': u'1214711.mp4',
From dcc2a706ef7df65839aa40ce5fda61f8cea36645 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Mon, 28 Oct 2013 19:23:48 +0100
Subject: [PATCH 005/132] Add support for http://www.xtube.com
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/xtube.py | 54 ++++++++++++++++++++++++++++++++
2 files changed, 55 insertions(+)
create mode 100644 youtube_dl/extractor/xtube.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d933986f..7efd097e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -149,6 +149,7 @@ from .worldstarhiphop import WorldStarHipHopIE
from .xhamster import XHamsterIE
from .xnxx import XNXXIE
from .xvideos import XVideosIE
+from .xtube import XTubeIE
from .yahoo import YahooIE, YahooSearchIE
from .youjizz import YouJizzIE
from .youku import YoukuIE
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
new file mode 100644
index 000000000..7d06a7021
--- /dev/null
+++ b/youtube_dl/extractor/xtube.py
@@ -0,0 +1,54 @@
+import os
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_parse_urlparse,
+ compat_urllib_request,
+ compat_urllib_parse,
+)
+
+class XTubeIE(InfoExtractor):
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<videoid>[^/?&]+))'
+ _TEST = {
+ u'url': u'http://www.xtube.com/watch.php?v=kVTUy_G222_',
+ u'file': u'kVTUy_G222_.mp4',
+ u'md5': u'092fbdd3cbe292c920ef6fc6a8a9cdab',
+ u'info_dict': {
+ u"title": u"strange erotica",
+ u"uploader": u"greenshowers",
+ u"age_limit": 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('videoid')
+ url = 'http://www.' + mobj.group('url')
+
+ req = compat_urllib_request.Request(url)
+ req.add_header('Cookie', 'age_verified=1')
+ webpage = self._download_webpage(req, video_id)
+
+ video_title = self._html_search_regex(r'<div class="p_5px[^>]*>([^<]+)', webpage, u'title')
+ video_uploader = self._html_search_regex(r'so_s\.addVariable\("owner_u", "([^"]+)', webpage, u'uploader', fatal=False)
+ video_description = self._html_search_regex(r'<p class="video_description">([^<]+)', webpage, u'description', default=None)
+ video_url= self._html_search_regex(r'var videoMp4 = "([^"]+)', webpage, u'video_url').replace('\\/', '/')
+ path = compat_urllib_parse_urlparse( video_url ).path
+ extension = os.path.splitext( path )[1][1:]
+ format = path.split('/')[5].split('_')[:2]
+ format[0] += 'p'
+ format[1] += 'k'
+ format = "-".join( format )
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'uploader': video_uploader,
+ 'description': video_description,
+ 'url': video_url,
+ 'ext': extension,
+ 'format': format,
+ 'format_id': format,
+ 'age_limit': 18,
+ }
From 702665c0854af6fb317600c4825c0b00e2a4c981 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Oct 2013 22:01:37 +0100
Subject: [PATCH 006/132] tests: build the filename from the info_dict if the
'file' key is missing
It will need to have the 'id' and 'ext' keys to work.
---
test/test_download.py | 39 +++++++++++++++++++++++----------------
youtube_dl/YoutubeDL.py | 2 +-
2 files changed, 24 insertions(+), 17 deletions(-)
diff --git a/test/test_download.py b/test/test_download.py
index b9a9be11d..f136176b1 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -60,9 +60,12 @@ def generator(test_case):
if not ie._WORKING:
print_skipping('IE marked as not _WORKING')
return
- if 'playlist' not in test_case and not test_case['file']:
- print_skipping('No output file specified')
- return
+ if 'playlist' not in test_case:
+ info_dict = test_case.get('info_dict', {})
+ if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
+ print_skipping('The output file cannot be know, the "file" '
+ 'key is missing or the info_dict is incomplete')
+ return
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
@@ -77,11 +80,17 @@ def generator(test_case):
finished_hook_called.add(status['filename'])
ydl.fd.add_progress_hook(_hook)
+ def get_tc_filename(tc):
+ return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
+
test_cases = test_case.get('playlist', [test_case])
- for tc in test_cases:
- try_rm(tc['file'])
- try_rm(tc['file'] + '.part')
- try_rm(tc['file'] + '.info.json')
+ def try_rm_tcs_files():
+ for tc in test_cases:
+ tc_filename = get_tc_filename(tc)
+ try_rm(tc_filename)
+ try_rm(tc_filename + '.part')
+ try_rm(tc_filename + '.info.json')
+ try_rm_tcs_files()
try:
for retry in range(1, RETRIES + 1):
try:
@@ -98,14 +107,15 @@ def generator(test_case):
break
for tc in test_cases:
+ tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
- self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
- self.assertTrue(tc['file'] in finished_hook_called)
- self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
+ self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
+ self.assertTrue(tc_filename in finished_hook_called)
+ self.assertTrue(os.path.exists(tc_filename + '.info.json'))
if 'md5' in tc:
- md5_for_file = _file_md5(tc['file'])
+ md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
- with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
+ with io.open(tc_filename + '.info.json', encoding='utf-8') as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get('info_dict', {}).items():
if isinstance(expected, compat_str) and expected.startswith('md5:'):
@@ -126,10 +136,7 @@ def generator(test_case):
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
finally:
- for tc in test_cases:
- try_rm(tc['file'])
- try_rm(tc['file'] + '.part')
- try_rm(tc['file'] + '.info.json')
+ try_rm_tcs_files()
return test_template
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 313295839..060678e9b 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -272,7 +272,7 @@ class YoutubeDL(object):
autonumber_size = 5
autonumber_templ = u'%0' + str(autonumber_size) + u'd'
template_dict['autonumber'] = autonumber_templ % self._num_downloads
- if template_dict['playlist_index'] is not None:
+ if template_dict.get('playlist_index') is not None:
template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
sanitize = lambda k, v: sanitize_filename(
From 2563bcc85cc09382d7e731709b2c8a4ad96c7ea3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Oct 2013 22:02:17 +0100
Subject: [PATCH 007/132] Add an extractor for MySpace (closes #1666)
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/myspace.py | 48 ++++++++++++++++++++++++++++++++
2 files changed, 49 insertions(+)
create mode 100644 youtube_dl/extractor/myspace.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d933986f..caaf54456 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -83,6 +83,7 @@ from .mit import TechTVMITIE, MITIE
from .mixcloud import MixcloudIE
from .mtv import MTVIE
from .muzu import MuzuTVIE
+from .myspace import MySpaceIE
from .myspass import MySpassIE
from .myvideo import MyVideoIE
from .naver import NaverIE
diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
new file mode 100644
index 000000000..050f54a5a
--- /dev/null
+++ b/youtube_dl/extractor/myspace.py
@@ -0,0 +1,48 @@
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_str,
+)
+
+
+class MySpaceIE(InfoExtractor):
+ _VALID_URL = r'https?://myspace\.com/([^/]+)/video/[^/]+/(?P<id>\d+)'
+
+ _TEST = {
+ u'url': u'https://myspace.com/coldplay/video/viva-la-vida/100008689',
+ u'info_dict': {
+ u'id': u'100008689',
+ u'ext': u'flv',
+ u'title': u'Viva La Vida',
+ u'description': u'The official Viva La Vida video, directed by Hype Williams',
+ u'uploader': u'Coldplay',
+ u'uploader_id': u'coldplay',
+ },
+ u'params': {
+ # rtmp download
+ u'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+ context = json.loads(self._search_regex(r'context = ({.*?});', webpage,
+ u'context'))
+ video = context['video']
+ rtmp_url, play_path = video['streamUrl'].split(';', 1)
+
+ return {
+ 'id': compat_str(video['mediaId']),
+ 'title': video['title'],
+ 'url': rtmp_url,
+ 'play_path': play_path,
+ 'ext': 'flv',
+ 'description': video['description'],
+ 'thumbnail': video['imageUrl'],
+ 'uploader': video['artistName'],
+ 'uploader_id': video['artistUsername'],
+ }
From dd508b7c4f0dd8881de07a4e8593d4fcdef9bae7 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo.valsorda@gmail.com>
Date: Mon, 28 Oct 2013 18:03:26 -0400
Subject: [PATCH 008/132] [tests] don't fail on network errors
This is suboptimal, but at least this way we will need to look at the logs
only to check for network errors that happen too often, instead of
parsing a ton of lines each time to see if there is some true test failing
---
test/helper.py | 17 +++++++++++++++++
test/test_download.py | 22 +++++++++++++++++-----
2 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/test/helper.py b/test/helper.py
index 777119ea5..d7bf7a828 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -5,9 +5,11 @@ import json
import os.path
import re
import types
+import sys
import youtube_dl.extractor
from youtube_dl import YoutubeDL
+from youtube_dl.utils import preferredencoding
def global_setup():
@@ -33,6 +35,21 @@ def try_rm(filename):
raise
+def report_warning(message):
+ '''
+ Print the message to stderr, it will be prefixed with 'WARNING:'
+ If stderr is a tty file the 'WARNING:' will be colored
+ '''
+ if sys.stderr.isatty() and os.name != 'nt':
+ _msg_header = u'\033[0;33mWARNING:\033[0m'
+ else:
+ _msg_header = u'WARNING:'
+ output = u'%s %s\n' % (_msg_header, message)
+ if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3:
+ output = output.encode(preferredencoding())
+ sys.stderr.write(output)
+
+
class FakeYDL(YoutubeDL):
def __init__(self, override=None):
# Different instances of the downloader can't share the same dictionary
diff --git a/test/test_download.py b/test/test_download.py
index f136176b1..565afa1b5 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -6,7 +6,14 @@ import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from test.helper import get_params, get_testcases, global_setup, try_rm, md5
+from test.helper import (
+ get_params,
+ get_testcases,
+ global_setup,
+ try_rm,
+ md5,
+ report_warning
+)
global_setup()
@@ -92,17 +99,22 @@ def generator(test_case):
try_rm(tc_filename + '.info.json')
try_rm_tcs_files()
try:
- for retry in range(1, RETRIES + 1):
+ try_num = 1
+ while True:
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
- if retry == RETRIES: raise
-
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
raise
- print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
+ if try_num == RETRIES:
+ report_warning(u'Failed due to network errors, skipping...')
+ return
+
+ print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
+
+ try_num += 1
else:
break
From 646e17a53d3885b84b03045728b3add3d50f513c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 28 Oct 2013 23:18:13 +0100
Subject: [PATCH 009/132] Fix YouTubeDL test
---
test/test_YoutubeDL.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f8cd1bdce..ffebb4ae5 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -62,10 +62,10 @@ class TestFormatSelection(unittest.TestCase):
def test_format_limit(self):
formats = [
- {u'format_id': u'meh'},
- {u'format_id': u'good'},
- {u'format_id': u'great'},
- {u'format_id': u'excellent'},
+ {u'format_id': u'meh', u'url': u'http://example.com/meh'},
+ {u'format_id': u'good', u'url': u'http://example.com/good'},
+ {u'format_id': u'great', u'url': u'http://example.com/great'},
+ {u'format_id': u'excellent', u'url': u'http://example.com/exc'},
]
info_dict = {
u'formats': formats, u'extractor': u'test', 'id': 'testvid'}
From 321a01f97110c3048e9d9c360a099d1ec8cd4479 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Oct 2013 23:37:01 +0100
Subject: [PATCH 010/132] [mtv] Remove the templates from the mediagen url
---
youtube_dl/extractor/mtv.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index e520e2bb4..e96d3952c 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -80,6 +80,8 @@ class MTVIE(InfoExtractor):
video_id = self._id_from_uri(uri)
self.report_extraction(video_id)
mediagen_url = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content'))).attrib['url']
+ # Remove the templates, like &device={device}
+ mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', u'', mediagen_url)
if 'acceptMethods' not in mediagen_url:
mediagen_url += '&acceptMethods=fms'
mediagen_page = self._download_webpage(mediagen_url, video_id,
From f6cc16f5d821a50df173b865164e4fa9cbe854af Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo.valsorda@gmail.com>
Date: Mon, 28 Oct 2013 19:07:16 -0400
Subject: [PATCH 011/132] [tests] a HTTP 503 is a transient issue
---
test/test_download.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/test/test_download.py b/test/test_download.py
index 565afa1b5..dfb04d010 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -26,6 +26,7 @@ import youtube_dl.YoutubeDL
from youtube_dl.utils import (
compat_str,
compat_urllib_error,
+ compat_HTTPError,
DownloadError,
ExtractorError,
UnavailableVideoError,
@@ -105,7 +106,7 @@ def generator(test_case):
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
- if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+ if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
From 795f28f871074aca2a74dfe67e1e75252b525c4c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Oct 2013 06:45:54 +0100
Subject: [PATCH 012/132] [youtube] Fix login (Fixes #1681)
---
youtube_dl/extractor/youtube.py | 11 ++---------
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d05d0a8c1..f3a2a32b4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -74,14 +74,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
return False
- galx = None
- dsh = None
- match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
- if match:
- galx = match.group(1)
- match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
- if match:
- dsh = match.group(1)
+ galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
+ login_page, u'Login GALX parameter')
# Log in
login_form_strs = {
@@ -95,7 +89,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
u'checkConnection': u'',
u'checkedDomains': u'youtube',
u'dnConn': u'',
- u'dsh': dsh,
u'pstMsg': u'0',
u'rmShown': u'1',
u'secTok': u'',
From 43d7895ea0f706c390bba30e1249bf4453e173bc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Oct 2013 06:48:39 +0100
Subject: [PATCH 013/132] release 2013.10.29
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 048afc8e7..1a94003bc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.10.28'
+__version__ = '2013.10.29'
From 912cbf5d4ef5b131af88e63815863c389083d077 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Oct 2013 14:00:01 +0100
Subject: [PATCH 014/132] [vevo] Fix timestamp handling
( / 1000 is implicit float division )
---
youtube_dl/extractor/vevo.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 1c1cc418d..26ec9fa1b 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -58,9 +58,9 @@ class VevoIE(InfoExtractor):
'width': int(attr['frameWidth']),
})
- date_epoch = int(self._search_regex(
- r'/Date\((\d+)\)/', video_info['launchDate'], u'launch date'))/1000
- upload_date = datetime.datetime.fromtimestamp(date_epoch)
+ timestamp_ms = int(self._search_regex(
+ r'/Date\((\d+)\)/', video_info['launchDate'], u'launch date'))
+ upload_date = datetime.datetime.fromtimestamp(timestamp_ms // 1000)
info = {
'id': video_id,
'title': video_info['title'],
From 57dd9a8f2f5885fb3d909c4905adb69b4749491c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Oct 2013 15:09:45 +0100
Subject: [PATCH 015/132] Nicer --list-formats output
---
youtube_dl/YoutubeDL.py | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 060678e9b..260cd2809 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -759,6 +759,8 @@ class YoutubeDL(object):
@staticmethod
def format_resolution(format, default='unknown'):
+ if format.get('_resolution') is not None:
+ return format['_resolution']
if format.get('height') is not None:
if format.get('width') is not None:
res = u'%sx%s' % (format['width'], format['height'])
@@ -769,19 +771,22 @@ class YoutubeDL(object):
return res
def list_formats(self, info_dict):
- formats_s = []
- for format in info_dict.get('formats', [info_dict]):
- formats_s.append(u'%-15s%-7s %-15s%s' % (
+ def line(format):
+ return (u'%-15s%-10s%-12s%s' % (
format['format_id'],
format['ext'],
- format.get('format_note', ''),
self.format_resolution(format),
+ format.get('format_note', ''),
)
)
+
+ formats_s = list(map(line, info_dict.get('formats', [info_dict])))
if len(formats_s) != 1:
- formats_s[0] += ' (worst)'
- formats_s[-1] += ' (best)'
- formats_s = "\n".join(formats_s)
- self.to_screen(u'[info] Available formats for %s:\n'
- u'format code extension note resolution\n%s' % (
- info_dict['id'], formats_s))
+ formats_s[0] += (' ' if formats_s[0] else '') + '(worst)'
+ formats_s[-1] += (' ' if formats_s[-1] else '') + '(best)'
+
+ header_line = line({
+ 'format_id': u'format code', 'ext': u'extension',
+ '_resolution': u'resolution', 'format_note': u'note'})
+ self.to_screen(u'[info] Available formats for %s:\n%s\n%s' %
+ (info_dict['id'], header_line, u"\n".join(formats_s)))
From e54fd4b23b8110779e8caff805d3078dcf042d0b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Oct 2013 15:10:09 +0100
Subject: [PATCH 016/132] [vevo] Add more format details
---
youtube_dl/extractor/vevo.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 26ec9fa1b..4d9f2a843 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -50,10 +50,11 @@ class VevoIE(InfoExtractor):
# Already sorted from worst to best quality
for rend in renditions.findall('rendition'):
attr = rend.attrib
- f_url = attr['url']
+ format_note = '%(videoCodec)s@%(videoBitrate)4sK, %(audioCodec)s@%(audioBitrate)3sK' % attr
formats.append({
- 'url': f_url,
- 'ext': determine_ext(f_url),
+ 'url': attr['url'],
+ 'format_id': attr['name'],
+ 'format_note': format_note,
'height': int(attr['frameheight']),
'width': int(attr['frameWidth']),
})
@@ -71,7 +72,4 @@ class VevoIE(InfoExtractor):
'duration': video_info['duration'],
}
- # TODO: Remove when #980 has been merged
- info.update(formats[-1])
-
return info
From 21c924f4068692786e0c5435689d10f3d17ef612 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 29 Oct 2013 20:58:49 +0100
Subject: [PATCH 017/132] [arte] Download the 'Originalversion' version if it's
the only one available (fixes #1682)
---
youtube_dl/extractor/arte.py | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index d39b48951..e10c74c11 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -158,7 +158,9 @@ class ArteTVPlus7IE(InfoExtractor):
'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
}
- formats = player_info['VSR'].values()
+ all_formats = player_info['VSR'].values()
+ # Some formats use the m3u8 protocol
+ all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
def _match_lang(f):
if f.get('versionCode') is None:
return True
@@ -170,11 +172,16 @@ class ArteTVPlus7IE(InfoExtractor):
regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l]
return any(re.match(r, f['versionCode']) for r in regexes)
# Some formats may not be in the same language as the url
- formats = filter(_match_lang, formats)
- # Some formats use the m3u8 protocol
- formats = filter(lambda f: f.get('videoFormat') != 'M3U8', formats)
- # We order the formats by quality
+ formats = filter(_match_lang, all_formats)
formats = list(formats) # in python3 filter returns an iterator
+ if not formats:
+ # Some videos are only available in the 'Originalversion'
+ # they aren't tagged as being in French or German
+ if all(f['versionCode'] == 'VO' for f in all_formats):
+ formats = all_formats
+ else:
+ raise ExtractorError(u'The formats list is empty')
+ # We order the formats by quality
if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
sort_key = lambda f: ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
else:
From b9a836515fad5df57a86412b2cd41c49869ec0d6 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo.valsorda@gmail.com>
Date: Tue, 29 Oct 2013 16:44:35 -0400
Subject: [PATCH 018/132] Update the Vimeo test vector md5
confirmed that this is indeed the first 10241 (we went off by one with
byte range 0-10240) of the full, playing mp4, so they probably
reencoded or something
---
youtube_dl/extractor/vimeo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b4dbcd2ee..c7d864a2b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -27,7 +27,7 @@ class VimeoIE(InfoExtractor):
{
u'url': u'http://vimeo.com/56015672#at=0',
u'file': u'56015672.mp4',
- u'md5': u'ae7a1d8b183758a0506b0622f37dfa14',
+ u'md5': u'8879b6cc097e987f02484baf890129e5',
u'info_dict': {
u"upload_date": u"20121220",
u"description": u"This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
From 94badb2599e54bfd711b38f3a74c552ff652d6d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Oct 2013 01:09:26 +0100
Subject: [PATCH 019/132] Fix output indenting for --list-formats
---
youtube_dl/YoutubeDL.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 260cd2809..898533496 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -780,10 +780,11 @@ class YoutubeDL(object):
)
)
- formats_s = list(map(line, info_dict.get('formats', [info_dict])))
- if len(formats_s) != 1:
- formats_s[0] += (' ' if formats_s[0] else '') + '(worst)'
- formats_s[-1] += (' ' if formats_s[-1] else '') + '(best)'
+ formats = info_dict.get('formats', [info_dict])
+ formats_s = list(map(line, formats))
+ if len(formats) > 1:
+ formats_s[0] += (' ' if formats[0].get('format_note') else '') + '(worst)'
+ formats_s[-1] += (' ' if formats[-1].get('format_note') else '') + '(best)'
header_line = line({
'format_id': u'format code', 'ext': u'extension',
From b5d0d817bc8a23ef6dc2a00d1af6fad893143206 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Oct 2013 01:09:44 +0100
Subject: [PATCH 020/132] Remove superfluous space
---
youtube_dl/extractor/common.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ce349fe20..cef4dce85 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -63,7 +63,7 @@ class InfoExtractor(object):
* ext Will be calculated from url if missing
* format A human-readable description of the format
("mp4 container with h264/opus").
- Calculated from the format_id, width, height
+ Calculated from the format_id, width, height.
and format_note fields if missing.
* format_id A short description of the format
("mp4_h264_opus" or "19")
From 72321ead7b176824d1a8b2895ad4926555e41b88 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Oct 2013 01:14:17 +0100
Subject: [PATCH 021/132] [vevo] Readd support for SMIL (Fixes #1683)
---
youtube_dl/extractor/vevo.py | 80 +++++++++++++++++++++++++++++-------
1 file changed, 66 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 4d9f2a843..3f6020f74 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -5,7 +5,7 @@ import datetime
from .common import InfoExtractor
from ..utils import (
- determine_ext,
+ compat_HTTPError,
ExtractorError,
)
@@ -16,26 +16,22 @@ class VevoIE(InfoExtractor):
(currently used by MTVIE)
"""
_VALID_URL = r'((http://www.vevo.com/watch/.*?/.*?/)|(vevo:))(?P<id>.*?)(\?|$)'
- _TEST = {
+ _TESTS = [{
u'url': u'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
u'file': u'GB1101300280.mp4',
+ u"md5": u"06bea460acb744eab74a9d7dcb4bfd61",
u'info_dict': {
u"upload_date": u"20130624",
u"uploader": u"Hurts",
u"title": u"Somebody to Die For",
- u'duration': 230,
+ u"duration": 230,
+ u"width": 1920,
+ u"height": 1080,
}
- }
+ }]
+ _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
-
- json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
- info_json = self._download_webpage(json_url, video_id, u'Downloading json info')
-
- self.report_extraction(video_id)
- video_info = json.loads(info_json)['video']
+ def _formats_from_json(self, video_info):
last_version = {'version': -1}
for version in video_info['videoVersions']:
# These are the HTTP downloads, other types are for different manifests
@@ -50,7 +46,7 @@ class VevoIE(InfoExtractor):
# Already sorted from worst to best quality
for rend in renditions.findall('rendition'):
attr = rend.attrib
- format_note = '%(videoCodec)s@%(videoBitrate)4sK, %(audioCodec)s@%(audioBitrate)3sK' % attr
+ format_note = '%(videoCodec)s@%(videoBitrate)4sk, %(audioCodec)s@%(audioBitrate)3sk' % attr
formats.append({
'url': attr['url'],
'format_id': attr['name'],
@@ -58,6 +54,62 @@ class VevoIE(InfoExtractor):
'height': int(attr['frameheight']),
'width': int(attr['frameWidth']),
})
+ return formats
+
+ def _formats_from_smil(self, smil_xml):
+ formats = []
+ smil_doc = xml.etree.ElementTree.fromstring(smil_xml.encode('utf-8'))
+ els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
+ for el in els:
+ src = el.attrib['src']
+ m = re.match(r'''(?xi)
+ (?P<ext>[a-z0-9]+):
+ (?P<path>
+ [/a-z0-9]+ # The directory and main part of the URL
+ _(?P<cbr>[0-9]+)k
+ _(?P<width>[0-9]+)x(?P<height>[0-9]+)
+ _(?P<vcodec>[a-z0-9]+)
+ _(?P<vbr>[0-9]+)
+ _(?P<acodec>[a-z0-9]+)
+ _(?P<abr>[0-9]+)
+ \.[a-z0-9]+ # File extension
+ )''', src)
+ if not m:
+ continue
+
+ format_url = self._SMIL_BASE_URL + m.group('path')
+ format_note = ('%(vcodec)s@%(vbr)4sk, %(acodec)s@%(abr)3sk' %
+ m.groupdict())
+ formats.append({
+ 'url': format_url,
+ 'format_id': u'SMIL_' + m.group('cbr'),
+ 'format_note': format_note,
+ 'ext': m.group('ext'),
+ 'width': int(m.group('width')),
+ 'height': int(m.group('height')),
+ })
+ return formats
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
+ info_json = self._download_webpage(json_url, video_id, u'Downloading json info')
+ video_info = json.loads(info_json)['video']
+
+ formats = self._formats_from_json(video_info)
+ try:
+ smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
+ self._SMIL_BASE_URL, video_id, video_id.lower())
+ smil_xml = self._download_webpage(smil_url, video_id,
+ u'Downloading SMIL info')
+ formats.extend(self._formats_from_smil(smil_xml))
+ except ExtractorError as ee:
+ if not isinstance(ee.cause, compat_HTTPError):
+ raise
+ self._downloader.report_warning(
+ u'Cannot download SMIL information, falling back to JSON ..')
timestamp_ms = int(self._search_regex(
r'/Date\((\d+)\)/', video_info['launchDate'], u'launch date'))
From 7193498811cb17a66ca57569a8588adb28ba2b27 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Oct 2013 01:17:00 +0100
Subject: [PATCH 022/132] Use index in formt string (Fixes vevo test on Python
2.6)
---
youtube_dl/YoutubeDL.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 898533496..7f73ea360 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -482,7 +482,7 @@ class YoutubeDL(object):
format['format'] = u'{id} - {res}{note}'.format(
id=format['format_id'],
res=self.format_resolution(format),
- note=u' ({})'.format(format['format_note']) if format.get('format_note') is not None else '',
+ note=u' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
)
# Automatically determine file extension if missing
if 'ext' not in format:
From 33b1d9595d853893b5d732863dc2f5eabd939637 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Oct 2013 01:17:20 +0100
Subject: [PATCH 023/132] release 2013.10.30
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1a94003bc..e8eade7ad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.10.29'
+__version__ = '2013.10.30'
From 9f1109a56424d118263963062bc5185d8415835e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 31 Oct 2013 00:20:49 +0100
Subject: [PATCH 024/132] [dailymotion] Fix support for age-restricted videos
(Fixes #1688)
---
youtube_dl/extractor/dailymotion.py | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 4c0488245..355b4ed0a 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -21,6 +21,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
"""Build a request with the family filter disabled"""
request = compat_urllib_request.Request(url)
request.add_header('Cookie', 'family_filter=off')
+ request.add_header('Cookie', 'ff=off')
return request
class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
@@ -61,6 +62,18 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
},
u'skip': u'VEVO is only available in some countries',
},
+ # age-restricted video
+ {
+ u'url': u'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
+ u'file': u'xyh2zz.mp4',
+ u'md5': u'0d667a7b9cebecc3c89ee93099c4159d',
+ u'info_dict': {
+ u'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
+ u'uploader': 'HotWaves1012',
+ u'age_limit': 18,
+ }
+
+ }
]
def _real_extract(self, url):
@@ -90,7 +103,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
# Looking for official user
r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
- webpage, 'video uploader')
+ webpage, 'video uploader', fatal=False)
+ age_limit = self._rta_search(webpage)
video_upload_date = None
mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
@@ -132,15 +146,16 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
self._list_available_subtitles(video_id)
return
- return [{
+ return {
'id': video_id,
'formats': formats,
'uploader': video_uploader,
'upload_date': video_upload_date,
'title': self._og_search_title(webpage),
'subtitles': video_subtitles,
- 'thumbnail': info['thumbnail_url']
- }]
+ 'thumbnail': info['thumbnail_url'],
+ 'age_limit': age_limit,
+ }
def _get_available_subtitles(self, video_id):
try:
From 0ef7ad5cd49d527a24c62e831cf80f2eb443276f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Oct 2013 07:55:03 +0100
Subject: [PATCH 025/132] Fix the test for dailymotion subtitles
The extractor returns a single info_dict now.
---
test/test_dailymotion_subtitles.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/test_dailymotion_subtitles.py b/test/test_dailymotion_subtitles.py
index c596415c4..ba3580ea4 100644
--- a/test/test_dailymotion_subtitles.py
+++ b/test/test_dailymotion_subtitles.py
@@ -22,7 +22,7 @@ class TestDailymotionSubtitles(unittest.TestCase):
return info_dict
def getSubtitles(self):
info_dict = self.getInfoDict()
- return info_dict[0]['subtitles']
+ return info_dict['subtitles']
def test_no_writesubtitles(self):
subtitles = self.getSubtitles()
self.assertEqual(subtitles, None)
From 5f1ea943ab6814c2f8ca2a383f990e3f4c9e5f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Oct 2013 08:07:26 +0100
Subject: [PATCH 026/132] [livestream] fix the extraction of events
It now uses a json dictionary from the webpage.
---
youtube_dl/extractor/livestream.py | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index d04da98c8..4531fd6ab 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -40,13 +40,9 @@ class LivestreamIE(InfoExtractor):
if video_id is None:
# This is an event page:
- player = get_meta_content('twitter:player', webpage)
- if player is None:
- raise ExtractorError('Couldn\'t extract event api url')
- api_url = player.replace('/player', '')
- api_url = re.sub(r'^(https?://)(new\.)', r'\1api.\2', api_url)
- info = json.loads(self._download_webpage(api_url, event_name,
- u'Downloading event info'))
+ config_json = self._search_regex(r'window.config = ({.*?});',
+ webpage, u'window config')
+ info = json.loads(config_json)['event']
videos = [self._extract_video_info(video_data['data'])
for video_data in info['feed']['data'] if video_data['type'] == u'video']
return self.playlist_result(videos, info['id'], info['full_name'])
From ac2547f5ffc30a352207336194e7bbb0435d01a7 Mon Sep 17 00:00:00 2001
From: Alex Van't Hof <alexvh@cs.columbia.edu>
Date: Thu, 31 Oct 2013 01:57:22 -0400
Subject: [PATCH 027/132] [teamcoco] Fix video url extraction for some videos
Video url extraction failed for some videos,
e.g. http://teamcoco.com/video/old-time-baseball
The url extracted was also occasionally suboptimal quality,
e.g. http://teamcoco.com/video/louis-ck-interview-george-w-bush
---
youtube_dl/extractor/teamcoco.py | 20 +++++++++++++++++---
1 file changed, 17 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index c910110ca..76246c7cc 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -3,6 +3,7 @@ import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
+ RegexNotFoundError,
)
@@ -11,7 +12,7 @@ class TeamcocoIE(InfoExtractor):
_TEST = {
u'url': u'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
u'file': u'19705.mp4',
- u'md5': u'27b6f7527da5acf534b15f21b032656e',
+ u'md5': u'cde9ba0fa3506f5f017ce11ead928f9a',
u'info_dict': {
u"description": u"Louis C.K. got starstruck by George W. Bush, so what? Part one.",
u"title": u"Louis C.K. Interview Pt. 1 11/3/11"
@@ -33,8 +34,21 @@ class TeamcocoIE(InfoExtractor):
data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
data = self._download_webpage(data_url, video_id, 'Downloading data webpage')
- video_url = self._html_search_regex(r'<file [^>]*type="high".*?>(.*?)</file>',
- data, u'video URL')
+
+ qualities = [ '1080p', '720p', '1000k', '480p', '500k' ]
+ best_quality_idx = len(qualities)+1 # First regex match may not be optimal
+ for idx, quality in enumerate(qualities):
+ regex = r'<file [^>]*type="(?:high|standard)".*?>(.*%s.*)</file>' % quality
+ try:
+ url = self._html_search_regex(regex, data, u'video URL')
+ if idx < best_quality_idx:
+ video_url = url
+ best_quality_idx = idx
+ except RegexNotFoundError:
+ # Just catch fatal exc. Don't want the fatal=False warning
+ continue
+ if not video_url:
+ raise RegexNotFoundError(u'Unable to extract video URL')
return [{
'id': video_id,
From ab4e15134719e6c01a3a9768f21a0f361e4b781d Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Fri, 1 Nov 2013 01:24:23 +0100
Subject: [PATCH 028/132] [CinemassacreIE] Support more embed urls
---
youtube_dl/extractor/cinemassacre.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 2fe1033f0..8f9396d6b 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -41,7 +41,7 @@ class CinemassacreIE(InfoExtractor):
webpage_url = u'http://' + mobj.group('url')
webpage = self._download_webpage(webpage_url, None) # Don't know video id yet
video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
- mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/(?:embed|player)\.php\?id=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
+ mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?id=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
if not mobj:
raise ExtractorError(u'Can\'t extract embed url and video id')
playerdata_url = mobj.group(u'embed_url')
From 66cf3ac3426b62fb960b4de770c4ea8203a0e205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Nov 2013 11:55:35 +0100
Subject: [PATCH 029/132] [metacafe] Fix support for age-restricted videos
(fixes #1696)
The 'Content-Type' header must be set for disabling the family filter.
The 'flashversion' cookie is only needed for AnyClip videos.
Added tests for standard metacafe videos and for age-restricted videos.
Also set the 'age_limit' field.
---
youtube_dl/extractor/metacafe.py | 51 ++++++++++++++++++++++++++++----
1 file changed, 46 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 234b9e80f..91480ba87 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -20,7 +20,9 @@ class MetacafeIE(InfoExtractor):
_DISCLAIMER = 'http://www.metacafe.com/family_filter/'
_FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
IE_NAME = u'metacafe'
- _TESTS = [{
+ _TESTS = [
+ # Youtube video
+ {
u"add_ie": ["Youtube"],
u"url": u"http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
u"file": u"_aUehQsCQtM.mp4",
@@ -32,15 +34,42 @@ class MetacafeIE(InfoExtractor):
u"uploader_id": u"PBS"
}
},
+ # Normal metacafe video
+ {
+ u'url': u'http://www.metacafe.com/watch/11121940/news_stuff_you_wont_do_with_your_playstation_4/',
+ u'md5': u'6e0bca200eaad2552e6915ed6fd4d9ad',
+ u'info_dict': {
+ u'id': u'11121940',
+ u'ext': u'mp4',
+ u'title': u'News: Stuff You Won\'t Do with Your PlayStation 4',
+ u'uploader': u'ign',
+ u'description': u'Sony released a massive FAQ on the PlayStation Blog detailing the PS4\'s capabilities and limitations.',
+ },
+ },
+ # AnyClip video
{
u"url": u"http://www.metacafe.com/watch/an-dVVXnuY7Jh77J/the_andromeda_strain_1971_stop_the_bomb_part_3/",
u"file": u"an-dVVXnuY7Jh77J.mp4",
u"info_dict": {
u"title": u"The Andromeda Strain (1971): Stop the Bomb Part 3",
u"uploader": u"anyclip",
- u"description": u"md5:38c711dd98f5bb87acf973d573442e67"
- }
- }]
+ u"description": u"md5:38c711dd98f5bb87acf973d573442e67",
+ },
+ },
+ # age-restricted video
+ {
+ u'url': u'http://www.metacafe.com/watch/5186653/bbc_internal_christmas_tape_79_uncensored_outtakes_etc/',
+ u'md5': u'98dde7c1a35d02178e8ab7560fe8bd09',
+ u'info_dict': {
+ u'id': u'5186653',
+ u'ext': u'mp4',
+ u'title': u'BBC INTERNAL Christmas Tape \'79 - UNCENSORED Outtakes, Etc.',
+ u'uploader': u'Dwayne Pipe',
+ u'description': u'md5:950bf4c581e2c059911fa3ffbe377e4b',
+ u'age_limit': 18,
+ },
+ },
+ ]
def report_disclaimer(self):
@@ -62,6 +91,7 @@ class MetacafeIE(InfoExtractor):
'submit': "Continue - I'm over 18",
}
request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
try:
self.report_age_confirmation()
compat_urllib_request.urlopen(request).read()
@@ -83,7 +113,12 @@ class MetacafeIE(InfoExtractor):
# Retrieve video webpage to extract further information
req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
- req.headers['Cookie'] = 'flashVersion=0;'
+
+ # AnyClip videos require the flashversion cookie so that we get the link
+ # to the mp4 file
+ mobj_an = re.match(r'^an-(.*?)$', video_id)
+ if mobj_an:
+ req.headers['Cookie'] = 'flashVersion=0;'
webpage = self._download_webpage(req, video_id)
# Extract URL, uploader and title from webpage
@@ -125,6 +160,11 @@ class MetacafeIE(InfoExtractor):
r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
webpage, u'uploader nickname', fatal=False)
+ if re.search(r'"contentRating":"restricted"', webpage) is not None:
+ age_limit = 18
+ else:
+ age_limit = 0
+
return {
'_type': 'video',
'id': video_id,
@@ -134,4 +174,5 @@ class MetacafeIE(InfoExtractor):
'upload_date': None,
'title': video_title,
'ext': video_ext,
+ 'age_limit': age_limit,
}
From 60d142aa8d896674ca2b062a53b3d18c644192ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Nov 2013 22:28:51 +0100
Subject: [PATCH 030/132] Add an extractor for vk.com (closes #1635)
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/vk.py | 45 ++++++++++++++++++++++++++++++++
2 files changed, 46 insertions(+)
create mode 100644 youtube_dl/extractor/vk.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index caaf54456..bcf1cce7f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -142,6 +142,7 @@ from .videofyme import VideofyMeIE
from .videopremium import VideoPremiumIE
from .vimeo import VimeoIE, VimeoChannelIE
from .vine import VineIE
+from .vk import VKIE
from .wat import WatIE
from .websurg import WeBSurgIE
from .weibo import WeiboIE
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
new file mode 100644
index 000000000..90d8a6d07
--- /dev/null
+++ b/youtube_dl/extractor/vk.py
@@ -0,0 +1,45 @@
+# encoding: utf-8
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_str,
+ unescapeHTML,
+)
+
+
+class VKIE(InfoExtractor):
+ IE_NAME = u'vk.com'
+ _VALID_URL = r'https?://vk\.com/(?:videos.*?\?.*?z=)?video(?P<id>.*?)(?:\?|%2F|$)'
+
+ _TEST = {
+ u'url': u'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521',
+ u'md5': u'0deae91935c54e00003c2a00646315f0',
+ u'info_dict': {
+ u'id': u'162222515',
+ u'ext': u'flv',
+ u'title': u'ProtivoGunz - Хуёвая песня',
+ u'uploader': u'Noize MC',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ info_url = 'http://vk.com/al_video.php?act=show&al=1&video=%s' % video_id
+ info_page = self._download_webpage(info_url, video_id)
+ m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
+ if m_yt is not None:
+ self.to_screen(u'Youtube video detected')
+ return self.url_result(m_yt.group(1), 'Youtube')
+ vars_json = self._search_regex(r'var vars = ({.*?});', info_page, u'vars')
+ vars = json.loads(vars_json)
+
+ return {
+ 'id': compat_str(vars['vid']),
+ 'url': vars['url240'],
+ 'title': unescapeHTML(vars['md_title']),
+ 'thumbnail': vars['jpg'],
+ 'uploader': vars['md_author'],
+ }
From 8eddf3e91ddab3bb766bc5176edb3120be5743ea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 2 Nov 2013 11:21:05 +0100
Subject: [PATCH 031/132] [youtube] Encode subtitle track name in request
(Fixes #1700)
---
youtube_dl/extractor/youtube.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f3a2a32b4..dc601de52 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1111,7 +1111,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
'lang': lang,
'v': video_id,
'fmt': self._downloader.params.get('subtitlesformat'),
- 'name': l[0],
+ 'name': l[0].encode('utf-8'),
})
url = u'http://www.youtube.com/api/timedtext?' + params
sub_lang_list[lang] = url
From aa2484e390d8a5e74d740fda61b4062a4a8c1d0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 2 Nov 2013 11:21:36 +0100
Subject: [PATCH 032/132] release 2013.11.02
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e8eade7ad..75a46a2d5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.10.30'
+__version__ = '2013.11.02'
From 31366066bd18cfd32de901264f53f42fe96f55c2 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sat, 2 Nov 2013 18:08:16 +0100
Subject: [PATCH 033/132] Add support for live parameter to rtmpdump
---
youtube_dl/FileDownloader.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 8ecabab1a..0804dfbe1 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -267,7 +267,7 @@ class FileDownloader(object):
self.to_screen(u'\r%s[download] 100%% of %s in %s' %
(clear_line, data_len_str, self.format_seconds(tot_time)))
- def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
+ def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live):
self.report_destination(filename)
tmpfilename = self.temp_name(filename)
test = self.params.get('test', False)
@@ -294,6 +294,8 @@ class FileDownloader(object):
basic_args += ['--tcUrl', url]
if test:
basic_args += ['--stop', '1']
+ if live:
+ basic_args += ['--live']
args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
if self.params.get('verbose', False):
try:
@@ -411,7 +413,8 @@ class FileDownloader(object):
info_dict.get('player_url', None),
info_dict.get('page_url', None),
info_dict.get('play_path', None),
- info_dict.get('tc_url', None))
+ info_dict.get('tc_url', None),
+ info_dict.get('live', False))
# Attempt to download using mplayer
if url.startswith('mms') or url.startswith('rtsp'):
From 0a43ddf3209e13f5e87b07c440e03a45deea3e57 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sat, 2 Nov 2013 18:08:35 +0100
Subject: [PATCH 034/132] [CinemassacreIE] Add live paramter to extracted info
as a workaround
---
youtube_dl/extractor/cinemassacre.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 2fe1033f0..79d879ced 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -65,6 +65,7 @@ class CinemassacreIE(InfoExtractor):
{
'url': url,
'play_path': 'mp4:' + sd_file,
+ 'live': True, # workaround
'ext': 'flv',
'format': 'sd',
'format_id': 'sd',
@@ -72,6 +73,7 @@ class CinemassacreIE(InfoExtractor):
{
'url': url,
'play_path': 'mp4:' + hd_file,
+ 'live': True, # workaround
'ext': 'flv',
'format': 'hd',
'format_id': 'hd',
From 72a5b4f70216fe1a5b1c22be34653ae0ff81058a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Nov 2013 19:01:01 +0100
Subject: [PATCH 035/132] Add an extractor for bambuser.com (#1702)
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/bambuser.py | 42 ++++++++++++++++++++++++++++++++
2 files changed, 43 insertions(+)
create mode 100644 youtube_dl/extractor/bambuser.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bcf1cce7f..a1e35eb46 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -9,6 +9,7 @@ from .arte import (
ArteTVFutureIE,
)
from .auengine import AUEngineIE
+from .bambuser import BambuserIE
from .bandcamp import BandcampIE
from .bliptv import BlipTVIE, BlipTVUserIE
from .bloomberg import BloombergIE
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
new file mode 100644
index 000000000..cf8da22e3
--- /dev/null
+++ b/youtube_dl/extractor/bambuser.py
@@ -0,0 +1,42 @@
+import re
+import json
+
+from .common import InfoExtractor
+
+
+class BambuserIE(InfoExtractor):
+ _VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
+ _API_KEY = '005f64509e19a868399060af746a00aa'
+
+ _TEST = {
+ u'url': u'http://bambuser.com/v/4050584',
+ u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
+ u'info_dict': {
+ u'id': u'4050584',
+ u'ext': u'flv',
+ u'title': u'Education engineering days - lightning talks',
+ u'duration': 3741,
+ u'uploader': u'pixelversity',
+ u'uploader_id': u'344706',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ info_url = ('http://player-c.api.bambuser.com/getVideo.json?'
+ '&api_key=%s&vid=%s' % (self._API_KEY, video_id))
+ info_json = self._download_webpage(info_url, video_id)
+ info = json.loads(info_json)['result']
+
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'url': info['url'],
+ 'thumbnail': info['preview'],
+ 'duration': int(info['length']),
+ 'view_count': int(info['views_total']),
+ 'uploader': info['username'],
+ 'uploader_id': info['uid'],
+ }
+
From 1f343eaabbb9e0daf67363b7737833cf5e2a3e16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Sat, 2 Nov 2013 18:01:05 +0100
Subject: [PATCH 036/132] [subtitles] refactor to support websites with
subtitle information the webpage.
I added the parameter webpage, so now it's similar to the way automatic
captions are handled. This is an improvement needed for websites like
TED.
---
youtube_dl/extractor/dailymotion.py | 6 +++---
youtube_dl/extractor/subtitles.py | 12 ++++++------
youtube_dl/extractor/youtube.py | 2 +-
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7d8353946..3aef82bcf 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -113,9 +113,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
video_url = info[max_quality]
# subtitles
- video_subtitles = self.extract_subtitles(video_id)
+ video_subtitles = self.extract_subtitles(video_id, webpage)
if self._downloader.params.get('listsubtitles', False):
- self._list_available_subtitles(video_id)
+ self._list_available_subtitles(video_id, webpage)
return
return [{
@@ -129,7 +129,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
'thumbnail': info['thumbnail_url']
}]
- def _get_available_subtitles(self, video_id):
+ def _get_available_subtitles(self, video_id, webpage):
try:
sub_list = self._download_webpage(
'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
index 90de7de3a..4b4c5235d 100644
--- a/youtube_dl/extractor/subtitles.py
+++ b/youtube_dl/extractor/subtitles.py
@@ -12,9 +12,9 @@ class SubtitlesInfoExtractor(InfoExtractor):
return any([self._downloader.params.get('writesubtitles', False),
self._downloader.params.get('writeautomaticsub')])
- def _list_available_subtitles(self, video_id, webpage=None):
+ def _list_available_subtitles(self, video_id, webpage):
""" outputs the available subtitles for the video """
- sub_lang_list = self._get_available_subtitles(video_id)
+ sub_lang_list = self._get_available_subtitles(video_id, webpage)
auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
sub_lang = ",".join(list(sub_lang_list.keys()))
self.to_screen(u'%s: Available subtitles for video: %s' %
@@ -23,7 +23,7 @@ class SubtitlesInfoExtractor(InfoExtractor):
self.to_screen(u'%s: Available automatic captions for video: %s' %
(video_id, auto_lang))
- def extract_subtitles(self, video_id, video_webpage=None):
+ def extract_subtitles(self, video_id, webpage):
"""
returns {sub_lang: sub} ,{} if subtitles not found or None if the
subtitles aren't requested.
@@ -32,9 +32,9 @@ class SubtitlesInfoExtractor(InfoExtractor):
return None
available_subs_list = {}
if self._downloader.params.get('writeautomaticsub', False):
- available_subs_list.update(self._get_available_automatic_caption(video_id, video_webpage))
+ available_subs_list.update(self._get_available_automatic_caption(video_id, webpage))
if self._downloader.params.get('writesubtitles', False):
- available_subs_list.update(self._get_available_subtitles(video_id))
+ available_subs_list.update(self._get_available_subtitles(video_id, webpage))
if not available_subs_list: # error, it didn't get the available subtitles
return {}
@@ -74,7 +74,7 @@ class SubtitlesInfoExtractor(InfoExtractor):
return
return sub
- def _get_available_subtitles(self, video_id):
+ def _get_available_subtitles(self, video_id, webpage):
"""
returns {sub_lang: url} or {} if not available
Must be redefined by the subclasses
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4347651d7..d7c9b38f9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1099,7 +1099,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
else:
raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
- def _get_available_subtitles(self, video_id):
+ def _get_available_subtitles(self, video_id, webpage):
try:
sub_list = self._download_webpage(
'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
From a9a3876d55be943a7eaf505cbeb8fb862514db6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Sat, 2 Nov 2013 19:48:39 +0100
Subject: [PATCH 037/132] [ted] Added support for subtitle download
---
test/test_ted_subtitles.py | 63 +++++++++++++++++++++++++++++++++++++
youtube_dl/extractor/ted.py | 28 ++++++++++++++---
2 files changed, 87 insertions(+), 4 deletions(-)
create mode 100644 test/test_ted_subtitles.py
diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
new file mode 100644
index 000000000..3283253ab
--- /dev/null
+++ b/test/test_ted_subtitles.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import hashlib
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.extractor import TEDIE
+from youtube_dl.utils import *
+from helper import FakeYDL
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class TestTedSubtitles(unittest.TestCase):
+ def setUp(self):
+ self.DL = FakeYDL()
+ self.url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
+ def getInfoDict(self):
+ IE = TEDIE(self.DL)
+ info_dict = IE.extract(self.url)
+ return info_dict
+ def getSubtitles(self):
+ info_dict = self.getInfoDict()
+ return info_dict[0]['subtitles']
+ def test_no_writesubtitles(self):
+ subtitles = self.getSubtitles()
+ self.assertEqual(subtitles, None)
+ def test_subtitles(self):
+ self.DL.params['writesubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d')
+ def test_subtitles_lang(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['subtitleslangs'] = ['fr']
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6')
+ def test_allsubtitles(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(len(subtitles.keys()), 28)
+ def test_list_subtitles(self):
+ self.DL.params['listsubtitles'] = True
+ info_dict = self.getInfoDict()
+ self.assertEqual(info_dict, [None])
+ def test_automatic_captions(self):
+ self.DL.params['writeautomaticsub'] = True
+ self.DL.params['subtitleslang'] = ['en']
+ subtitles = self.getSubtitles()
+ self.assertTrue(len(subtitles.keys()) == 0)
+ def test_multiple_langs(self):
+ self.DL.params['writesubtitles'] = True
+ langs = ['es', 'fr', 'de']
+ self.DL.params['subtitleslangs'] = langs
+ subtitles = self.getSubtitles()
+ for lang in langs:
+ self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index dfa1176a3..239e2a448 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -1,10 +1,9 @@
import json
import re
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
-
-class TEDIE(InfoExtractor):
+class TEDIE(SubtitlesInfoExtractor):
_VALID_URL=r'''http://www\.ted\.com/
(
((?P<type_playlist>playlists)/(?P<playlist_id>\d+)) # We have a playlist
@@ -82,11 +81,21 @@ class TEDIE(InfoExtractor):
'url': stream['file'],
'format': stream['id']
} for stream in info['htmlStreams']]
+
+ video_id = info['id']
+
+ # subtitles
+ video_subtitles = self.extract_subtitles(video_id, webpage)
+ if self._downloader.params.get('listsubtitles', False):
+ self._list_available_subtitles(video_id, webpage)
+ return
+
info = {
- 'id': info['id'],
+ 'id': video_id,
'title': title,
'thumbnail': thumbnail,
'description': desc,
+ 'subtitles': video_subtitles,
'formats': formats,
}
@@ -94,3 +103,14 @@ class TEDIE(InfoExtractor):
info.update(info['formats'][-1])
return info
+
+ def _get_available_subtitles(self, video_id, webpage):
+ options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
+ languages = re.findall(r'(?:<option value=")(\S+)"', options)
+ if languages:
+ sub_lang_list = {}
+ for l in languages:
+ url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
+ sub_lang_list[l] = url
+ return sub_lang_list
+ return {}
From 165e3bb67a6d737f33d0aa2024c652b363d85ebe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Nov 2013 19:50:57 +0100
Subject: [PATCH 038/132] [bambuser] Add an extractor for channels (closes
#1702)
---
test/test_playlists.py | 9 +++++++
youtube_dl/extractor/__init__.py | 2 +-
youtube_dl/extractor/bambuser.py | 40 +++++++++++++++++++++++++++++++-
3 files changed, 49 insertions(+), 2 deletions(-)
diff --git a/test/test_playlists.py b/test/test_playlists.py
index d6a8d56df..de1e8d88e 100644
--- a/test/test_playlists.py
+++ b/test/test_playlists.py
@@ -20,6 +20,7 @@ from youtube_dl.extractor import (
SoundcloudUserIE,
LivestreamIE,
NHLVideocenterIE,
+ BambuserChannelIE,
)
@@ -85,5 +86,13 @@ class TestPlaylists(unittest.TestCase):
self.assertEqual(result['title'], u'Highlights')
self.assertEqual(len(result['entries']), 12)
+ def test_bambuser_channel(self):
+ dl = FakeYDL()
+ ie = BambuserChannelIE(dl)
+ result = ie.extract('http://bambuser.com/channel/pixelversity')
+ self.assertIsPlaylist(result)
+ self.assertEqual(result['title'], u'pixelversity')
+ self.assertTrue(len(result['entries']) >= 66)
+
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a1e35eb46..a69c08f51 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -9,7 +9,7 @@ from .arte import (
ArteTVFutureIE,
)
from .auengine import AUEngineIE
-from .bambuser import BambuserIE
+from .bambuser import BambuserIE, BambuserChannelIE
from .bandcamp import BandcampIE
from .bliptv import BlipTVIE, BlipTVUserIE
from .bloomberg import BloombergIE
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index cf8da22e3..f3b36f473 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -1,10 +1,15 @@
import re
import json
+import itertools
from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_request,
+)
class BambuserIE(InfoExtractor):
+ IE_NAME = u'bambuser'
_VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
_API_KEY = '005f64509e19a868399060af746a00aa'
@@ -33,10 +38,43 @@ class BambuserIE(InfoExtractor):
'id': video_id,
'title': info['title'],
'url': info['url'],
- 'thumbnail': info['preview'],
+ 'thumbnail': info.get('preview'),
'duration': int(info['length']),
'view_count': int(info['views_total']),
'uploader': info['username'],
'uploader_id': info['uid'],
}
+
+class BambuserChannelIE(InfoExtractor):
+ IE_NAME = u'bambuser:channel'
+ _VALID_URL = r'http://bambuser.com/channel/(?P<user>.*?)(?:/|#|\?|$)'
+ # The maximum number we can get with each request
+ _STEP = 50
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ user = mobj.group('user')
+ urls = []
+ last_id = ''
+ for i in itertools.count(1):
+ req_url = ('http://bambuser.com/xhr-api/index.php?username={user}'
+ '&sort=created&access_mode=0%2C1%2C2&limit={count}'
+ '&method=broadcast&format=json&vid_older_than={last}'
+ ).format(user=user, count=self._STEP, last=last_id)
+ req = compat_urllib_request.Request(req_url)
+ # Without setting this header, we wouldn't get any result
+ req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
+ info_json = self._download_webpage(req, user,
+ u'Downloading page %d' % i)
+ results = json.loads(info_json)['result']
+ if len(results) == 0:
+ break
+ last_id = results[-1]['vid']
+ urls.extend(self.url_result(v['page'], 'Bambuser') for v in results)
+
+ return {
+ '_type': 'playlist',
+ 'title': user,
+ 'entries': urls,
+ }
From cf519235455f312ac45e1d9829018eb5ecbec628 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Nov 2013 20:46:26 +0100
Subject: [PATCH 039/132] [youtube] Remove vevo test
The video is no longer available and it seems that vevo video don't use encrypted signatures anymore.
---
youtube_dl/extractor/youtube.py | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc601de52..a19abe1f0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -339,18 +339,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
}
},
- {
- u"url": u"http://www.youtube.com/watch?v=1ltcDfZMA3U",
- u"file": u"1ltcDfZMA3U.mp4",
- u"note": u"Test VEVO video (#897)",
- u"info_dict": {
- u"upload_date": u"20070518",
- u"title": u"Maps - It Will Find You",
- u"description": u"Music video by Maps performing It Will Find You.",
- u"uploader": u"MuteUSA",
- u"uploader_id": u"MuteUSA"
- }
- },
{
u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
u"file": u"UxxajLWwzqY.mp4",
From 98d7efb537975b29ccaea64ff2765a0ec7bdb07d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Nov 2013 20:51:09 +0100
Subject: [PATCH 040/132] [exfm] skip tests
The site is down too often.
---
youtube_dl/extractor/exfm.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
index c74556579..a51d79b08 100644
--- a/youtube_dl/extractor/exfm.py
+++ b/youtube_dl/extractor/exfm.py
@@ -21,6 +21,7 @@ class ExfmIE(InfoExtractor):
u'description': u'Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive',
},
u'note': u'Soundcloud song',
+ u'skip': u'The site is down too often',
},
{
u'url': u'http://ex.fm/song/wddt8',
@@ -30,6 +31,7 @@ class ExfmIE(InfoExtractor):
u'title': u'Safe and Sound',
u'uploader': u'Capital Cities',
},
+ u'skip': u'The site is down too often',
},
]
From f52f01b5d2ed117070475b0c7593a55d417e8e41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Nov 2013 21:20:46 +0100
Subject: [PATCH 041/132] [brightcove] Don't set the extension
If the video only has the 'FLVFullLengthURL' key, it can still be an mp4 file.
---
youtube_dl/extractor/brightcove.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 1392f382a..0d9b87a34 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -23,7 +23,7 @@ class BrightcoveIE(InfoExtractor):
# From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
u'file': u'2371591881001.mp4',
- u'md5': u'9e80619e0a94663f0bdc849b4566af19',
+ u'md5': u'8eccab865181d29ec2958f32a6a754f5',
u'note': u'Test Brightcove downloads and detection in GenericIE',
u'info_dict': {
u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
@@ -122,12 +122,10 @@ class BrightcoveIE(InfoExtractor):
best_format = renditions[-1]
info.update({
'url': best_format['defaultURL'],
- 'ext': 'mp4',
})
elif video_info.get('FLVFullLengthURL') is not None:
info.update({
'url': video_info['FLVFullLengthURL'],
- 'ext': 'flv',
})
else:
raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
From 86ad94bb2ecdbc781f36f0e5fb49c91008e68cc8 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sat, 2 Nov 2013 22:33:49 +0100
Subject: [PATCH 042/132] [ExtremeTubeIE] Set age_limit to 18 and fix uploader
extraction
---
youtube_dl/extractor/extremetube.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 981de430d..0f1eec40f 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -31,15 +31,13 @@ class ExtremeTubeIE(InfoExtractor):
webpage = self._download_webpage(req, video_id)
video_title = self._html_search_regex(r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, u'title')
- uploader = self._html_search_regex(r'>Posted by:(?=<)(\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
+ uploader = self._html_search_regex(r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, u'video_url'))
path = compat_urllib_parse_urlparse( video_url ).path
extension = os.path.splitext( path )[1][1:]
format = path.split('/')[5].split('_')[:2]
format = "-".join( format )
- age_limit = self._rta_search(webpage)
-
return {
'id': video_id,
'title': video_title,
@@ -48,5 +46,5 @@ class ExtremeTubeIE(InfoExtractor):
'ext': extension,
'format': format,
'format_id': format,
- 'age_limit': age_limit,
+ 'age_limit': 18,
}
From 137bbb3e37a41bd49f7c946ae18fb2cd0d1ba144 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sat, 2 Nov 2013 22:45:48 +0100
Subject: [PATCH 043/132] [XTubeIE] Add description to TEST
---
youtube_dl/extractor/xtube.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 7d06a7021..483fb0791 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -16,6 +16,7 @@ class XTubeIE(InfoExtractor):
u'md5': u'092fbdd3cbe292c920ef6fc6a8a9cdab',
u'info_dict': {
u"title": u"strange erotica",
+ u"description": u"surreal gay themed erotica...almost an ET kind of thing",
u"uploader": u"greenshowers",
u"age_limit": 18,
}
From a3dd924871a6fa01d84cadf0a6f60ef622189f09 Mon Sep 17 00:00:00 2001
From: Craig Markwardt <cbmarkwardt+github@gmail.com>
Date: Sat, 2 Nov 2013 22:40:48 -0400
Subject: [PATCH 044/132] Add YoutubeSearchDateIE extractor to youtube.py &
__init__.py, which searches by publication date.
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/youtube.py | 3 +++
2 files changed, 4 insertions(+)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bcf1cce7f..abdee8eb0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -159,6 +159,7 @@ from .youtube import (
YoutubeIE,
YoutubePlaylistIE,
YoutubeSearchIE,
+ YoutubeSearchDateIE,
YoutubeUserIE,
YoutubeChannelIE,
YoutubeShowIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc601de52..6b5ce068d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1743,6 +1743,9 @@ class YoutubeSearchIE(SearchInfoExtractor):
videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
return self.playlist_result(videos, query)
+class YoutubeSearchDateIE(YoutubeSearchIE):
+ _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
+ _SEARCH_KEY = 'ytsearchdate'
class YoutubeShowIE(InfoExtractor):
IE_DESC = u'YouTube.com (multi-season) shows'
From b6c45014aed4b3176be1142958be98d7cb9dbaff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 11:56:45 +0100
Subject: [PATCH 045/132] Set the extra_info inside YoutubeDL.process_ie_result
and set only if the keys are missing
---
test/test_YoutubeDL.py | 12 ++++++++++++
youtube_dl/YoutubeDL.py | 26 +++++++++++++-------------
2 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index ffebb4ae5..58cf9c313 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -128,6 +128,18 @@ class TestFormatSelection(unittest.TestCase):
downloaded = ydl.downloaded_info_dicts[0]
self.assertEqual(downloaded['format_id'], u'35')
+ def test_add_extra_info(self):
+ test_dict = {
+ 'extractor': 'Foo',
+ }
+ extra_info = {
+ 'extractor': 'Bar',
+ 'playlist': 'funny videos',
+ }
+ YDL.add_extra_info(test_dict, extra_info)
+ self.assertEqual(test_dict['extractor'], 'Foo')
+ self.assertEqual(test_dict['playlist'], 'funny videos')
+
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7f73ea360..a3e0a700f 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -318,6 +318,12 @@ class YoutubeDL(object):
% info_dict)
return None
+ @staticmethod
+ def add_extra_info(info_dict, extra_info):
+ '''Set the keys from extra_info in info dict if they are missing'''
+ for key, value in extra_info.items():
+ info_dict.setdefault(key, value)
+
def extract_info(self, url, download=True, ie_key=None, extra_info={}):
'''
Returns a list with a dictionary for each video we find.
@@ -344,17 +350,13 @@ class YoutubeDL(object):
break
if isinstance(ie_result, list):
# Backwards compatibility: old IE result format
- for result in ie_result:
- result.update(extra_info)
ie_result = {
'_type': 'compat_list',
'entries': ie_result,
}
- else:
- ie_result.update(extra_info)
if 'extractor' not in ie_result:
ie_result['extractor'] = ie.IE_NAME
- return self.process_ie_result(ie_result, download=download)
+ return self.process_ie_result(ie_result, download, extra_info)
except ExtractorError as de: # An error we somewhat expected
self.report_error(compat_str(de), de.format_traceback())
break
@@ -378,7 +380,7 @@ class YoutubeDL(object):
result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
if result_type == 'video':
- ie_result.update(extra_info)
+ self.add_extra_info(ie_result, extra_info)
return self.process_video_result(ie_result)
elif result_type == 'url':
# We have to add extra_info to the results because it may be
@@ -388,6 +390,7 @@ class YoutubeDL(object):
ie_key=ie_result.get('ie_key'),
extra_info=extra_info)
elif result_type == 'playlist':
+ self.add_extra_info(ie_result, extra_info)
# We process each entry in the playlist
playlist = ie_result.get('title', None) or ie_result.get('id', None)
self.to_screen(u'[download] Downloading playlist: %s' % playlist)
@@ -413,12 +416,8 @@ class YoutubeDL(object):
extra = {
'playlist': playlist,
'playlist_index': i + playliststart,
+ 'extractor': ie_result['extractor'],
}
- if not 'extractor' in entry:
- # We set the extractor, if it's an url it will be set then to
- # the new extractor, but if it's already a video we must make
- # sure it's present: see issue #877
- entry['extractor'] = ie_result['extractor']
entry_result = self.process_ie_result(entry,
download=download,
extra_info=extra)
@@ -427,10 +426,11 @@ class YoutubeDL(object):
return ie_result
elif result_type == 'compat_list':
def _fixup(r):
- r.setdefault('extractor', ie_result['extractor'])
+ self.add_extra_info(r,
+ {'extractor': ie_result['extractor']})
return r
ie_result['entries'] = [
- self.process_ie_result(_fixup(r), download=download)
+ self.process_ie_result(_fixup(r), download, extra_info)
for r in ie_result['entries']
]
return ie_result
From 9103bbc5cd11957de2e906e4401dcf4df9511d28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 12:11:13 +0100
Subject: [PATCH 046/132] Add the 'webpage_url' field to info_dict
The url for the video page, it must allow to reproduce the result.
It's automatically set by YoutubeDL if it's missing.
---
test/test_download.py | 3 +++
youtube_dl/YoutubeDL.py | 13 ++++++++++---
youtube_dl/extractor/common.py | 3 +++
youtube_dl/extractor/vimeo.py | 13 ++++++-------
youtube_dl/extractor/youtube.py | 3 ++-
5 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/test/test_download.py b/test/test_download.py
index dfb04d010..d6cc9ec33 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -148,6 +148,9 @@ def generator(test_case):
# Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
+ # Check for mandatory fields that are automatically set by YoutubeDL
+ for key in ['webpage_url', 'extractor']:
+ self.assertTrue(info_dict.get(key), u'Missing field: %s' % key)
finally:
try_rm_tcs_files()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a3e0a700f..8938a2cd3 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -354,8 +354,11 @@ class YoutubeDL(object):
'_type': 'compat_list',
'entries': ie_result,
}
- if 'extractor' not in ie_result:
- ie_result['extractor'] = ie.IE_NAME
+ self.add_extra_info(ie_result,
+ {
+ 'extractor': ie.IE_NAME,
+ 'webpage_url': url
+ })
return self.process_ie_result(ie_result, download, extra_info)
except ExtractorError as de: # An error we somewhat expected
self.report_error(compat_str(de), de.format_traceback())
@@ -417,6 +420,7 @@ class YoutubeDL(object):
'playlist': playlist,
'playlist_index': i + playliststart,
'extractor': ie_result['extractor'],
+ 'webpage_url': ie_result['webpage_url'],
}
entry_result = self.process_ie_result(entry,
download=download,
@@ -427,7 +431,10 @@ class YoutubeDL(object):
elif result_type == 'compat_list':
def _fixup(r):
self.add_extra_info(r,
- {'extractor': ie_result['extractor']})
+ {
+ 'extractor': ie_result['extractor'],
+ 'webpage_url': ie_result['webpage_url'],
+ })
return r
ie_result['entries'] = [
self.process_ie_result(_fixup(r), download, extra_info)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cef4dce85..e0ccba533 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -71,6 +71,9 @@ class InfoExtractor(object):
("3D" or "DASH video")
* width Width of the video, if known
* height Height of the video, if known
+ webpage_url: The url to the video webpage, if given to youtube-dl it
+ should allow to get the same result again. (It will be set
+ by YoutubeDL if it's missing)
Unless mentioned otherwise, the fields should be Unicode strings.
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c7d864a2b..62273fd33 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -20,7 +20,7 @@ class VimeoIE(InfoExtractor):
"""Information extractor for vimeo.com."""
# _VALID_URL matches Vimeo URLs
- _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)/?(?:[?].*)?(?:#.*)?$'
+ _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|(?P<player>player))\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)/?(?:[?].*)?(?:#.*)?$'
_NETRC_MACHINE = 'vimeo'
IE_NAME = u'vimeo'
_TESTS = [
@@ -128,11 +128,9 @@ class VimeoIE(InfoExtractor):
raise ExtractorError(u'Invalid URL: %s' % url)
video_id = mobj.group('id')
- if not mobj.group('proto'):
- url = 'https://' + url
- elif mobj.group('pro'):
+ if mobj.group('pro') or mobj.group('player'):
url = 'http://player.vimeo.com/video/' + video_id
- elif mobj.group('direct_link'):
+ else:
url = 'https://vimeo.com/' + video_id
# Retrieve video webpage to extract further information
@@ -234,7 +232,7 @@ class VimeoIE(InfoExtractor):
if len(formats) == 0:
raise ExtractorError(u'No known codec found')
- return [{
+ return {
'id': video_id,
'uploader': video_uploader,
'uploader_id': video_uploader_id,
@@ -243,7 +241,8 @@ class VimeoIE(InfoExtractor):
'thumbnail': video_thumbnail,
'description': video_description,
'formats': formats,
- }]
+ 'webpage_url': url,
+ }
class VimeoChannelIE(InfoExtractor):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a19abe1f0..6ddd6ef06 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1485,7 +1485,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
'subtitles': video_subtitles,
'duration': video_duration,
'age_limit': 18 if age_gate else 0,
- 'annotations': video_annotations
+ 'annotations': video_annotations,
+ 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id,
})
return results
From be97abc247d26bc36d1ef8cad5c17fc2a99d9101 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 12:14:44 +0100
Subject: [PATCH 047/132] Set the 'extractor_key' field in the info_dict
It's the string returned by the class method 'ie_key', which allows to retrieve the extractor with 'get_info_extractor'
---
test/test_download.py | 2 +-
youtube_dl/YoutubeDL.py | 5 ++++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/test/test_download.py b/test/test_download.py
index d6cc9ec33..73379beb1 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -149,7 +149,7 @@ def generator(test_case):
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
# Check for mandatory fields that are automatically set by YoutubeDL
- for key in ['webpage_url', 'extractor']:
+ for key in ['webpage_url', 'extractor', 'extractor_key']:
self.assertTrue(info_dict.get(key), u'Missing field: %s' % key)
finally:
try_rm_tcs_files()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8938a2cd3..86a6fd043 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -357,7 +357,8 @@ class YoutubeDL(object):
self.add_extra_info(ie_result,
{
'extractor': ie.IE_NAME,
- 'webpage_url': url
+ 'webpage_url': url,
+ 'extractor_key': ie.ie_key(),
})
return self.process_ie_result(ie_result, download, extra_info)
except ExtractorError as de: # An error we somewhat expected
@@ -421,6 +422,7 @@ class YoutubeDL(object):
'playlist_index': i + playliststart,
'extractor': ie_result['extractor'],
'webpage_url': ie_result['webpage_url'],
+ 'extractor_key': ie_result['extractor_key'],
}
entry_result = self.process_ie_result(entry,
download=download,
@@ -434,6 +436,7 @@ class YoutubeDL(object):
{
'extractor': ie_result['extractor'],
'webpage_url': ie_result['webpage_url'],
+ 'extractor_key': ie_result['extractor_key'],
})
return r
ie_result['entries'] = [
From a56f9de156c7cca29dfa45de1dadc66e10a265f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 14:03:17 +0100
Subject: [PATCH 048/132] Style fixes for extractors: remove spaces around
(,),{ and }
---
youtube_dl/extractor/depositfiles.py | 2 +-
youtube_dl/extractor/extremetube.py | 6 +++---
youtube_dl/extractor/hypem.py | 4 ++--
youtube_dl/extractor/keezmovies.py | 6 +++---
youtube_dl/extractor/mofosex.py | 6 +++---
youtube_dl/extractor/pornhub.py | 6 +++---
youtube_dl/extractor/spankwire.py | 6 +++---
youtube_dl/extractor/tube8.py | 6 +++---
youtube_dl/extractor/vimeo.py | 2 +-
youtube_dl/extractor/xtube.py | 6 +++---
youtube_dl/extractor/yahoo.py | 2 +-
youtube_dl/extractor/youku.py | 6 +++---
youtube_dl/extractor/youporn.py | 8 ++++----
13 files changed, 33 insertions(+), 33 deletions(-)
diff --git a/youtube_dl/extractor/depositfiles.py b/youtube_dl/extractor/depositfiles.py
index d43348955..2c9fb5f2e 100644
--- a/youtube_dl/extractor/depositfiles.py
+++ b/youtube_dl/extractor/depositfiles.py
@@ -25,7 +25,7 @@ class DepositFilesIE(InfoExtractor):
url = 'http://depositfiles.com/en/files/' + file_id
# Retrieve file webpage with 'Free download' button pressed
- free_download_indication = { 'gateway_result' : '1' }
+ free_download_indication = {'gateway_result' : '1'}
request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
try:
self.report_download_webpage(file_id)
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 0f1eec40f..1c20e4364 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -33,10 +33,10 @@ class ExtremeTubeIE(InfoExtractor):
video_title = self._html_search_regex(r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, u'title')
uploader = self._html_search_regex(r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, u'video_url'))
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2]
- format = "-".join( format )
+ format = "-".join(format)
return {
'id': video_id,
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index ab2b59103..9bd06e7c7 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -30,7 +30,7 @@ class HypemIE(InfoExtractor):
raise ExtractorError(u'Invalid URL: %s' % url)
track_id = mobj.group(1)
- data = { 'ax': 1, 'ts': time.time() }
+ data = {'ax': 1, 'ts': time.time()}
data_encoded = compat_urllib_parse.urlencode(data)
complete_url = url + "?" + data_encoded
request = compat_urllib_request.Request(complete_url)
@@ -68,4 +68,4 @@ class HypemIE(InfoExtractor):
'ext': "mp3",
'title': title,
'artist': artist,
- }]
\ No newline at end of file
+ }]
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 786924445..29658a7d6 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -43,10 +43,10 @@ class KeezMoviesIE(InfoExtractor):
if webpage.find('encrypted=true')!=-1:
password = self._html_search_regex(r'video_title=(.+?)&', webpage, u'password')
video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[4].split('_')[:2]
- format = "-".join( format )
+ format = "-".join(format)
age_limit = self._rta_search(webpage)
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index a0c926cd1..b9430b09b 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -31,10 +31,10 @@ class MofosexIE(InfoExtractor):
video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, u'title')
video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, u'video_url'))
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2]
- format = "-".join( format )
+ format = "-".join(format)
age_limit = self._rta_search(webpage)
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 5e2454f1b..75cf4bb9f 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -47,10 +47,10 @@ class PornHubIE(InfoExtractor):
formats = []
for video_url in video_urls:
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2]
- format = "-".join( format )
+ format = "-".join(format)
formats.append({
'url': video_url,
'ext': extension,
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 32df0a7fb..97f9c268a 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -49,10 +49,10 @@ class SpankwireIE(InfoExtractor):
formats = []
for video_url in video_urls:
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[4].split('_')[:2]
- format = "-".join( format )
+ format = "-".join(format)
formats.append({
'url': video_url,
'ext': extension,
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index aea9d9a24..d4b7603c7 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -46,10 +46,10 @@ class Tube8IE(InfoExtractor):
if webpage.find('"encrypted":true')!=-1:
password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, u'password')
video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[4].split('_')[:2]
- format = "-".join( format )
+ format = "-".join(format)
return {
'id': video_id,
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 62273fd33..d465bf20b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -203,7 +203,7 @@ class VimeoIE(InfoExtractor):
# Vimeo specific: extract video codec and quality information
# First consider quality, then codecs, then take everything
codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
- files = { 'hd': [], 'sd': [], 'other': []}
+ files = {'hd': [], 'sd': [], 'other': []}
config_files = config["video"].get("files") or config["request"].get("files")
for codec_name, codec_extension in codecs:
for quality in config_files.get(codec_name, []):
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 483fb0791..03ad88bed 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -35,12 +35,12 @@ class XTubeIE(InfoExtractor):
video_uploader = self._html_search_regex(r'so_s\.addVariable\("owner_u", "([^"]+)', webpage, u'uploader', fatal=False)
video_description = self._html_search_regex(r'<p class="video_description">([^<]+)', webpage, u'description', default=None)
video_url= self._html_search_regex(r'var videoMp4 = "([^"]+)', webpage, u'video_url').replace('\\/', '/')
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2]
format[0] += 'p'
format[1] += 'k'
- format = "-".join( format )
+ format = "-".join(format)
return {
'id': video_id,
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 464b498f5..34e6afb20 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -132,7 +132,7 @@ class YahooSearchIE(SearchInfoExtractor):
mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
res['entries'].append(e)
- if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1 )):
+ if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1)):
break
return res
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 9d88c17f5..a8fd40c83 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -18,7 +18,7 @@ class YoukuIE(InfoExtractor):
u"url": u"http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
u"file": u"XNDgyMDQ2NTQw_part00.flv",
u"md5": u"ffe3f2e435663dc2d1eea34faeff5b5b",
- u"params": { u"test": False },
+ u"params": {u"test": False},
u"info_dict": {
u"title": u"youtube-dl test video \"'/\\ä↭𝕐"
}
@@ -37,8 +37,8 @@ class YoukuIE(InfoExtractor):
source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
seed = float(seed)
for i in range(len(source)):
- seed = (seed * 211 + 30031 ) % 65536
- index = math.floor(seed / 65536 * len(source) )
+ seed = (seed * 211 + 30031) % 65536
+ index = math.floor(seed / 65536 * len(source))
mixed.append(source[int(index)])
source.remove(source[int(index)])
#return ''.join(mixed)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index e46a9b4d6..bd0f2cae0 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -81,14 +81,14 @@ class YouPornIE(InfoExtractor):
# http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
# A path looks like this:
# /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
- video_url = unescapeHTML( link )
- path = compat_urllib_parse_urlparse( video_url ).path
- extension = os.path.splitext( path )[1][1:]
+ video_url = unescapeHTML(link)
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
format = path.split('/')[4].split('_')[:2]
# size = format[0]
# bitrate = format[1]
- format = "-".join( format )
+ format = "-".join(format)
# title = u'%s-%s-%s' % (video_title, size, bitrate)
formats.append({
From 12ebdd150624adc20a841f5fb174676b123ef826 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 3 Nov 2013 15:49:59 +0100
Subject: [PATCH 049/132] [viddler] Support non-digit IDs (Fixes #1714)
---
youtube_dl/extractor/viddler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 12c84a985..826804af3 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -8,7 +8,7 @@ from ..utils import (
class ViddlerIE(InfoExtractor):
- _VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler.com)/(?:v|embed|player)/(?P<id>[0-9]+)'
+ _VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler.com)/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
_TEST = {
u"url": u"http://www.viddler.com/v/43903784",
u'file': u'43903784.mp4',
From 165e179764e7d276d5e6ed79a8e63b63852cdd3e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 3 Nov 2013 15:50:36 +0100
Subject: [PATCH 050/132] release 2013.11.03
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 75a46a2d5..cc0f9cb4e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.02'
+__version__ = '2013.11.03'
From 08fb86c49b92b53df8963065ab23fd558b8a90d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 3 Nov 2013 15:58:52 +0100
Subject: [PATCH 051/132] [youtube] Add description for YoutubeSearchDateIE
(#1710)
---
youtube_dl/extractor/youtube.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 14e8f59e6..74a381fe2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1735,6 +1735,7 @@ class YoutubeSearchIE(SearchInfoExtractor):
class YoutubeSearchDateIE(YoutubeSearchIE):
_API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
_SEARCH_KEY = 'ytsearchdate'
+ IE_DESC = u'YouTube.com searches, newest videos first'
class YoutubeShowIE(InfoExtractor):
IE_DESC = u'YouTube.com (multi-season) shows'
From e7e6b54d8a02f7f76bd56e35bcd87a8723f71642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 17:48:12 +0100
Subject: [PATCH 052/132] [teamcoco] Parse the xml file and extract all the
formats
---
youtube_dl/extractor/teamcoco.py | 47 ++++++++++++++++++++------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 76246c7cc..bc48620f0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -1,9 +1,9 @@
import re
+import xml.etree.ElementTree
from .common import InfoExtractor
from ..utils import (
ExtractorError,
- RegexNotFoundError,
)
@@ -32,29 +32,40 @@ class TeamcocoIE(InfoExtractor):
self.report_extraction(video_id)
data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
- data = self._download_webpage(data_url, video_id, 'Downloading data webpage')
+ data_xml = self._download_webpage(data_url, video_id, 'Downloading data webpage')
+ data = xml.etree.ElementTree.fromstring(data_xml.encode('utf-8'))
- qualities = [ '1080p', '720p', '1000k', '480p', '500k' ]
- best_quality_idx = len(qualities)+1 # First regex match may not be optimal
- for idx, quality in enumerate(qualities):
- regex = r'<file [^>]*type="(?:high|standard)".*?>(.*%s.*)</file>' % quality
+ qualities = ['500k', '480p', '1000k', '720p', '1080p']
+ formats = []
+ for file in data.findall('files/file'):
+ if file.attrib.get('playmode') == 'all':
+ # it just duplicates one of the entries
+ break
+ file_url = file.text
+ m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
+ if m_format is not None:
+ format_id = m_format.group(1)
+ else:
+ format_id = file.attrib['bitrate']
+ formats.append({
+ 'url': file_url,
+ 'ext': 'mp4',
+ 'format_id': format_id,
+ })
+ def sort_key(f):
try:
- url = self._html_search_regex(regex, data, u'video URL')
- if idx < best_quality_idx:
- video_url = url
- best_quality_idx = idx
- except RegexNotFoundError:
- # Just catch fatal exc. Don't want the fatal=False warning
- continue
- if not video_url:
+ return qualities.index(f['format_id'])
+ except ValueError:
+ return -1
+ formats.sort(key=sort_key)
+ if not formats:
raise RegexNotFoundError(u'Unable to extract video URL')
- return [{
+ return {
'id': video_id,
- 'url': video_url,
- 'ext': 'mp4',
+ 'formats': formats,
'title': self._og_search_title(webpage),
'thumbnail': self._og_search_thumbnail(webpage),
'description': self._og_search_description(webpage),
- }]
+ }
From 19b0668251583d5c7dbc55696746539979b2ba11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 Nov 2013 22:20:22 +0100
Subject: [PATCH 053/132] [canal2c] Accept more urls (fixes #1723)
The url only needs to have the 'idVideo' field in the query, in any position.
We have to set the 'void=oui' in the webpage url, so that we get the file name.
---
youtube_dl/extractor/canalc2.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index e7f4fa9fd..3d8d7f9d2 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
class Canalc2IE(InfoExtractor):
IE_NAME = 'canalc2.tv'
- _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?idVideo=(\d+)&voir=oui'
+ _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?.*?idVideo=(?P<id>\d+)'
_TEST = {
u'url': u'http://www.canalc2.tv/video.asp?idVideo=12163&voir=oui',
@@ -18,7 +18,9 @@ class Canalc2IE(InfoExtractor):
}
def _real_extract(self, url):
- video_id = re.match(self._VALID_URL, url).group(1)
+ video_id = re.match(self._VALID_URL, url).group('id')
+ # We need to set the voir field for getting the file name
+ url = 'http://www.canalc2.tv/video.asp?idVideo=%s&voir=oui' % video_id
webpage = self._download_webpage(url, video_id)
file_name = self._search_regex(
r"so\.addVariable\('file','(.*?)'\);",
From 2dcf7d8f99604de9bdfa1b1e12a2bd5840d523d2 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Tue, 5 Nov 2013 02:08:02 +0100
Subject: [PATCH 054/132] [GenericIE] Also detect youtube if src url of iframe
is embedded in ' instaed of "
---
youtube_dl/extractor/generic.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2c8fcf5ae..b3fec8e86 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -160,9 +160,9 @@ class GenericIE(InfoExtractor):
# Look for embedded YouTube player
mobj = re.search(
- r'<iframe[^>]+?src="(https?://(?:www\.)?youtube.com/embed/.+?)"', webpage)
+ r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?youtube.com/embed/.+?)\1', webpage)
if mobj:
- surl = unescapeHTML(mobj.group(1))
+ surl = unescapeHTML(mobj.group(u'url'))
return self.url_result(surl, 'Youtube')
# Look for Bandcamp pages with custom domain
From 4ed3e51080dea651b3db8889b0c8f2e40d261075 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Tue, 5 Nov 2013 12:00:13 +0100
Subject: [PATCH 055/132] [ted] fixed error in case of no subtitles present
I created a test, but I leave it commented since TED videos get
new subtitles frequently.
---
test/test_ted_subtitles.py | 6 ++++++
youtube_dl/extractor/ted.py | 24 ++++++++++++++++--------
2 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
index 3283253ab..a0dd7eeed 100644
--- a/test/test_ted_subtitles.py
+++ b/test/test_ted_subtitles.py
@@ -51,6 +51,12 @@ class TestTedSubtitles(unittest.TestCase):
self.DL.params['subtitleslang'] = ['en']
subtitles = self.getSubtitles()
self.assertTrue(len(subtitles.keys()) == 0)
+ # def test_nosubtitles(self):
+ # self.DL.expect_warning(u'video doesn\'t have subtitles')
+ # self.url = 'http://www.ted.com/talks/rodrigo_canales_the_deadly_genius_of_drug_cartels.html'
+ # self.DL.params['writesubtitles'] = True
+ # self.DL.params['allsubtitles'] = True
+ # subtitles = self.getSubtitles()
def test_multiple_langs(self):
self.DL.params['writesubtitles'] = True
langs = ['es', 'fr', 'de']
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 239e2a448..1b006bc9b 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -3,6 +3,11 @@ import re
from .subtitles import SubtitlesInfoExtractor
+from ..utils import (
+ compat_str,
+ RegexNotFoundError,
+)
+
class TEDIE(SubtitlesInfoExtractor):
_VALID_URL=r'''http://www\.ted\.com/
(
@@ -105,12 +110,15 @@ class TEDIE(SubtitlesInfoExtractor):
return info
def _get_available_subtitles(self, video_id, webpage):
- options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
- languages = re.findall(r'(?:<option value=")(\S+)"', options)
- if languages:
- sub_lang_list = {}
- for l in languages:
- url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
- sub_lang_list[l] = url
- return sub_lang_list
+ try:
+ options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
+ languages = re.findall(r'(?:<option value=")(\S+)"', options)
+ if languages:
+ sub_lang_list = {}
+ for l in languages:
+ url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
+ sub_lang_list[l] = url
+ return sub_lang_list
+ except RegexNotFoundError as err:
+ self._downloader.report_warning(u'video doesn\'t have subtitles')
return {}
From a8eeb0597b11dbc9d1b48f95264cc2815311aa15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= <saper@saper.info>
Date: Tue, 5 Nov 2013 23:19:29 +0100
Subject: [PATCH 056/132] Fix AssertionError when og property not found
On tvp.pl some webpages contain OpenGraph
metadata and some don't.
If og property is not found, _og_search_description
fails with
WARNING: unable to extract OpenGraph description; please report this issue on http://yt-dl.org/bug
Traceback (most recent call last):
File "/usr/home/saper/bin/youtube-dl", line 18, in <module>
youtube_dl.main()
File "/usr/home/saper/sw/youtube-dl/youtube_dl/__init__.py", line 766, in main
_real_main(argv)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/__init__.py", line 719, in _real_main
retcode = ydl.download(all_urls)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/YoutubeDL.py", line 715, in download
videos = self.extract_info(url)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/YoutubeDL.py", line 348, in extract_info
ie_result = ie.extract(url)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/common.py", line 125, in extract
return self._real_extract(url)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/tvp.py", line 56, in _real_extract
info['description'] = self._og_search_description(webpage)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/common.py", line 331, in _og_search_description
return self._og_search_property('description', html, fatal=False, **kargs)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/common.py", line 325, in _og_search_property
return unescapeHTML(escaped)
File "/usr/home/saper/sw/youtube-dl/youtube_dl/utils.py", line 494, in unescapeHTML
assert type(s) == type(u'')
AssertionError
The patch allows me to use:
try:
info['description'] = self._og_search_description(webpage)
info['thumbnail'] = self._og_search_thumbnail(webpage)
except RegexNotFoundError:
pass
---
youtube_dl/extractor/common.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e0ccba533..fb2d50a09 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -322,7 +322,9 @@ class InfoExtractor(object):
if name is None:
name = 'OpenGraph %s' % prop
escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
- return unescapeHTML(escaped)
+ if not escaped is None:
+ return unescapeHTML(escaped)
+ return None
def _og_search_thumbnail(self, html, **kargs):
return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
From 5137ebac0b1438d22fe2c007e6172ee65e9311a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= <saper@saper.info>
Date: Tue, 5 Nov 2013 23:30:25 +0100
Subject: [PATCH 057/132] [tvp] Telewizja Polska: new extractor for tvp.pl,
fixes #1719
Thanks-To: mplonski
https://github.com/mplonski/linux/blob/master/tvp-dl.py
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/tvp.py | 60 ++++++++++++++++++++++++++++++++
2 files changed, 61 insertions(+)
create mode 100644 youtube_dl/extractor/tvp.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 888a91cce..78f84cea3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -132,6 +132,7 @@ from .tube8 import Tube8IE
from .tudou import TudouIE
from .tumblr import TumblrIE
from .tutv import TutvIE
+from .tvp import TvpIE
from .unistra import UnistraIE
from .ustream import UstreamIE, UstreamChannelIE
from .vbox7 import Vbox7IE
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
new file mode 100644
index 000000000..63fb57bbe
--- /dev/null
+++ b/youtube_dl/extractor/tvp.py
@@ -0,0 +1,60 @@
+# encoding: utf-8
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ determine_ext,
+ ExtractorError,
+ RegexNotFoundError,
+)
+
+class TvpIE(InfoExtractor):
+ IE_NAME = u'tvp.pl'
+ _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
+ _INFO_URL = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s'
+
+
+ _TEST = {
+ u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
+ u'file': u'31.10.2013-12878238.wmv',
+ u'info_dict': {
+ u'title': u'31.10.2013',
+ u'description': u'31.10.2013',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id, "Downloading video webpage")
+ json_params = self._download_webpage(self._INFO_URL % video_id, video_id, "Downloading video metadata")
+
+ try:
+ params = json.loads(json_params)
+ except:
+ raise ExtractorError(u'Invalid JSON')
+
+ self.report_extraction(video_id)
+ try:
+ video_url = params['video_url']
+ except KeyError:
+ raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+
+ try:
+ title = self._og_search_title(webpage)
+ except RegexNotFoundError:
+ title = video_id
+ info = {
+ 'id': video_id,
+ 'title': title,
+ 'ext': 'wmv',
+ 'url': video_url,
+ }
+ try:
+ info['description'] = self._og_search_description(webpage)
+ info['thumbnail'] = self._og_search_thumbnail(webpage)
+ except RegexNotFoundError:
+ pass
+
+ return info
From 76e67c2cb6bd041a84756ec291edf4028171fdfe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 14:01:43 +0100
Subject: [PATCH 058/132] Clean up imports
---
youtube_dl/FileDownloader.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 8ecabab1a..47f678811 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -4,12 +4,19 @@ import re
import subprocess
import sys
import time
-import traceback
if os.name == 'nt':
import ctypes
-from .utils import *
+from .utils import (
+ compat_urllib_error,
+ compat_urllib_request,
+ ContentTooShortError,
+ determine_ext,
+ encodeFilename,
+ sanitize_open,
+ timeconvert,
+)
class FileDownloader(object):
@@ -194,7 +201,7 @@ class FileDownloader(object):
if old_filename == new_filename:
return
os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
- except (IOError, OSError) as err:
+ except (IOError, OSError):
self.report_error(u'unable to rename file')
def try_utime(self, filename, last_modified_hdr):
@@ -251,7 +258,7 @@ class FileDownloader(object):
"""Report file has already been fully downloaded."""
try:
self.to_screen(u'[download] %s has already been downloaded' % file_name)
- except (UnicodeEncodeError) as err:
+ except UnicodeEncodeError:
self.to_screen(u'[download] The file has already been downloaded')
def report_unable_to_resume(self):
From 50a886b7abdcce61986a2b589ca2366cc0984a58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 14:02:33 +0100
Subject: [PATCH 059/132] Fix reporting when file size is unkown (Fixes #1731)
---
youtube_dl/FileDownloader.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 47f678811..b0f26fc6f 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -557,7 +557,7 @@ class FileDownloader(object):
# Progress message
speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
if data_len is None:
- self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+ self.report_progress('Unknown %', data_len_str, speed, 'Unknown ETA')
eta = None
else:
percent = self.calc_percent(byte_counter, data_len)
From da54be877a59bfd040570168743501d7fd618278 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 14:02:52 +0100
Subject: [PATCH 060/132] release 2013.11.06
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cc0f9cb4e..57b4d419c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.03'
+__version__ = '2013.11.06'
From 9ee2b5f6f2a6e07ab901e47ede59eb5382ebb6ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 19:14:53 +0100
Subject: [PATCH 061/132] tests: don't run the test if any of the extractors
listed in the 'add_ie' field is marked as not working
---
test/test_download.py | 8 +++++++-
youtube_dl/extractor/generic.py | 2 ++
youtube_dl/extractor/mtv.py | 1 +
youtube_dl/extractor/slashdot.py | 1 +
youtube_dl/extractor/weibo.py | 1 +
5 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/test/test_download.py b/test/test_download.py
index 73379beb1..16f200809 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -31,6 +31,7 @@ from youtube_dl.utils import (
ExtractorError,
UnavailableVideoError,
)
+from youtube_dl.extractor import get_info_extractor
RETRIES = 3
@@ -63,9 +64,10 @@ def generator(test_case):
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
+ other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
- if not ie._WORKING:
+ if not ie.working():
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case:
@@ -77,6 +79,10 @@ def generator(test_case):
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
+ for other_ie in other_ies:
+ if not other_ie.working():
+ print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
+ return
params = get_params(test_case.get('params', {}))
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b3fec8e86..76d369273 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -33,6 +33,7 @@ class GenericIE(InfoExtractor):
},
# embedded vimeo video
{
+ u'add_ie': ['Vimeo'],
u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
u'file': u'22444065.mp4',
u'md5': u'2903896e23df39722c33f015af0666e2',
@@ -44,6 +45,7 @@ class GenericIE(InfoExtractor):
},
# bandcamp page with custom domain
{
+ u'add_ie': ['Bandcamp'],
u'url': u'http://bronyrock.com/track/the-pony-mash',
u'file': u'3235767654.mp3',
u'info_dict': {
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index e96d3952c..24a79ae13 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -26,6 +26,7 @@ class MTVIE(InfoExtractor):
},
},
{
+ u'add_ie': ['Vevo'],
u'url': u'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
u'file': u'USCJY1331283.mp4',
u'md5': u'73b4e7fcadd88929292fe52c3ced8caf',
diff --git a/youtube_dl/extractor/slashdot.py b/youtube_dl/extractor/slashdot.py
index 2cba53076..f5003c7f9 100644
--- a/youtube_dl/extractor/slashdot.py
+++ b/youtube_dl/extractor/slashdot.py
@@ -7,6 +7,7 @@ class SlashdotIE(InfoExtractor):
_VALID_URL = r'https?://tv.slashdot.org/video/\?embed=(?P<id>.*?)(&|$)'
_TEST = {
+ u'add_ie': ['Ooyala'],
u'url': u'http://tv.slashdot.org/video/?embed=JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz',
u'file': u'JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz.mp4',
u'md5': u'd2222e7a4a4c1541b3e0cf732fb26735',
diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 0757495bd..fa784ab99 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -13,6 +13,7 @@ class WeiboIE(InfoExtractor):
_VALID_URL = r'https?://video\.weibo\.com/v/weishipin/t_(?P<id>.+?)\.htm'
_TEST = {
+ u'add_ie': ['Sina'],
u'url': u'http://video.weibo.com/v/weishipin/t_zjUw2kZ.htm',
u'file': u'98322879.flv',
u'info_dict': {
From eeb165e674e07aaa798f69e15f16faa01bc8feaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 16:40:24 +0100
Subject: [PATCH 062/132] [brightcove] Add the extraction of the url from
generic
---
youtube_dl/extractor/brightcove.py | 16 +++++++++++++++-
youtube_dl/extractor/generic.py | 20 +++++++++++++++++---
2 files changed, 32 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0d9b87a34..b3c3dc0fd 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -9,6 +9,7 @@ from ..utils import (
compat_urllib_parse,
find_xpath_attr,
compat_urlparse,
+ compat_str,
ExtractorError,
)
@@ -71,6 +72,19 @@ class BrightcoveIE(InfoExtractor):
data = compat_urllib_parse.urlencode(params)
return cls._FEDERATED_URL_TEMPLATE % data
+ @classmethod
+ def _extract_brightcove_url(cls, webpage):
+ """Try to extract the brightcove url from the wepbage, returns None
+ if it can't be found
+ """
+ m_brightcove = re.search(
+ r'<object[^>]+?class=([\'"])[^>]*?BrightcoveExperience.*?\1.+?</object>',
+ webpage, re.DOTALL)
+ if m_brightcove is not None:
+ return cls._build_brighcove_url(m_brightcove.group())
+ else:
+ return None
+
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
query_str = mobj.group('query')
@@ -109,7 +123,7 @@ class BrightcoveIE(InfoExtractor):
def _extract_video_info(self, video_info):
info = {
- 'id': video_info['id'],
+ 'id': compat_str(video_info['id']),
'title': video_info['displayName'],
'description': video_info.get('shortDescription'),
'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 76d369273..04b7212f4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -54,6 +54,21 @@ class GenericIE(InfoExtractor):
},
u'skip': u'There is a limit of 200 free downloads / month for the test song',
},
+ # embedded brightcove video
+ {
+ u'add_ie': ['Brightcove'],
+ u'url': u'http://www.scientificamerican.com/article.cfm?id=soap-bubble-physics',
+ u'info_dict': {
+ u'id': u'2365799484001',
+ u'ext': u'mp4',
+ u'title': u'Bubble Simulation',
+ u'description': u'A visualization from a new computer model of foam behavior.',
+ u'uploader': u'Scientific American',
+ },
+ u'params': {
+ u'skip_download': True,
+ },
+ },
]
def report_download_webpage(self, video_id):
@@ -146,10 +161,9 @@ class GenericIE(InfoExtractor):
self.report_extraction(video_id)
# Look for BrightCove:
- m_brightcove = re.search(r'<object[^>]+?class=([\'"])[^>]*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
- if m_brightcove is not None:
+ bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+ if bc_url is not None:
self.to_screen(u'Brightcove video detected.')
- bc_url = BrightcoveIE._build_brighcove_url(m_brightcove.group())
return self.url_result(bc_url, 'Brightcove')
# Look for embedded Vimeo player
From fc4a0c2aec42035281771c74575d001f7e8adcd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 17:25:38 +0100
Subject: [PATCH 063/132] [brightcove] Change the 'videoId' or 'videoID' field
to '@videoPlayer' (fixes #1697)
It seems to be needed when using the htmlFederated page
---
youtube_dl/extractor/brightcove.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b3c3dc0fd..0c6e13b9c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -42,6 +42,17 @@ class BrightcoveIE(InfoExtractor):
u'uploader': u'Oracle',
},
},
+ {
+ # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
+ u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
+ u'info_dict': {
+ u'id': u'2750934548001',
+ u'ext': u'mp4',
+ u'title': u'This Bracelet Acts as a Personal Thermostat',
+ u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0',
+ u'uploader': u'Mashable',
+ },
+ },
]
@classmethod
@@ -86,6 +97,8 @@ class BrightcoveIE(InfoExtractor):
return None
def _real_extract(self, url):
+ # Change the 'videoId' or 'videoID' field to '@videoPlayer'
+ url = re.sub(r'(?<=[?&])videoI(d|D)', '%40videoPlayer', url)
mobj = re.match(self._VALID_URL, url)
query_str = mobj.group('query')
query = compat_urlparse.parse_qs(query_str)
From 065472936a81fcad263dc2f1b04fdfe4a221eeb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 17:37:39 +0100
Subject: [PATCH 064/132] Add an extractor for space.com (fixes #1718)
It uses Brightcove, but requires some special process for getting a url with the playerKey field in some videos
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/space.py | 35 ++++++++++++++++++++++++++++++++
2 files changed, 36 insertions(+)
create mode 100644 youtube_dl/extractor/space.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 888a91cce..d8626fb39 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -115,6 +115,7 @@ from .slideshare import SlideshareIE
from .sohu import SohuIE
from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE
from .southparkstudios import SouthParkStudiosIE
+from .space import SpaceIE
from .spankwire import SpankwireIE
from .spiegel import SpiegelIE
from .stanfordoc import StanfordOpenClassroomIE
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
new file mode 100644
index 000000000..0d32a0688
--- /dev/null
+++ b/youtube_dl/extractor/space.py
@@ -0,0 +1,35 @@
+import re
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+from ..utils import RegexNotFoundError, ExtractorError
+
+
+class SpaceIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.space\.com/\d+-(?P<title>[^/\.\?]*?)-video.html'
+ _TEST = {
+ u'add_ie': ['Brightcove'],
+ u'url': u'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
+ u'info_dict': {
+ u'id': u'2780937028001',
+ u'ext': u'mp4',
+ u'title': u'Huge Martian Landforms\' Detail Revealed By European Probe | Video',
+ u'description': u'md5:db81cf7f3122f95ed234b631a6ea1e61',
+ u'uploader': u'TechMedia Networks',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ title = mobj.group('title')
+ webpage = self._download_webpage(url, title)
+ try:
+ # Some videos require the playerKey field, which isn't define in
+ # the BrightcoveExperience object
+ brightcove_url = self._og_search_video_url(webpage)
+ except RegexNotFoundError:
+ # Other videos works fine with the info from the object
+ brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
+ if brightcove_url is None:
+ raise ExtractorError(u'The webpage does not contain a video', expected=True)
+ return self.url_result(brightcove_url, BrightcoveIE.ie_key())
From b0759f0c19fe223406bf3c7e59222183920eb714 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 19:05:41 +0100
Subject: [PATCH 065/132] [brightcove] Extract all the available formats
---
youtube_dl/extractor/brightcove.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0c6e13b9c..f0b79898c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -146,10 +146,11 @@ class BrightcoveIE(InfoExtractor):
renditions = video_info.get('renditions')
if renditions:
renditions = sorted(renditions, key=lambda r: r['size'])
- best_format = renditions[-1]
- info.update({
- 'url': best_format['defaultURL'],
- })
+ info['formats'] = [{
+ 'url': rend['defaultURL'],
+ 'height': rend.get('frameHeight'),
+ 'width': rend.get('frameWidth'),
+ } for rend in renditions]
elif video_info.get('FLVFullLengthURL') is not None:
info.update({
'url': video_info['FLVFullLengthURL'],
From 5d7b253ea071e4bf16e99242a84f1b0eab2d6f32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 20:05:28 +0100
Subject: [PATCH 066/132] Add an extractor for eitb.tv (fixes #1608)
The BrighcoveExperience object doesn't contain the video id, the extractor adds it and passes the url to BrightcoveIE.
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/eitb.py | 37 ++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+)
create mode 100644 youtube_dl/extractor/eitb.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d8626fb39..f9caca4ef 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -38,6 +38,7 @@ from .defense import DefenseGouvFrIE
from .ebaumsworld import EbaumsWorldIE
from .ehow import EHowIE
from .eighttracks import EightTracksIE
+from .eitb import EitbIE
from .escapist import EscapistIE
from .exfm import ExfmIE
from .extremetube import ExtremeTubeIE
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
new file mode 100644
index 000000000..022d90abc
--- /dev/null
+++ b/youtube_dl/extractor/eitb.py
@@ -0,0 +1,37 @@
+# encoding: utf-8
+import re
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+from ..utils import ExtractorError
+
+
+class EitbIE(InfoExtractor):
+ IE_NAME = u'eitb.tv'
+ _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
+
+ _TEST = {
+ u'add_ie': ['Brightcove'],
+ u'url': u'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
+ u'md5': u'edf4436247185adee3ea18ce64c47998',
+ u'info_dict': {
+ u'id': u'2743577154001',
+ u'ext': u'mp4',
+ u'title': u'60 minutos (Lasa y Zabala, 30 años)',
+ # All videos from eitb has this description in the brightcove info
+ u'description': u'.',
+ u'uploader': u'Euskal Telebista',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ chapter_id = mobj.group('chapter_id')
+ webpage = self._download_webpage(url, chapter_id)
+ bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+ if bc_url is None:
+ raise ExtractorError(u'Could not extract the Brightcove url')
+ # The BrightcoveExperience object doesn't contain the video id, we set
+ # it manually
+ bc_url += '&%40videoPlayer={}'.format(chapter_id)
+ return self.url_result(bc_url, BrightcoveIE.ie_key())
From 4f045eef8fe63bcb925faaa5108a42d00b83f93a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 21:42:33 +0100
Subject: [PATCH 067/132] [youtube:channel] Fix the extraction
The page don't include the 'load more' button anymore, now we directly get the 'c4_browse_ajax' pages.
---
youtube_dl/extractor/youtube.py | 31 ++++++++++---------------------
1 file changed, 10 insertions(+), 21 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 74a381fe2..f745b8b14 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1572,7 +1572,6 @@ class YoutubePlaylistIE(InfoExtractor):
class YoutubeChannelIE(InfoExtractor):
IE_DESC = u'YouTube.com channels'
_VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
- _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
_MORE_PAGES_INDICATOR = 'yt-uix-load-more'
_MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
IE_NAME = u'youtube:channel'
@@ -1593,30 +1592,20 @@ class YoutubeChannelIE(InfoExtractor):
# Download channel page
channel_id = mobj.group(1)
video_ids = []
- pagenum = 1
- url = self._TEMPLATE_URL % (channel_id, pagenum)
- page = self._download_webpage(url, channel_id,
- u'Downloading page #%s' % pagenum)
+ # Download all channel pages using the json-based channel_ajax query
+ for pagenum in itertools.count(1):
+ url = self._MORE_PAGES_URL % (pagenum, channel_id)
+ page = self._download_webpage(url, channel_id,
+ u'Downloading page #%s' % pagenum)
- # Extract video identifiers
- ids_in_page = self.extract_videos_from_page(page)
- video_ids.extend(ids_in_page)
+ page = json.loads(page)
- # Download any subsequent channel pages using the json-based channel_ajax query
- if self._MORE_PAGES_INDICATOR in page:
- for pagenum in itertools.count(1):
- url = self._MORE_PAGES_URL % (pagenum, channel_id)
- page = self._download_webpage(url, channel_id,
- u'Downloading page #%s' % pagenum)
+ ids_in_page = self.extract_videos_from_page(page['content_html'])
+ video_ids.extend(ids_in_page)
- page = json.loads(page)
-
- ids_in_page = self.extract_videos_from_page(page['content_html'])
- video_ids.extend(ids_in_page)
-
- if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
- break
+ if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
+ break
self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
From 51040b72edab854aecb436a415e31789a86166ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 22:03:00 +0100
Subject: [PATCH 068/132] [brightcove] Support redirected urls from bcove.me
(fixes #1732)
'bctid' needs to be changed to '@videoPlayer', and 'bckey' to 'playerKey'.
---
youtube_dl/extractor/brightcove.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f0b79898c..0e60271f1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -97,8 +97,10 @@ class BrightcoveIE(InfoExtractor):
return None
def _real_extract(self, url):
- # Change the 'videoId' or 'videoID' field to '@videoPlayer'
- url = re.sub(r'(?<=[?&])videoI(d|D)', '%40videoPlayer', url)
+ # Change the 'videoId' and others field to '@videoPlayer'
+ url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
+ # Change bckey (used by bcove.me urls) to playerKey
+ url = re.sub(r'(?<=[?&])bckey', 'playerKey', url)
mobj = re.match(self._VALID_URL, url)
query_str = mobj.group('query')
query = compat_urlparse.parse_qs(query_str)
From 672fe94dcb69d6412a91c0996bf5ea1a6aa5ce93 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 22:11:46 +0100
Subject: [PATCH 069/132] release 2013.11.06.1
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 57b4d419c..ad253e361 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.06'
+__version__ = '2013.11.06.1'
From b1a80ec1a986219e3b04fb443a2622dacfba15f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 23:45:01 +0100
Subject: [PATCH 070/132] [xnxx] Accept urls that start with 'www' (fixes
#1734)
---
youtube_dl/extractor/xnxx.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 8a0eb1afd..1177a4b14 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -9,7 +9,7 @@ from ..utils import (
class XNXXIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)'
+ _VALID_URL = r'^(?:https?://)?(?:video|www)\.xnxx\.com/video([0-9]+)/(.*)'
VIDEO_URL_RE = r'flv_url=(.*?)&'
VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&'
From 4ac5306ae794d26400b7ae4513eb1be4f877403a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 Nov 2013 08:03:35 +0100
Subject: [PATCH 071/132] Fix the report progress when file_size is unknown
(#1731)
The report_progress function will accept eta and percent with None value and will set the message to 'Unknow ETA' or 'Unknown %'.
Otherwise the values must be numbers.
---
youtube_dl/FileDownloader.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index b0f26fc6f..20f57b6bf 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -234,8 +234,14 @@ class FileDownloader(object):
if self.params.get('noprogress', False):
return
clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
- eta_str = self.format_eta(eta)
- percent_str = self.format_percent(percent)
+ if eta is not None:
+ eta_str = self.format_eta(eta)
+ else:
+ eta_str = 'Unknown ETA'
+ if percent is not None:
+ percent_str = self.format_percent(percent)
+ else:
+ percent_str = 'Unknown %'
speed_str = self.format_speed(speed)
if self.params.get('progress_with_newline', False):
self.to_screen(u'[download] %s of %s at %s ETA %s' %
@@ -557,12 +563,11 @@ class FileDownloader(object):
# Progress message
speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
if data_len is None:
- self.report_progress('Unknown %', data_len_str, speed, 'Unknown ETA')
- eta = None
+ eta = percent = None
else:
percent = self.calc_percent(byte_counter, data_len)
eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
- self.report_progress(percent, data_len_str, speed, eta)
+ self.report_progress(percent, data_len_str, speed, eta)
self._hook_progress({
'downloaded_bytes': byte_counter,
From 6161d17579687abb4063380ae64c271398463e6b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 7 Nov 2013 11:06:34 +0100
Subject: [PATCH 072/132] release 2013.11.07
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ad253e361..84bf0f35c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.06.1'
+__version__ = '2013.11.07'
From dd5bcdc4c9cad0ee7e2d61343129ee1111048189 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 Nov 2013 21:06:48 +0100
Subject: [PATCH 073/132] [brightcove] Set the 'Referer' header if the url has
the 'linkBaseUrl' parameter (fixes #1553)
---
youtube_dl/extractor/brightcove.py | 17 +++++++++++++----
youtube_dl/extractor/generic.py | 12 +++++++-----
2 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0e60271f1..d8c35465a 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -10,10 +10,12 @@ from ..utils import (
find_xpath_attr,
compat_urlparse,
compat_str,
+ compat_urllib_request,
ExtractorError,
)
+
class BrightcoveIE(InfoExtractor):
_VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
_FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
@@ -80,6 +82,9 @@ class BrightcoveIE(InfoExtractor):
videoPlayer = find_xpath_attr(object_doc, './param', 'name', '@videoPlayer')
if videoPlayer is not None:
params['@videoPlayer'] = videoPlayer.attrib['value']
+ linkBase = find_xpath_attr(object_doc, './param', 'name', 'linkBaseURL')
+ if linkBase is not None:
+ params['linkBaseURL'] = linkBase.attrib['value']
data = compat_urllib_parse.urlencode(params)
return cls._FEDERATED_URL_TEMPLATE % data
@@ -107,14 +112,18 @@ class BrightcoveIE(InfoExtractor):
videoPlayer = query.get('@videoPlayer')
if videoPlayer:
- return self._get_video_info(videoPlayer[0], query_str)
+ return self._get_video_info(videoPlayer[0], query_str, query)
else:
player_key = query['playerKey']
return self._get_playlist_info(player_key[0])
- def _get_video_info(self, video_id, query):
- request_url = self._FEDERATED_URL_TEMPLATE % query
- webpage = self._download_webpage(request_url, video_id)
+ def _get_video_info(self, video_id, query_str, query):
+ request_url = self._FEDERATED_URL_TEMPLATE % query_str
+ req = compat_urllib_request.Request(request_url)
+ linkBase = query.get('linkBaseURL')
+ if linkBase is not None:
+ req.add_header('Referer', linkBase[0])
+ webpage = self._download_webpage(req, video_id)
self.report_extraction(video_id)
info = self._search_regex(r'var experienceJSON = ({.*?});', webpage, 'json')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 04b7212f4..c7552fddb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -55,15 +55,17 @@ class GenericIE(InfoExtractor):
u'skip': u'There is a limit of 200 free downloads / month for the test song',
},
# embedded brightcove video
+ # it also tests brightcove videos that need to set the 'Referer' in the
+ # http requests
{
u'add_ie': ['Brightcove'],
- u'url': u'http://www.scientificamerican.com/article.cfm?id=soap-bubble-physics',
+ u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
u'info_dict': {
- u'id': u'2365799484001',
+ u'id': u'2765128793001',
u'ext': u'mp4',
- u'title': u'Bubble Simulation',
- u'description': u'A visualization from a new computer model of foam behavior.',
- u'uploader': u'Scientific American',
+ u'title': u'Le cours de bourse : l’analyse technique',
+ u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9',
+ u'uploader': u'BFM BUSINESS',
},
u'params': {
u'skip_download': True,
From be07375b66df1a803eab27fee7112214f1f2b392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 16:40:00 +0100
Subject: [PATCH 074/132] Don't recode the video with m3u8 downloads (fixes
#1741)
---
youtube_dl/FileDownloader.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 20f57b6bf..35fa3ca61 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -379,7 +379,8 @@ class FileDownloader(object):
self.report_destination(filename)
tmpfilename = self.temp_name(filename)
- args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', tmpfilename]
+ args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+ '-absf', 'aac_adtstoasc', tmpfilename]
# Check for ffmpeg first
try:
subprocess.call(['ffmpeg', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
From 20aafee7fa17407124eccb9bcc047079acd31fcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 21:47:02 +0100
Subject: [PATCH 075/132] [kankan] Fix the video url
It now requires two additional parameters, one is a timestamp we get from the getCdnresource_flv page and the other is a key we have to build.
---
youtube_dl/extractor/kankan.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index 445d46501..50916f4a6 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -1,8 +1,10 @@
import re
+import hashlib
from .common import InfoExtractor
from ..utils import determine_ext
+_md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
class KankanIE(InfoExtractor):
_VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P<id>\d+)\.shtml'
@@ -30,7 +32,10 @@ class KankanIE(InfoExtractor):
video_id, u'Downloading video url info')
ip = self._search_regex(r'ip:"(.+?)"', video_info_page, u'video url ip')
path = self._search_regex(r'path:"(.+?)"', video_info_page, u'video url path')
- video_url = 'http://%s%s' % (ip, path)
+ param1 = self._search_regex(r'param1:(\d+)', video_info_page, u'param1')
+ param2 = self._search_regex(r'param2:(\d+)', video_info_page, u'param2')
+ key = _md5('xl_mp43651' + param1 + param2)
+ video_url = 'http://%s%s?key=%s&key1=%s' % (ip, path, key, param2)
return {'id': video_id,
'title': title,
From 12c167c881875360345b9b542052ac204b7c34a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 18:06:09 +0100
Subject: [PATCH 076/132] [soundcloud] Allow to download tracks marked as not
'streamable'
They use the rtmp protocol but if the are marked as 'downloadable' it can use the direct download link.
---
test/test_playlists.py | 9 ++
youtube_dl/extractor/soundcloud.py | 130 +++++++++++------------------
2 files changed, 59 insertions(+), 80 deletions(-)
diff --git a/test/test_playlists.py b/test/test_playlists.py
index de1e8d88e..706b6bdca 100644
--- a/test/test_playlists.py
+++ b/test/test_playlists.py
@@ -17,6 +17,7 @@ from youtube_dl.extractor import (
DailymotionUserIE,
VimeoChannelIE,
UstreamChannelIE,
+ SoundcloudSetIE,
SoundcloudUserIE,
LivestreamIE,
NHLVideocenterIE,
@@ -61,6 +62,14 @@ class TestPlaylists(unittest.TestCase):
self.assertEqual(result['id'], u'5124905')
self.assertTrue(len(result['entries']) >= 11)
+ def test_soundcloud_set(self):
+ dl = FakeYDL()
+ ie = SoundcloudSetIE(dl)
+ result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep')
+ self.assertIsPlaylist(result)
+ self.assertEqual(result['title'], u'The Royal Concept EP')
+ self.assertTrue(len(result['entries']) >= 6)
+
def test_soundcloud_user(self):
dl = FakeYDL()
ie = SoundcloudUserIE(dl)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 29cd5617c..4717fbb77 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,17 +29,34 @@ class SoundcloudIE(InfoExtractor):
)
'''
IE_NAME = u'soundcloud'
- _TEST = {
- u'url': u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
- u'file': u'62986583.mp3',
- u'md5': u'ebef0a451b909710ed1d7787dddbf0d7',
- u'info_dict': {
- u"upload_date": u"20121011",
- u"description": u"No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
- u"uploader": u"E.T. ExTerrestrial Music",
- u"title": u"Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
- }
- }
+ _TESTS = [
+ {
+ u'url': u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
+ u'file': u'62986583.mp3',
+ u'md5': u'ebef0a451b909710ed1d7787dddbf0d7',
+ u'info_dict': {
+ u"upload_date": u"20121011",
+ u"description": u"No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
+ u"uploader": u"E.T. ExTerrestrial Music",
+ u"title": u"Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
+ }
+ },
+ # not streamable song
+ {
+ u'url': u'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
+ u'info_dict': {
+ u'id': u'47127627',
+ u'ext': u'mp3',
+ u'title': u'Goldrushed',
+ u'uploader': u'The Royal Concept',
+ u'upload_date': u'20120521',
+ },
+ u'params': {
+ # rtmp
+ u'skip_download': True,
+ },
+ },
+ ]
_CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
@@ -56,16 +73,16 @@ class SoundcloudIE(InfoExtractor):
return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
def _extract_info_dict(self, info, full_title=None, quiet=False):
- video_id = info['id']
- name = full_title or video_id
+ track_id = compat_str(info['id'])
+ name = full_title or track_id
if quiet == False:
self.report_extraction(name)
thumbnail = info['artwork_url']
if thumbnail is not None:
thumbnail = thumbnail.replace('-large', '-t500x500')
- return {
- 'id': info['id'],
+ result = {
+ 'id': track_id,
'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
'uploader': info['user']['username'],
'upload_date': unified_strdate(info['created_at']),
@@ -74,6 +91,21 @@ class SoundcloudIE(InfoExtractor):
'description': info['description'],
'thumbnail': thumbnail,
}
+ if info.get('downloadable', False):
+ result['url'] = 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(track_id, self._CLIENT_ID)
+ if not info.get('streamable', False):
+ # We have to get the rtmp url
+ stream_json = self._download_webpage(
+ 'http://api.soundcloud.com/i1/tracks/{0}/streams?client_id={1}'.format(track_id, self._CLIENT_ID),
+ track_id, u'Downloading track url')
+ rtmp_url = json.loads(stream_json)['rtmp_mp3_128_url']
+ # The url doesn't have an rtmp app, we have to extract the playpath
+ url, path = rtmp_url.split('mp3:', 1)
+ result.update({
+ 'url': url,
+ 'play_path': 'mp3:' + path,
+ })
+ return result
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
@@ -106,70 +138,8 @@ class SoundcloudIE(InfoExtractor):
class SoundcloudSetIE(SoundcloudIE):
_VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$'
IE_NAME = u'soundcloud:set'
- _TEST = {
- u"url":"https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep",
- u"playlist": [
- {
- u"file":"30510138.mp3",
- u"md5":"f9136bf103901728f29e419d2c70f55d",
- u"info_dict": {
- u"upload_date": u"20111213",
- u"description": u"The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
- u"uploader": u"The Royal Concept",
- u"title": u"D-D-Dance"
- }
- },
- {
- u"file":"47127625.mp3",
- u"md5":"09b6758a018470570f8fd423c9453dd8",
- u"info_dict": {
- u"upload_date": u"20120521",
- u"description": u"The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
- u"uploader": u"The Royal Concept",
- u"title": u"The Royal Concept - Gimme Twice"
- }
- },
- {
- u"file":"47127627.mp3",
- u"md5":"154abd4e418cea19c3b901f1e1306d9c",
- u"info_dict": {
- u"upload_date": u"20120521",
- u"uploader": u"The Royal Concept",
- u"title": u"Goldrushed"
- }
- },
- {
- u"file":"47127629.mp3",
- u"md5":"2f5471edc79ad3f33a683153e96a79c1",
- u"info_dict": {
- u"upload_date": u"20120521",
- u"description": u"The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
- u"uploader": u"The Royal Concept",
- u"title": u"In the End"
- }
- },
- {
- u"file":"47127631.mp3",
- u"md5":"f9ba87aa940af7213f98949254f1c6e2",
- u"info_dict": {
- u"upload_date": u"20120521",
- u"description": u"The Royal Concept from Stockholm\r\nFilip / David / Povel / Magnus\r\nwww.theroyalconceptband.com",
- u"uploader": u"The Royal Concept",
- u"title": u"Knocked Up"
- }
- },
- {
- u"file":"75206121.mp3",
- u"md5":"f9d1fe9406717e302980c30de4af9353",
- u"info_dict": {
- u"upload_date": u"20130116",
- u"description": u"The unreleased track World on Fire premiered on the CW's hit show Arrow (8pm/7pm central). \r\nAs a gift to our fans we would like to offer you a free download of the track! ",
- u"uploader": u"The Royal Concept",
- u"title": u"World On Fire"
- }
- }
- ]
- }
+ # it's in tests/test_playlists.py
+ _TESTS = []
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
@@ -208,7 +178,7 @@ class SoundcloudUserIE(SoundcloudIE):
IE_NAME = u'soundcloud:user'
# it's in tests/test_playlists.py
- _TEST = None
+ _TESTS = []
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
From c2b6a482d5aadd40972944694307fca3d4d352f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 18:10:11 +0100
Subject: [PATCH 077/132] [brightcove] the format function requires to specify
the index in python2.6
---
youtube_dl/extractor/eitb.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 022d90abc..4ba323148 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -33,5 +33,5 @@ class EitbIE(InfoExtractor):
raise ExtractorError(u'Could not extract the Brightcove url')
# The BrightcoveExperience object doesn't contain the video id, we set
# it manually
- bc_url += '&%40videoPlayer={}'.format(chapter_id)
+ bc_url += '&%40videoPlayer={0}'.format(chapter_id)
return self.url_result(bc_url, BrightcoveIE.ie_key())
From 81be02d2f97f63fef0ec6216b8dca2b7184b36a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 18:16:32 +0100
Subject: [PATCH 078/132] [cnn] Accept www.cnn.com urls (fixes #1740)
---
youtube_dl/extractor/cnn.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index a79f881cd..34adf6dda 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -6,7 +6,7 @@ from ..utils import determine_ext
class CNNIE(InfoExtractor):
- _VALID_URL = r'''(?x)https?://(edition\.)?cnn\.com/video/(data/.+?|\?)/
+ _VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/
(?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))'''
_TESTS = [{
From 566d4e0425245e46f211fec8cd38d369e2c8de2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:01:23 +0100
Subject: [PATCH 079/132] [arte] Make sure the format_id is unique (closes
#1739)
Include the bitrate and use the height instead of the quality field.
---
youtube_dl/extractor/arte.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index e10c74c11..a636c6be5 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -10,6 +10,7 @@ from ..utils import (
unified_strdate,
determine_ext,
get_element_by_id,
+ compat_str,
)
# There are different sources of video in arte.tv, the extraction process
@@ -191,10 +192,13 @@ class ArteTVPlus7IE(InfoExtractor):
formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f.get('versionCode', '')) is None)
# Pick the best quality
def _format(format_info):
- quality = format_info['quality']
- m_quality = re.match(r'\w*? - (\d*)p', quality)
- if m_quality is not None:
- quality = m_quality.group(1)
+ quality = ''
+ height = format_info.get('height')
+ if height is not None:
+ quality = compat_str(height)
+ bitrate = format_info.get('bitrate')
+ if bitrate is not None:
+ quality += '-%d' % bitrate
if format_info.get('versionCode') is not None:
format_id = u'%s-%s' % (quality, format_info['versionCode'])
else:
@@ -203,7 +207,7 @@ class ArteTVPlus7IE(InfoExtractor):
'format_id': format_id,
'format_note': format_info.get('versionLibelle'),
'width': format_info.get('width'),
- 'height': format_info.get('height'),
+ 'height': height,
}
if format_info['mediaType'] == u'rtmp':
info['url'] = format_info['streamer']
From f470c6c812882d3726fe379d21069adac3cb2c69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:05:19 +0100
Subject: [PATCH 080/132] [arte] Improve the format sorting
Also use the bitrate.
Prefer normal version and sourds/mal version over original version with subtitles.
---
youtube_dl/extractor/arte.py | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index a636c6be5..b35a679e3 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -182,15 +182,22 @@ class ArteTVPlus7IE(InfoExtractor):
formats = all_formats
else:
raise ExtractorError(u'The formats list is empty')
- # We order the formats by quality
+
if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
- sort_key = lambda f: ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
+ def sort_key(f):
+ return ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
else:
- sort_key = lambda f: int(f.get('height',-1))
+ def sort_key(f):
+ return (
+ # Sort first by quality
+ int(f.get('height',-1)),
+ int(f.get('bitrate',-1)),
+ # The original version with subtitles has lower relevance
+ re.match(r'VO-ST(F|A)', f.get('versionCode', '')) is None,
+ # The version with sourds/mal subtitles has also lower relevance
+ re.match(r'VO?(F|A)-STM\1', f.get('versionCode', '')) is None,
+ )
formats = sorted(formats, key=sort_key)
- # Prefer videos without subtitles in the same language
- formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f.get('versionCode', '')) is None)
- # Pick the best quality
def _format(format_info):
quality = ''
height = format_info.get('height')
From 897d6cc43a25f68c941cda80afaa47acaf7779bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:07:34 +0100
Subject: [PATCH 081/132] Improve format listing for long format ids
Now arte.tv videos have quite long ids.
---
youtube_dl/YoutubeDL.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 86a6fd043..5253c39e1 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -782,7 +782,7 @@ class YoutubeDL(object):
def list_formats(self, info_dict):
def line(format):
- return (u'%-15s%-10s%-12s%s' % (
+ return (u'%-20s%-10s%-12s%s' % (
format['format_id'],
format['ext'],
self.format_resolution(format),
From 2a9e9b210b27778a71d920d4c7a0508c0cd446f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:21:30 +0100
Subject: [PATCH 082/132] Fix the documentation of '--autonumber-size' (#1743)
it's '--auto-number' not '--autonumber'
---
youtube_dl/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 48ffcbf8e..ab7879c5d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -349,7 +349,7 @@ def parseOpts(overrideArguments=None):
'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
filesystem.add_option('--autonumber-size',
dest='autonumber_size', metavar='NUMBER',
- help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
+ help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
filesystem.add_option('--restrict-filenames',
action='store_true', dest='restrictfilenames',
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
From be6dfd1b4954e32009f788ea55ee06bb82781f4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 12:09:12 +0100
Subject: [PATCH 083/132] [ted] Return a single info_dict for talks urls
It failed with the --list-subs option
---
test/test_ted_subtitles.py | 8 +++-----
youtube_dl/extractor/ted.py | 2 +-
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
index a0dd7eeed..4c164ae99 100644
--- a/test/test_ted_subtitles.py
+++ b/test/test_ted_subtitles.py
@@ -10,9 +10,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from youtube_dl.extractor import TEDIE
from youtube_dl.utils import *
-from helper import FakeYDL
-
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+from test.helper import FakeYDL, md5
class TestTedSubtitles(unittest.TestCase):
def setUp(self):
@@ -24,7 +22,7 @@ class TestTedSubtitles(unittest.TestCase):
return info_dict
def getSubtitles(self):
info_dict = self.getInfoDict()
- return info_dict[0]['subtitles']
+ return info_dict['subtitles']
def test_no_writesubtitles(self):
subtitles = self.getSubtitles()
self.assertEqual(subtitles, None)
@@ -45,7 +43,7 @@ class TestTedSubtitles(unittest.TestCase):
def test_list_subtitles(self):
self.DL.params['listsubtitles'] = True
info_dict = self.getInfoDict()
- self.assertEqual(info_dict, [None])
+ self.assertEqual(info_dict, None)
def test_automatic_captions(self):
self.DL.params['writeautomaticsub'] = True
self.DL.params['subtitleslang'] = ['en']
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1b006bc9b..76cfdfb90 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -36,7 +36,7 @@ class TEDIE(SubtitlesInfoExtractor):
def _real_extract(self, url):
m=re.match(self._VALID_URL, url, re.VERBOSE)
if m.group('type_talk'):
- return [self._talk_info(url)]
+ return self._talk_info(url)
else :
playlist_id=m.group('playlist_id')
name=m.group('name')
From ae84f879d77bab5ea6fd9a4d74b82b46a11c9072 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 12:28:21 +0100
Subject: [PATCH 084/132] Merge all the subtitles test into a single file
They reuse a base class
---
test/test_dailymotion_subtitles.py | 70 ----------
test/test_subtitles.py | 211 +++++++++++++++++++++++++++++
test/test_ted_subtitles.py | 67 ---------
test/test_youtube_subtitles.py | 95 -------------
4 files changed, 211 insertions(+), 232 deletions(-)
delete mode 100644 test/test_dailymotion_subtitles.py
create mode 100644 test/test_subtitles.py
delete mode 100644 test/test_ted_subtitles.py
delete mode 100644 test/test_youtube_subtitles.py
diff --git a/test/test_dailymotion_subtitles.py b/test/test_dailymotion_subtitles.py
deleted file mode 100644
index ba3580ea4..000000000
--- a/test/test_dailymotion_subtitles.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import FakeYDL, global_setup, md5
-global_setup()
-
-
-from youtube_dl.extractor import DailymotionIE
-
-class TestDailymotionSubtitles(unittest.TestCase):
- def setUp(self):
- self.DL = FakeYDL()
- self.url = 'http://www.dailymotion.com/video/xczg00'
- def getInfoDict(self):
- IE = DailymotionIE(self.DL)
- info_dict = IE.extract(self.url)
- return info_dict
- def getSubtitles(self):
- info_dict = self.getInfoDict()
- return info_dict['subtitles']
- def test_no_writesubtitles(self):
- subtitles = self.getSubtitles()
- self.assertEqual(subtitles, None)
- def test_subtitles(self):
- self.DL.params['writesubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
- def test_subtitles_lang(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['subtitleslangs'] = ['fr']
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
- def test_allsubtitles(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['allsubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(len(subtitles.keys()), 5)
- def test_list_subtitles(self):
- self.DL.expect_warning(u'Automatic Captions not supported by this server')
- self.DL.params['listsubtitles'] = True
- info_dict = self.getInfoDict()
- self.assertEqual(info_dict, None)
- def test_automatic_captions(self):
- self.DL.expect_warning(u'Automatic Captions not supported by this server')
- self.DL.params['writeautomaticsub'] = True
- self.DL.params['subtitleslang'] = ['en']
- subtitles = self.getSubtitles()
- self.assertTrue(len(subtitles.keys()) == 0)
- def test_nosubtitles(self):
- self.DL.expect_warning(u'video doesn\'t have subtitles')
- self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
- self.DL.params['writesubtitles'] = True
- self.DL.params['allsubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(len(subtitles), 0)
- def test_multiple_langs(self):
- self.DL.params['writesubtitles'] = True
- langs = ['es', 'fr', 'de']
- self.DL.params['subtitleslangs'] = langs
- subtitles = self.getSubtitles()
- for lang in langs:
- self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
new file mode 100644
index 000000000..06a304879
--- /dev/null
+++ b/test/test_subtitles.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup, md5
+global_setup()
+
+
+from youtube_dl.extractor import (
+ YoutubeIE,
+ DailymotionIE,
+ TEDIE,
+)
+
+
+class BaseTestSubtitles(unittest.TestCase):
+ url = None
+ IE = None
+ def setUp(self):
+ self.DL = FakeYDL()
+ self.ie = self.IE(self.DL)
+
+ def getInfoDict(self):
+ info_dict = self.ie.extract(self.url)
+ return info_dict
+
+ def getSubtitles(self):
+ info_dict = self.getInfoDict()
+ return info_dict['subtitles']
+
+
+class TestYoutubeSubtitles(BaseTestSubtitles):
+ url = 'QRS8MkLhQmM'
+ IE = YoutubeIE
+
+ def getSubtitles(self):
+ info_dict = self.getInfoDict()
+ return info_dict[0]['subtitles']
+
+ def test_youtube_no_writesubtitles(self):
+ self.DL.params['writesubtitles'] = False
+ subtitles = self.getSubtitles()
+ self.assertEqual(subtitles, None)
+
+ def test_youtube_subtitles(self):
+ self.DL.params['writesubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
+
+ def test_youtube_subtitles_lang(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['subtitleslangs'] = ['it']
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
+
+ def test_youtube_allsubtitles(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(len(subtitles.keys()), 13)
+
+ def test_youtube_subtitles_sbv_format(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['subtitlesformat'] = 'sbv'
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
+
+ def test_youtube_subtitles_vtt_format(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['subtitlesformat'] = 'vtt'
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
+
+ def test_youtube_list_subtitles(self):
+ self.DL.expect_warning(u'Video doesn\'t have automatic captions')
+ self.DL.params['listsubtitles'] = True
+ info_dict = self.getInfoDict()
+ self.assertEqual(info_dict, None)
+
+ def test_youtube_automatic_captions(self):
+ self.url = '8YoUxe5ncPo'
+ self.DL.params['writeautomaticsub'] = True
+ self.DL.params['subtitleslangs'] = ['it']
+ subtitles = self.getSubtitles()
+ self.assertTrue(subtitles['it'] is not None)
+
+ def test_youtube_nosubtitles(self):
+ self.DL.expect_warning(u'video doesn\'t have subtitles')
+ self.url = 'sAjKT8FhjI8'
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(len(subtitles), 0)
+
+ def test_youtube_multiple_langs(self):
+ self.url = 'QRS8MkLhQmM'
+ self.DL.params['writesubtitles'] = True
+ langs = ['it', 'fr', 'de']
+ self.DL.params['subtitleslangs'] = langs
+ subtitles = self.getSubtitles()
+ for lang in langs:
+ self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+
+class TestDailymotionSubtitles(BaseTestSubtitles):
+ url = 'http://www.dailymotion.com/video/xczg00'
+ IE = DailymotionIE
+
+ def test_no_writesubtitles(self):
+ subtitles = self.getSubtitles()
+ self.assertEqual(subtitles, None)
+
+ def test_subtitles(self):
+ self.DL.params['writesubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
+
+ def test_subtitles_lang(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['subtitleslangs'] = ['fr']
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
+
+ def test_allsubtitles(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(len(subtitles.keys()), 5)
+
+ def test_list_subtitles(self):
+ self.DL.expect_warning(u'Automatic Captions not supported by this server')
+ self.DL.params['listsubtitles'] = True
+ info_dict = self.getInfoDict()
+ self.assertEqual(info_dict, None)
+
+ def test_automatic_captions(self):
+ self.DL.expect_warning(u'Automatic Captions not supported by this server')
+ self.DL.params['writeautomaticsub'] = True
+ self.DL.params['subtitleslang'] = ['en']
+ subtitles = self.getSubtitles()
+ self.assertTrue(len(subtitles.keys()) == 0)
+
+ def test_nosubtitles(self):
+ self.DL.expect_warning(u'video doesn\'t have subtitles')
+ self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(len(subtitles), 0)
+
+ def test_multiple_langs(self):
+ self.DL.params['writesubtitles'] = True
+ langs = ['es', 'fr', 'de']
+ self.DL.params['subtitleslangs'] = langs
+ subtitles = self.getSubtitles()
+ for lang in langs:
+ self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+
+class TestTedSubtitles(BaseTestSubtitles):
+ url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
+ IE = TEDIE
+
+ def test_no_writesubtitles(self):
+ subtitles = self.getSubtitles()
+ self.assertEqual(subtitles, None)
+
+ def test_subtitles(self):
+ self.DL.params['writesubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d')
+
+ def test_subtitles_lang(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['subtitleslangs'] = ['fr']
+ subtitles = self.getSubtitles()
+ self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6')
+
+ def test_allsubtitles(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(len(subtitles.keys()), 28)
+
+ def test_list_subtitles(self):
+ self.DL.expect_warning(u'Automatic Captions not supported by this server')
+ self.DL.params['listsubtitles'] = True
+ info_dict = self.getInfoDict()
+ self.assertEqual(info_dict, None)
+
+ def test_automatic_captions(self):
+ self.DL.expect_warning(u'Automatic Captions not supported by this server')
+ self.DL.params['writeautomaticsub'] = True
+ self.DL.params['subtitleslang'] = ['en']
+ subtitles = self.getSubtitles()
+ self.assertTrue(len(subtitles.keys()) == 0)
+
+ def test_multiple_langs(self):
+ self.DL.params['writesubtitles'] = True
+ langs = ['es', 'fr', 'de']
+ self.DL.params['subtitleslangs'] = langs
+ subtitles = self.getSubtitles()
+ for lang in langs:
+ self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
deleted file mode 100644
index 4c164ae99..000000000
--- a/test/test_ted_subtitles.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import unittest
-import hashlib
-
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from youtube_dl.extractor import TEDIE
-from youtube_dl.utils import *
-from test.helper import FakeYDL, md5
-
-class TestTedSubtitles(unittest.TestCase):
- def setUp(self):
- self.DL = FakeYDL()
- self.url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
- def getInfoDict(self):
- IE = TEDIE(self.DL)
- info_dict = IE.extract(self.url)
- return info_dict
- def getSubtitles(self):
- info_dict = self.getInfoDict()
- return info_dict['subtitles']
- def test_no_writesubtitles(self):
- subtitles = self.getSubtitles()
- self.assertEqual(subtitles, None)
- def test_subtitles(self):
- self.DL.params['writesubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d')
- def test_subtitles_lang(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['subtitleslangs'] = ['fr']
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6')
- def test_allsubtitles(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['allsubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(len(subtitles.keys()), 28)
- def test_list_subtitles(self):
- self.DL.params['listsubtitles'] = True
- info_dict = self.getInfoDict()
- self.assertEqual(info_dict, None)
- def test_automatic_captions(self):
- self.DL.params['writeautomaticsub'] = True
- self.DL.params['subtitleslang'] = ['en']
- subtitles = self.getSubtitles()
- self.assertTrue(len(subtitles.keys()) == 0)
- # def test_nosubtitles(self):
- # self.DL.expect_warning(u'video doesn\'t have subtitles')
- # self.url = 'http://www.ted.com/talks/rodrigo_canales_the_deadly_genius_of_drug_cartels.html'
- # self.DL.params['writesubtitles'] = True
- # self.DL.params['allsubtitles'] = True
- # subtitles = self.getSubtitles()
- def test_multiple_langs(self):
- self.DL.params['writesubtitles'] = True
- langs = ['es', 'fr', 'de']
- self.DL.params['subtitleslangs'] = langs
- subtitles = self.getSubtitles()
- for lang in langs:
- self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/test/test_youtube_subtitles.py b/test/test_youtube_subtitles.py
deleted file mode 100644
index 00430a338..000000000
--- a/test/test_youtube_subtitles.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env python
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import FakeYDL, global_setup, md5
-global_setup()
-
-
-from youtube_dl.extractor import YoutubeIE
-
-
-class TestYoutubeSubtitles(unittest.TestCase):
- def setUp(self):
- self.DL = FakeYDL()
- self.url = 'QRS8MkLhQmM'
-
- def getInfoDict(self):
- IE = YoutubeIE(self.DL)
- info_dict = IE.extract(self.url)
- return info_dict
-
- def getSubtitles(self):
- info_dict = self.getInfoDict()
- return info_dict[0]['subtitles']
-
- def test_youtube_no_writesubtitles(self):
- self.DL.params['writesubtitles'] = False
- subtitles = self.getSubtitles()
- self.assertEqual(subtitles, None)
-
- def test_youtube_subtitles(self):
- self.DL.params['writesubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
-
- def test_youtube_subtitles_lang(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['subtitleslangs'] = ['it']
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
-
- def test_youtube_allsubtitles(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['allsubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(len(subtitles.keys()), 13)
-
- def test_youtube_subtitles_sbv_format(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['subtitlesformat'] = 'sbv'
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
-
- def test_youtube_subtitles_vtt_format(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['subtitlesformat'] = 'vtt'
- subtitles = self.getSubtitles()
- self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
-
- def test_youtube_list_subtitles(self):
- self.DL.expect_warning(u'Video doesn\'t have automatic captions')
- self.DL.params['listsubtitles'] = True
- info_dict = self.getInfoDict()
- self.assertEqual(info_dict, None)
-
- def test_youtube_automatic_captions(self):
- self.url = '8YoUxe5ncPo'
- self.DL.params['writeautomaticsub'] = True
- self.DL.params['subtitleslangs'] = ['it']
- subtitles = self.getSubtitles()
- self.assertTrue(subtitles['it'] is not None)
-
- def test_youtube_nosubtitles(self):
- self.DL.expect_warning(u'video doesn\'t have subtitles')
- self.url = 'sAjKT8FhjI8'
- self.DL.params['writesubtitles'] = True
- self.DL.params['allsubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(len(subtitles), 0)
-
- def test_youtube_multiple_langs(self):
- self.url = 'QRS8MkLhQmM'
- self.DL.params['writesubtitles'] = True
- langs = ['it', 'fr', 'de']
- self.DL.params['subtitleslangs'] = langs
- subtitles = self.getSubtitles()
- for lang in langs:
- self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
-
-if __name__ == '__main__':
- unittest.main()
From 0ed05a1d2db2335168497eb26b8588f2ee0039d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 12:45:17 +0100
Subject: [PATCH 085/132] Use the 'rtmp_live' field for the live parameter of
rtmpdump
---
youtube_dl/FileDownloader.py | 2 +-
youtube_dl/extractor/cinemassacre.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 04784f602..d0b3dd8fb 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -428,7 +428,7 @@ class FileDownloader(object):
info_dict.get('page_url', None),
info_dict.get('play_path', None),
info_dict.get('tc_url', None),
- info_dict.get('live', False))
+ info_dict.get('rtmp_live', False))
# Attempt to download using mplayer
if url.startswith('mms') or url.startswith('rtsp'):
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index d26f0f434..f0d08cebf 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -65,7 +65,7 @@ class CinemassacreIE(InfoExtractor):
{
'url': url,
'play_path': 'mp4:' + sd_file,
- 'live': True, # workaround
+ 'rtmp_live': True, # workaround
'ext': 'flv',
'format': 'sd',
'format_id': 'sd',
@@ -73,7 +73,7 @@ class CinemassacreIE(InfoExtractor):
{
'url': url,
'play_path': 'mp4:' + hd_file,
- 'live': True, # workaround
+ 'rtmp_live': True, # workaround
'ext': 'flv',
'format': 'hd',
'format_id': 'hd',
From 801dbbdffd9df57e4051e8fd01efbb6104f99c26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 16:47:03 +0100
Subject: [PATCH 086/132] Use avconv for downloading with m3u8 manifests if
it's available (fixes #1735)
---
youtube_dl/FileDownloader.py | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index d0b3dd8fb..088f59586 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -381,16 +381,20 @@ class FileDownloader(object):
self.report_destination(filename)
tmpfilename = self.temp_name(filename)
- args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
- '-absf', 'aac_adtstoasc', tmpfilename]
- # Check for ffmpeg first
- try:
- subprocess.call(['ffmpeg', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
- except (OSError, IOError):
- self.report_error(u'm3u8 download detected but "%s" could not be run' % args[0] )
- return False
+ args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+ '-bsf:a', 'aac_adtstoasc', tmpfilename]
- retval = subprocess.call(args)
+ for program in ['avconv', 'ffmpeg']:
+ try:
+ subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+ break
+ except (OSError, IOError):
+ pass
+ else:
+ self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found')
+ cmd = [program] + args
+
+ retval = subprocess.call(cmd)
if retval == 0:
fsize = os.path.getsize(encodeFilename(tmpfilename))
self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
From d4b7da84c3c20e86b67489a0521731c2c93385f7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 11 Nov 2013 14:21:09 +0100
Subject: [PATCH 087/132] Clarify -c. Do not pass it in if you don't know what
you're doing
Suggested in #1743
---
youtube_dl/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ab7879c5d..1f1db9f67 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -358,7 +358,7 @@ def parseOpts(overrideArguments=None):
filesystem.add_option('-w', '--no-overwrites',
action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
filesystem.add_option('-c', '--continue',
- action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
+ action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
filesystem.add_option('--no-continue',
action='store_false', dest='continue_dl',
help='do not resume partially downloaded files (restart from beginning)')
From 1777d5a952ab8d6728172b0a10afe20f060b432a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 11 Nov 2013 18:28:17 +0100
Subject: [PATCH 088/132] release 2013.11.11
---
README.md | 6 ++++--
youtube_dl/version.py | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/README.md b/README.md
index a2b296613..6632e5865 100644
--- a/README.md
+++ b/README.md
@@ -92,12 +92,14 @@ which means you can modify it, redistribute it or use it however you like.
ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
--autonumber-size NUMBER Specifies the number of digits in %(autonumber)s
when it is present in output filename template or
- --autonumber option is given
+ --auto-number option is given
--restrict-filenames Restrict filenames to only ASCII characters, and
avoid "&" and spaces in filenames
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
-w, --no-overwrites do not overwrite files
- -c, --continue resume partially downloaded files
+ -c, --continue force resume of partially downloaded files. By
+ default, youtube-dl will resume downloads if
+ possible.
--no-continue do not resume partially downloaded files (restart
from beginning)
--cookies FILE file to read cookies from and dump cookie jar in
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 84bf0f35c..338e7ba1f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.07'
+__version__ = '2013.11.11'
From eb0a83986642cf660820b168bd83c8770e3e5ce6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 12 Nov 2013 10:36:23 +0100
Subject: [PATCH 089/132] [common] Simplify og_search_property
---
youtube_dl/extractor/common.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fb2d50a09..9c20d30b4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -322,9 +322,9 @@ class InfoExtractor(object):
if name is None:
name = 'OpenGraph %s' % prop
escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
- if not escaped is None:
- return unescapeHTML(escaped)
- return None
+ if escaped is None:
+ return None
+ return unescapeHTML(escaped)
def _og_search_thumbnail(self, html, **kargs):
return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
From 72b18c5d34d7bf02f83d335b886921ff25c501f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 12 Nov 2013 20:38:13 +0100
Subject: [PATCH 090/132] FFmpegMetadataPP: don't enclose the values with "
(fixes #1756)
---
youtube_dl/PostProcessor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/PostProcessor.py b/youtube_dl/PostProcessor.py
index 13b56ede5..69aedf87a 100644
--- a/youtube_dl/PostProcessor.py
+++ b/youtube_dl/PostProcessor.py
@@ -501,7 +501,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
options = ['-c', 'copy']
for (name, value) in metadata.items():
- options.extend(['-metadata', '%s="%s"' % (name, value)])
+ options.extend(['-metadata', '%s=%s' % (name, value)])
options.extend(['-f', ext])
self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename)
From 8b8cbd8f6d3b525dcee3bc3df98c1ad0f093231d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 12 Nov 2013 20:50:52 +0100
Subject: [PATCH 091/132] [vine] Fix uploader extraction
---
youtube_dl/extractor/vine.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c4ec1f06f..651ba317d 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -27,7 +27,7 @@ class VineIE(InfoExtractor):
video_url = self._html_search_regex(r'<meta property="twitter:player:stream" content="(.+?)"',
webpage, u'video URL')
- uploader = self._html_search_regex(r'<div class="user">.*?<h2>(.+?)</h2>',
+ uploader = self._html_search_regex(r'<p class="username">(.*?)</p>',
webpage, u'uploader', fatal=False, flags=re.DOTALL)
return [{
From eb9b5bffef9d247bba4f2e8d387ddfbc47ece77b Mon Sep 17 00:00:00 2001
From: Jelle van der Waa <jelle@vdwaa.nl>
Date: Wed, 13 Nov 2013 10:30:41 +0100
Subject: [PATCH 092/132] Add extractor for gamekings.tv
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/gamekings.py | 39 +++++++++++++++++++++++++++++++
2 files changed, 40 insertions(+)
create mode 100644 youtube_dl/extractor/gamekings.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f9caca4ef..67a0ad5cb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -57,6 +57,7 @@ from .francetv import (
)
from .freesound import FreesoundIE
from .funnyordie import FunnyOrDieIE
+from .gamekings import GamekingsIE
from .gamespot import GameSpotIE
from .gametrailers import GametrailersIE
from .generic import GenericIE
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
new file mode 100644
index 000000000..eca71ab05
--- /dev/null
+++ b/youtube_dl/extractor/gamekings.py
@@ -0,0 +1,39 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ determine_ext,
+)
+
+
+class GamekingsIE(InfoExtractor):
+ _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-])'
+ _TEST = {
+ u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
+ u'file': u'20130811_PhoenixWright.mp4',
+ u'md5': u'8d42d15381e2dfa81dee86c7956d35ff',
+ u'info_dict': {
+ u"title": u"Phoenix Wright: Ace Attorney – Dual Destinies Review",
+ u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",
+ }
+ }
+
+ def _real_extract(self, url):
+
+ mobj = re.match(self._VALID_URL, url)
+ name = mobj.group('name')
+ webpage = self._download_webpage(url, name)
+ gamekings_url = self._og_search_video_url(webpage)
+
+ video = re.search(r'[0-9]+',gamekings_url)
+ video_id = video.group(0)
+
+ # Todo: add medium format
+ gamekings_url = gamekings_url.replace(video_id,'large/' + video_id)
+
+ return {'id': video_id,
+ 'ext': 'mp4',
+ 'url': gamekings_url,
+ 'title': self._og_search_title(webpage),
+ 'description': self._og_search_description(webpage),
+ }
From 384b98cd8f90ef2ac25c1f4f20ba9385adabaca8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 10:50:53 +0100
Subject: [PATCH 093/132] [gamekings] Minor fixes (#1759)
---
youtube_dl/extractor/gamekings.py | 29 +++++++++++++++--------------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index eca71ab05..4b4259447 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -7,13 +7,13 @@ from ..utils import (
class GamekingsIE(InfoExtractor):
- _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-])'
+ _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
_TEST = {
u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
- u'file': u'20130811_PhoenixWright.mp4',
- u'md5': u'8d42d15381e2dfa81dee86c7956d35ff',
+ u'file': u'20130811.mp4',
+ u'md5': u'17f6088f7d0149ff2b46f2714bdb1954',
u'info_dict': {
- u"title": u"Phoenix Wright: Ace Attorney – Dual Destinies Review",
+ u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review",
u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",
}
}
@@ -23,17 +23,18 @@ class GamekingsIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url)
name = mobj.group('name')
webpage = self._download_webpage(url, name)
- gamekings_url = self._og_search_video_url(webpage)
+ video_url = self._og_search_video_url(webpage)
- video = re.search(r'[0-9]+',gamekings_url)
+ video = re.search(r'[0-9]+', video_url)
video_id = video.group(0)
- # Todo: add medium format
- gamekings_url = gamekings_url.replace(video_id,'large/' + video_id)
+ # Todo: add medium format
+ video_url = video_url.replace(video_id, 'large/' + video_id)
- return {'id': video_id,
- 'ext': 'mp4',
- 'url': gamekings_url,
- 'title': self._og_search_title(webpage),
- 'description': self._og_search_description(webpage),
- }
+ return {
+ 'id': video_id,
+ 'ext': 'mp4',
+ 'url': video_url,
+ 'title': self._og_search_title(webpage),
+ 'description': self._og_search_description(webpage),
+ }
From b5bdc2699a5fead926114f6db6f1178181d75c58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 10:52:22 +0100
Subject: [PATCH 094/132] Credit @jelly for gamekings extractor (#1759)
---
youtube_dl/__init__.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 1f1db9f67..254fcd39c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -32,6 +32,7 @@ __authors__ = (
'Ismael Mejía',
'Steffan \'Ruirize\' James',
'Andras Elso',
+ 'Jelle van der Waa',
)
__license__ = 'Public Domain'
From c3a3028f9f7fa79ee7357b65252ff2c9a062bdc8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 11:06:53 +0100
Subject: [PATCH 095/132] [tvp] Minor improvements (#1730)
---
youtube_dl/extractor/tvp.py | 47 +++++++++++--------------------------
1 file changed, 14 insertions(+), 33 deletions(-)
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 63fb57bbe..32e0f5037 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,23 +1,17 @@
-# encoding: utf-8
-import re
import json
+import re
from .common import InfoExtractor
-from ..utils import (
- determine_ext,
- ExtractorError,
- RegexNotFoundError,
-)
+
class TvpIE(InfoExtractor):
IE_NAME = u'tvp.pl'
_VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
- _INFO_URL = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s'
-
_TEST = {
u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
- u'file': u'31.10.2013-12878238.wmv',
+ u'md5': u'148408967a6a468953c0a75cbdaf0d7a',
+ u'file': u'12878238.wmv',
u'info_dict': {
u'title': u'31.10.2013',
u'description': u'31.10.2013',
@@ -27,34 +21,21 @@ class TvpIE(InfoExtractor):
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- webpage = self._download_webpage(url, video_id, "Downloading video webpage")
- json_params = self._download_webpage(self._INFO_URL % video_id, video_id, "Downloading video metadata")
-
- try:
- params = json.loads(json_params)
- except:
- raise ExtractorError(u'Invalid JSON')
+ webpage = self._download_webpage(url, video_id)
+ json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id
+ json_params = self._download_webpage(
+ json_url, video_id, u"Downloading video metadata")
+ params = json.loads(json_params)
self.report_extraction(video_id)
- try:
- video_url = params['video_url']
- except KeyError:
- raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+ video_url = params['video_url']
- try:
- title = self._og_search_title(webpage)
- except RegexNotFoundError:
- title = video_id
- info = {
+ title = self._og_search_title(webpage, fatal=True)
+ return {
'id': video_id,
'title': title,
'ext': 'wmv',
'url': video_url,
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
}
- try:
- info['description'] = self._og_search_description(webpage)
- info['thumbnail'] = self._og_search_thumbnail(webpage)
- except RegexNotFoundError:
- pass
-
- return info
From d37936386f0217a411efe96483b97e590b7eceb3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 11:08:07 +0100
Subject: [PATCH 096/132] Credit @saper for tvp IE (#1730)
---
youtube_dl/__init__.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 254fcd39c..4dee487ab 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -33,6 +33,7 @@ __authors__ = (
'Steffan \'Ruirize\' James',
'Andras Elso',
'Jelle van der Waa',
+ 'Marcin Cieślak',
)
__license__ = 'Public Domain'
From 80b9bbce8687f800b79edb36edf8c193dcf26a78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 11:09:04 +0100
Subject: [PATCH 097/132] release 2013.11.13
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 338e7ba1f..26b91105f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.11'
+__version__ = '2013.11.13'
From ea7a7af1d46ecb51566db0af3e8779ab2a04b516 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 13 Nov 2013 17:13:06 +0100
Subject: [PATCH 098/132] [gamekings] Fix the test video checksum
---
youtube_dl/extractor/gamekings.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 4b4259447..6aa657ef4 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -11,7 +11,7 @@ class GamekingsIE(InfoExtractor):
_TEST = {
u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
u'file': u'20130811.mp4',
- u'md5': u'17f6088f7d0149ff2b46f2714bdb1954',
+ u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3',
u'info_dict': {
u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review",
u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",
From ca715127a2f95da30b6700a5e217a2acc904b459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 13 Nov 2013 17:06:02 +0100
Subject: [PATCH 099/132] Don't assume the 'subtitlesformat' is set in the
params dict (fixes #1750)
---
youtube_dl/YoutubeDL.py | 2 +-
youtube_dl/extractor/youtube.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5253c39e1..f615911de 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -640,7 +640,7 @@ class YoutubeDL(object):
# subtitles download errors are already managed as troubles in relevant IE
# that way it will silently go on when used with unsupporting IE
subtitles = info_dict['subtitles']
- sub_format = self.params.get('subtitlesformat')
+ sub_format = self.params.get('subtitlesformat', 'srt')
for sub_lang in subtitles.keys():
sub = subtitles[sub_lang]
if sub is None:
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c992cba97..56d30f52a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1098,7 +1098,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
params = compat_urllib_parse.urlencode({
'lang': lang,
'v': video_id,
- 'fmt': self._downloader.params.get('subtitlesformat'),
+ 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
'name': l[0].encode('utf-8'),
})
url = u'http://www.youtube.com/api/timedtext?' + params
@@ -1111,7 +1111,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
def _get_available_automatic_caption(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an
argument to speed up the process."""
- sub_format = self._downloader.params.get('subtitlesformat')
+ sub_format = self._downloader.params.get('subtitlesformat', 'srt')
self.to_screen(u'%s: Looking for automatic captions' % video_id)
mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
err_msg = u'Couldn\'t find automatic captions for %s' % video_id
From c66d2baa9cb33327a3318f49cbb89f9ac559c978 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Nov 2013 13:16:32 +0100
Subject: [PATCH 100/132] [livestream] Add an extractor for the original
version of livestream (closes #1764)
The two versions use different systems.
---
youtube_dl/extractor/__init__.py | 2 +-
youtube_dl/extractor/livestream.py | 44 ++++++++++++++++++++++++++++++
2 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0594a3666..ee3173468 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -80,7 +80,7 @@ from .keezmovies import KeezMoviesIE
from .kickstarter import KickStarterIE
from .keek import KeekIE
from .liveleak import LiveLeakIE
-from .livestream import LivestreamIE
+from .livestream import LivestreamIE, LivestreamOriginalIE
from .metacafe import MetacafeIE
from .metacritic import MetacriticIE
from .mit import TechTVMITIE, MITIE
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 4531fd6ab..1a3e0ae6b 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -1,16 +1,19 @@
import re
import json
+import xml.etree.ElementTree
from .common import InfoExtractor
from ..utils import (
compat_urllib_parse_urlparse,
compat_urlparse,
get_meta_content,
+ xpath_with_ns,
ExtractorError,
)
class LivestreamIE(InfoExtractor):
+ IE_NAME = u'livestream'
_VALID_URL = r'http://new.livestream.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$'
_TEST = {
u'url': u'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
@@ -54,3 +57,44 @@ class LivestreamIE(InfoExtractor):
info = json.loads(self._download_webpage(api_url, video_id,
u'Downloading video info'))
return self._extract_video_info(info)
+
+
+# The original version of Livestream uses a different system
+class LivestreamOriginalIE(InfoExtractor):
+ IE_NAME = u'livestream:original'
+ _VALID_URL = r'https?://www\.livestream\.com/(?P<user>[^/]+)/video\?.*?clipId=(?P<id>.*?)(&|$)'
+ _TEST = {
+ u'url': u'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
+ u'info_dict': {
+ u'id': u'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
+ u'ext': u'flv',
+ u'title': u'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
+ },
+ u'params': {
+ # rtmp
+ u'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ user = mobj.group('user')
+ api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
+
+ api_response = self._download_webpage(api_url, video_id)
+ info = xml.etree.ElementTree.fromstring(api_response.encode('utf-8'))
+ item = info.find('channel').find('item')
+ ns = {'media': 'http://search.yahoo.com/mrss'}
+ thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url']
+ # Remove the extension and number from the path (like 1.jpg)
+ path = self._search_regex(r'(user-files/.+)_.*?\.jpg$', thumbnail_url, u'path')
+
+ return {
+ 'id': video_id,
+ 'title': item.find('title').text,
+ 'url': 'rtmp://extondemand.livestream.com/ondemand',
+ 'play_path': 'mp4:trans/dv15/mogulus-{0}.mp4'.format(path),
+ 'ext': 'flv',
+ 'thumbnail': thumbnail_url,
+ }
From e3b9ab5e187a590143f7f6110e55d43fb78f15bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Nov 2013 19:45:39 +0100
Subject: [PATCH 101/132] [soundlcoud] Set the correct extension for the tracks
(fixes #1766)
Some tracks are not in mp3 format, they can be wav files.
---
youtube_dl/extractor/soundcloud.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 4717fbb77..83e1f055f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -87,7 +87,7 @@ class SoundcloudIE(InfoExtractor):
'uploader': info['user']['username'],
'upload_date': unified_strdate(info['created_at']),
'title': info['title'],
- 'ext': u'mp3',
+ 'ext': info.get('original_format', u'mp3'),
'description': info['description'],
'thumbnail': thumbnail,
}
From 9f9be844fcc5155ab3e832c8428c8f016bea819b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 01:45:34 +0100
Subject: [PATCH 102/132] [youtube] Fix protocol-independent URLs (Fixes #1768)
---
youtube_dl/extractor/youtube.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c992cba97..ed82e4fc0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1019,6 +1019,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
"""Turn the encrypted s field into a working signature"""
if player_url is not None:
+ if player_url.startswith(u'//'):
+ player_url = u'https:' + player_url
try:
player_id = (player_url, len(s))
if player_id not in self._player_cache:
From a25a5cfeecc1c6371ab28d6d458cd066baf7013d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 01:47:15 +0100
Subject: [PATCH 103/132] release 2013.11.15
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 26b91105f..cd9f0b546 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.13'
+__version__ = '2013.11.15'
From feee2ecfa9fbc6fd34246c7e167ac9542ae7def2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 11:04:26 +0100
Subject: [PATCH 104/132] Pass the 'download' argument to
'process_video_result' (fixes #1769)
---
youtube_dl/YoutubeDL.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f615911de..b5c670dd4 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -385,7 +385,7 @@ class YoutubeDL(object):
result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
if result_type == 'video':
self.add_extra_info(ie_result, extra_info)
- return self.process_video_result(ie_result)
+ return self.process_video_result(ie_result, download=download)
elif result_type == 'url':
# We have to add extra_info to the results because it may be
# contained in a playlist
From b9643eed7c9b081c91e72257b380ccbd92555254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 11:51:45 +0100
Subject: [PATCH 105/132] [youtube:channel] Fix the extraction of autogenerated
channels
The ajax pages are empty, now it looks directly in the channel's /videos page
---
youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++------------
1 file changed, 24 insertions(+), 13 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7d7aeb461..8c0e6f252 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1594,20 +1594,31 @@ class YoutubeChannelIE(InfoExtractor):
# Download channel page
channel_id = mobj.group(1)
video_ids = []
+ url = 'https://www.youtube.com/channel/%s/videos' % channel_id
+ channel_page = self._download_webpage(url, channel_id)
+ if re.search(r'channel-header-autogenerated-label', channel_page) is not None:
+ autogenerated = True
+ else:
+ autogenerated = False
- # Download all channel pages using the json-based channel_ajax query
- for pagenum in itertools.count(1):
- url = self._MORE_PAGES_URL % (pagenum, channel_id)
- page = self._download_webpage(url, channel_id,
- u'Downloading page #%s' % pagenum)
-
- page = json.loads(page)
-
- ids_in_page = self.extract_videos_from_page(page['content_html'])
- video_ids.extend(ids_in_page)
-
- if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
- break
+ if autogenerated:
+ # The videos are contained in a single page
+ # the ajax pages can't be used, they are empty
+ video_ids = self.extract_videos_from_page(channel_page)
+ else:
+ # Download all channel pages using the json-based channel_ajax query
+ for pagenum in itertools.count(1):
+ url = self._MORE_PAGES_URL % (pagenum, channel_id)
+ page = self._download_webpage(url, channel_id,
+ u'Downloading page #%s' % pagenum)
+
+ page = json.loads(page)
+
+ ids_in_page = self.extract_videos_from_page(page['content_html'])
+ video_ids.extend(ids_in_page)
+
+ if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
+ break
self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
From 85d61685f15bdc62709c699e849af512db78089f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:10:22 +0100
Subject: [PATCH 106/132] [tvp] Update the title and the description of the
test video
---
youtube_dl/extractor/tvp.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 32e0f5037..76721a986 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -13,8 +13,8 @@ class TvpIE(InfoExtractor):
u'md5': u'148408967a6a468953c0a75cbdaf0d7a',
u'file': u'12878238.wmv',
u'info_dict': {
- u'title': u'31.10.2013',
- u'description': u'31.10.2013',
+ u'title': u'31.10.2013 - Odcinek 2',
+ u'description': u'31.10.2013 - Odcinek 2',
},
}
From ab2d524780736249c8988313db021e83642c24d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:24:54 +0100
Subject: [PATCH 107/132] Improve the OpenGraph regex
* Do not accept '>' between the property and content attributes.
* Recognize the properties if the content attribute is before the property attribute using two regexes (fixes the extraction of the description for SlideshareIE).
---
youtube_dl/extractor/common.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c20d30b4..e02176852 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -315,13 +315,17 @@ class InfoExtractor(object):
# Helper functions for extracting OpenGraph info
@staticmethod
- def _og_regex(prop):
- return r'<meta.+?property=[\'"]og:%s[\'"].+?content=(?:"(.+?)"|\'(.+?)\')' % re.escape(prop)
+ def _og_regexes(prop):
+ esc_prop = re.escape(prop)
+ return [
+ r'<meta[^>]+?property=[\'"]og:%s[\'"][^>]+?content=(?:"(.+?)"|\'(.+?)\')' % esc_prop,
+ r'<meta[^>]+?content=(?:"(.+?)"|\'(.+?)\')[^>]+?property=[\'"]og:%s[\'"]' % esc_prop,
+ ]
def _og_search_property(self, prop, html, name=None, **kargs):
if name is None:
name = 'OpenGraph %s' % prop
- escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
+ escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
if escaped is None:
return None
return unescapeHTML(escaped)
@@ -336,8 +340,8 @@ class InfoExtractor(object):
return self._og_search_property('title', html, **kargs)
def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
- regexes = [self._og_regex('video')]
- if secure: regexes.insert(0, self._og_regex('video:secure_url'))
+ regexes = self._og_regexes('video')
+ if secure: regexes = self._og_regexes('video:secure_url') + regexes
return self._html_search_regex(regexes, html, name, **kargs)
def _rta_search(self, html):
From 78fb87b2837e15124b5855734a951598dfe025fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:54:13 +0100
Subject: [PATCH 108/132] Don't accept '>' inside the content attribute in
OpenGraph regexes
---
youtube_dl/extractor/common.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e02176852..45dd01789 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -316,10 +316,12 @@ class InfoExtractor(object):
# Helper functions for extracting OpenGraph info
@staticmethod
def _og_regexes(prop):
- esc_prop = re.escape(prop)
+ content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')'
+ property_re = r'property=[\'"]og:%s[\'"]' % re.escape(prop)
+ template = r'<meta[^>]+?%s[^>]+?%s'
return [
- r'<meta[^>]+?property=[\'"]og:%s[\'"][^>]+?content=(?:"(.+?)"|\'(.+?)\')' % esc_prop,
- r'<meta[^>]+?content=(?:"(.+?)"|\'(.+?)\')[^>]+?property=[\'"]og:%s[\'"]' % esc_prop,
+ template % (property_re, content_re),
+ template % (content_re, property_re),
]
def _og_search_property(self, prop, html, name=None, **kargs):
From d24ffe1cfadca8fdc37a7b2fb5c2f080c785ad1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:57:59 +0100
Subject: [PATCH 109/132] [rtlnow] Remove the test for nitro
The videos expire.
---
youtube_dl/extractor/rtlnow.py | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index 9ac7c3be8..2f238de35 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -62,18 +62,6 @@ class RTLnowIE(InfoExtractor):
u'skip_download': True,
},
},
- {
- u'url': u'http://www.rtlnitronow.de/recht-ordnung/stadtpolizei-frankfurt-gerichtsvollzieher-leipzig.php?film_id=129679&player=1&season=1',
- u'file': u'129679.flv',
- u'info_dict': {
- u'upload_date': u'20131016',
- u'title': u'Recht & Ordnung - Stadtpolizei Frankfurt/ Gerichtsvollzieher...',
- u'description': u'Stadtpolizei Frankfurt/ Gerichtsvollzieher Leipzig',
- },
- u'params': {
- u'skip_download': True,
- },
- },
{
u'url': u'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10',
u'file': u'124903.flv',
From 463a908705674b9411a01b7a696c84348ec0244e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 14:06:38 +0100
Subject: [PATCH 110/132] [ted] simplify
---
youtube_dl/extractor/ted.py | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 76cfdfb90..8001ca5a3 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -85,7 +85,7 @@ class TEDIE(SubtitlesInfoExtractor):
'ext': 'mp4',
'url': stream['file'],
'format': stream['id']
- } for stream in info['htmlStreams']]
+ } for stream in info['htmlStreams']]
video_id = info['id']
@@ -95,7 +95,7 @@ class TEDIE(SubtitlesInfoExtractor):
self._list_available_subtitles(video_id, webpage)
return
- info = {
+ return {
'id': video_id,
'title': title,
'thumbnail': thumbnail,
@@ -104,11 +104,6 @@ class TEDIE(SubtitlesInfoExtractor):
'formats': formats,
}
- # TODO: Remove when #980 has been merged
- info.update(info['formats'][-1])
-
- return info
-
def _get_available_subtitles(self, video_id, webpage):
try:
options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
From fc2ef392bee9e46564d4106d3b0e5f6b8d71e37b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 14:33:51 +0100
Subject: [PATCH 111/132] [ted] Fix playlists (Fixes #1770)
---
youtube_dl/extractor/ted.py | 29 ++++++++++++++---------------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 8001ca5a3..2e497c86e 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -43,26 +43,25 @@ class TEDIE(SubtitlesInfoExtractor):
self.to_screen(u'Getting info of playlist %s: "%s"' % (playlist_id,name))
return [self._playlist_videos_info(url,name,playlist_id)]
- def _playlist_videos_info(self,url,name,playlist_id=0):
+
+ def _playlist_videos_info(self, url, name, playlist_id):
'''Returns the videos of the playlist'''
- video_RE=r'''
- <li\ id="talk_(\d+)"([.\s]*?)data-id="(?P<video_id>\d+)"
- ([.\s]*?)data-playlist_item_id="(\d+)"
- ([.\s]*?)data-mediaslug="(?P<mediaSlug>.+?)"
- '''
- video_name_RE=r'<p\ class="talk-title"><a href="(?P<talk_url>/talks/(.+).html)">(?P<fullname>.+?)</a></p>'
- webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage')
- m_videos=re.finditer(video_RE,webpage,re.VERBOSE)
- m_names=re.finditer(video_name_RE,webpage)
+
+ webpage = self._download_webpage(
+ url, playlist_id, u'Downloading playlist webpage')
+ matches = re.finditer(
+ r'<p\s+class="talk-title[^"]*"><a\s+href="(?P<talk_url>/talks/[^"]+\.html)">[^<]*</a></p>',
+ webpage)
playlist_title = self._html_search_regex(r'div class="headline">\s*?<h1>\s*?<span>(.*?)</span>',
webpage, 'playlist title')
- playlist_entries = []
- for m_video, m_name in zip(m_videos,m_names):
- talk_url='http://www.ted.com%s' % m_name.group('talk_url')
- playlist_entries.append(self.url_result(talk_url, 'TED'))
- return self.playlist_result(playlist_entries, playlist_id = playlist_id, playlist_title = playlist_title)
+ playlist_entries = [
+ self.url_result(u'http://www.ted.com' + m.group('talk_url'), 'TED')
+ for m in matches
+ ]
+ return self.playlist_result(
+ playlist_entries, playlist_id=playlist_id, playlist_title=playlist_title)
def _talk_info(self, url, video_id=0):
"""Return the video for the talk in the url"""
From aa13b2dffde73978fe2a169d3f99de7e5e7754cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 14:35:00 +0100
Subject: [PATCH 112/132] release 2013.11.15.1
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cd9f0b546..b04238eb5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.15'
+__version__ = '2013.11.15.1'
From 91c7271aabdd74c833ef570db59018e2d9f9d803 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:08:43 +0100
Subject: [PATCH 113/132] Add automatic generation of format note based on
bitrate and codecs
---
youtube_dl/YoutubeDL.py | 18 +++++++++++++++++-
youtube_dl/extractor/common.py | 4 ++++
youtube_dl/extractor/vevo.py | 7 ++++---
3 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5c670dd4..9c79af1f2 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -781,12 +781,28 @@ class YoutubeDL(object):
return res
def list_formats(self, info_dict):
+ def format_note(fdict):
+ if fdict.get('format_note') is not None:
+ return fdict['format_note']
+ res = u''
+ if fdict.get('vcodec') is not None:
+ res += fdict['vcodec']
+ if fdict.get('vbr') is not None:
+ res += u'@%4dk' % fdict['vbr']
+ if fdict.get('acodec') is not None:
+ if res:
+ res += u', '
+ res += fdict['acodec']
+ if fdict.get('abr') is not None:
+ res += u'@%3dk' % fdict['abr']
+ return res
+
def line(format):
return (u'%-20s%-10s%-12s%s' % (
format['format_id'],
format['ext'],
self.format_resolution(format),
- format.get('format_note', ''),
+ format_note(format),
)
)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 45dd01789..f787d0a3c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -71,6 +71,10 @@ class InfoExtractor(object):
("3D" or "DASH video")
* width Width of the video, if known
* height Height of the video, if known
+ * abr Average audio bitrate in KBit/s
+ * acodec Name of the audio codec in use
+ * vbr Average video bitrate in KBit/s
+ * vcodec Name of the video codec in use
webpage_url: The url to the video webpage, if given to youtube-dl it
should allow to get the same result again. (It will be set
by YoutubeDL if it's missing)
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 3f6020f74..4378b1780 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -78,12 +78,13 @@ class VevoIE(InfoExtractor):
continue
format_url = self._SMIL_BASE_URL + m.group('path')
- format_note = ('%(vcodec)s@%(vbr)4sk, %(acodec)s@%(abr)3sk' %
- m.groupdict())
formats.append({
'url': format_url,
'format_id': u'SMIL_' + m.group('cbr'),
- 'format_note': format_note,
+ 'vcodec': m.group('vcodec'),
+ 'acodec': m.group('acodec'),
+ 'vbr': int(m.group('vbr')),
+ 'abr': int(m.group('abr')),
'ext': m.group('ext'),
'width': int(m.group('width')),
'height': int(m.group('height')),
From 7150858d49f05a5650a12b5f4694f91dfb9595d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:33:12 +0100
Subject: [PATCH 114/132] [spiegel] Implement format selection
---
youtube_dl/YoutubeDL.py | 10 +++++--
youtube_dl/extractor/spiegel.py | 50 ++++++++++++++++++++++++---------
2 files changed, 44 insertions(+), 16 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9c79af1f2..273f7d977 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -786,13 +786,19 @@ class YoutubeDL(object):
return fdict['format_note']
res = u''
if fdict.get('vcodec') is not None:
- res += fdict['vcodec']
+ res += u'%-5s' % fdict['vcodec']
+ elif fdict.get('vbr') is not None:
+ res += u'video'
if fdict.get('vbr') is not None:
res += u'@%4dk' % fdict['vbr']
if fdict.get('acodec') is not None:
if res:
res += u', '
- res += fdict['acodec']
+ res += u'%-5s' % fdict['acodec']
+ elif fdict.get('abr') is not None:
+ if res:
+ res += u', '
+ res += 'audio'
if fdict.get('abr') is not None:
res += u'@%3dk' % fdict['abr']
return res
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 13c86401c..6dc2eda6d 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -2,18 +2,27 @@ import re
import xml.etree.ElementTree
from .common import InfoExtractor
+from ..utils import determine_ext
class SpiegelIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$'
- _TEST = {
+ _TESTS = [{
u'url': u'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
u'file': u'1259285.mp4',
u'md5': u'2c2754212136f35fb4b19767d242f66e',
u'info_dict': {
u"title": u"Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
}
- }
+ },
+ {
+ u'url': u'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
+ u'file': u'1309159.mp4',
+ u'md5': u'f2cdf638d7aa47654e251e1aee360af1',
+ u'info_dict': {
+ u'title': u'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers'
+ }
+ }]
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
@@ -21,25 +30,38 @@ class SpiegelIE(InfoExtractor):
webpage = self._download_webpage(url, video_id)
- video_title = self._html_search_regex(r'<div class="module-title">(.*?)</div>',
- webpage, u'title')
+ video_title = self._html_search_regex(
+ r'<div class="module-title">(.*?)</div>', webpage, u'title')
xml_url = u'http://video2.spiegel.de/flash/' + video_id + u'.xml'
- xml_code = self._download_webpage(xml_url, video_id,
- note=u'Downloading XML', errnote=u'Failed to download XML')
+ xml_code = self._download_webpage(
+ xml_url, video_id,
+ note=u'Downloading XML', errnote=u'Failed to download XML')
idoc = xml.etree.ElementTree.fromstring(xml_code)
- last_type = idoc[-1]
- filename = last_type.findall('./filename')[0].text
- duration = float(last_type.findall('./duration')[0].text)
- video_url = 'http://video2.spiegel.de/flash/' + filename
- video_ext = filename.rpartition('.')[2]
+ formats = [
+ {
+ 'format_id': n.tag.rpartition('type')[2],
+ 'url': u'http://video2.spiegel.de/flash/' + n.find('./filename').text,
+ 'width': int(n.find('./width').text),
+ 'height': int(n.find('./height').text),
+ 'abr': int(n.find('./audiobitrate').text),
+ 'vbr': int(n.find('./videobitrate').text),
+ 'vcodec': n.find('./codec').text,
+ 'acodec': 'MP4A',
+ }
+ for n in list(idoc)
+ # Blacklist type 6, it's extremely LQ and not available on the same server
+ if n.tag.startswith('type') and n.tag != 'type6'
+ ]
+ formats.sort(key=lambda f: f['vbr'])
+ duration = float(idoc[0].findall('./duration')[0].text)
+
info = {
'id': video_id,
- 'url': video_url,
- 'ext': video_ext,
'title': video_title,
'duration': duration,
+ 'formats': formats,
}
- return [info]
+ return info
From b5349e8721d0580a50519593926726a2ea832c9b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:39:45 +0100
Subject: [PATCH 115/132] Fix indentation of (best) and (worst) in
--list-formats
---
youtube_dl/YoutubeDL.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 273f7d977..d29e8bec5 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -815,8 +815,8 @@ class YoutubeDL(object):
formats = info_dict.get('formats', [info_dict])
formats_s = list(map(line, formats))
if len(formats) > 1:
- formats_s[0] += (' ' if formats[0].get('format_note') else '') + '(worst)'
- formats_s[-1] += (' ' if formats[-1].get('format_note') else '') + '(best)'
+ formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)'
+ formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)'
header_line = line({
'format_id': u'format code', 'ext': u'extension',
From f058e340114cb599e21001cef25ab3d02cfb194f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:56:23 +0100
Subject: [PATCH 116/132] [dailymotion] Fix playlists
---
youtube_dl/extractor/dailymotion.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index e87690f9d..71f5e03ee 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -186,7 +186,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
webpage = self._download_webpage(request,
id, u'Downloading page %s' % pagenum)
- playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
+ playlist_el = get_element_by_attribute(u'class', u'row video_list', webpage)
video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))
if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
From ce152341a118c94e442f42db8008de95aebec56b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:59:28 +0100
Subject: [PATCH 117/132] [bambuser] Do not test for MD5, seems to be flaky
---
youtube_dl/extractor/bambuser.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index f3b36f473..967568c4a 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -15,7 +15,8 @@ class BambuserIE(InfoExtractor):
_TEST = {
u'url': u'http://bambuser.com/v/4050584',
- u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
+ # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
+ #u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
u'info_dict': {
u'id': u'4050584',
u'ext': u'flv',
From 52d703d3d1b0d24f6c4c01ef8fcf33bfee78928b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 02:09:30 +0100
Subject: [PATCH 118/132] [tvp] Skip tests
---
youtube_dl/extractor/tvp.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 76721a986..bfed9dd04 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -16,6 +16,7 @@ class TvpIE(InfoExtractor):
u'title': u'31.10.2013 - Odcinek 2',
u'description': u'31.10.2013 - Odcinek 2',
},
+ u'skip': u'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
}
def _real_extract(self, url):
From 21ea3e06c9e39ac2cbfa24e7c02ebac936202893 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 02:31:02 +0100
Subject: [PATCH 119/132] [gamekings] remove unnecessary import
---
youtube_dl/extractor/gamekings.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 6aa657ef4..64715581d 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -1,9 +1,6 @@
import re
from .common import InfoExtractor
-from ..utils import (
- determine_ext,
-)
class GamekingsIE(InfoExtractor):
From eab2724138670159f1946608359f5e5bd5a7e7af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 02:32:08 +0100
Subject: [PATCH 120/132] [gamekings] Do not test md5 sum, precise file changes
regularly
---
youtube_dl/extractor/gamekings.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 64715581d..c91669b0e 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -8,7 +8,8 @@ class GamekingsIE(InfoExtractor):
_TEST = {
u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
u'file': u'20130811.mp4',
- u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3',
+ # MD5 is flaky, seems to change regularly
+ #u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3',
u'info_dict': {
u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review",
u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",
From d1c252048bf74add82de0b9812b589956f3c8a69 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 10:30:09 +0100
Subject: [PATCH 121/132] [redtube] Do not test md5, seems to vary
---
youtube_dl/extractor/redtube.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 994778e16..3bbda128e 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -8,7 +8,9 @@ class RedTubeIE(InfoExtractor):
_TEST = {
u'url': u'http://www.redtube.com/66418',
u'file': u'66418.mp4',
- u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d',
+ # md5 varies from time to time, as in
+ # https://travis-ci.org/rg3/youtube-dl/jobs/14052463#L295
+ #u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d',
u'info_dict': {
u"title": u"Sucked on a toilet",
u"age_limit": 18,
From ddf49c63445d236138846d778b18db6ede067fc8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 11:05:49 +0100
Subject: [PATCH 122/132] [arte] remove two typos
---
youtube_dl/extractor/arte.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b35a679e3..44d0b5d70 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -69,7 +69,7 @@ class ArteTvIE(InfoExtractor):
lang = mobj.group('lang')
return self._extract_liveweb(url, name, lang)
- if re.search(self._LIVE_URL, video_id) is not None:
+ if re.search(self._LIVE_URL, url) is not None:
raise ExtractorError(u'Arte live streams are not yet supported, sorry')
# self.extractLiveStream(url)
# return
@@ -115,7 +115,7 @@ class ArteTvIE(InfoExtractor):
event_doc = config_doc.find('event')
url_node = event_doc.find('video').find('urlHd')
if url_node is None:
- url_node = video_doc.find('urlSd')
+ url_node = event_doc.find('urlSd')
return {'id': video_id,
'title': event_doc.find('name%s' % lang.capitalize()).text,
From 1d699755e0978f2ec0932ef8d6562394a4799871 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 11:06:16 +0100
Subject: [PATCH 123/132] [youtube] Add view_count (Fixes #1781)
---
youtube_dl/extractor/youtube.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8c0e6f252..1aa549740 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1301,6 +1301,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
else:
raise ExtractorError(u'"token" parameter not in video info for unknown reason')
+ if 'view_count' in video_info:
+ view_count = int(video_info['view_count'][0])
+ else:
+ view_count = None
+
# Check for "rental" videos
if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
raise ExtractorError(u'"rental" videos not supported')
@@ -1489,6 +1494,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
'age_limit': 18 if age_gate else 0,
'annotations': video_annotations,
'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id,
+ 'view_count': view_count,
})
return results
From 1e5b9a95fd2049e024b3ee2f13b4da5c308d2e9c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 11:39:52 +0100
Subject: [PATCH 124/132] Move console_title to YoutubeDL
---
youtube_dl/FileDownloader.py | 17 +++--------------
youtube_dl/YoutubeDL.py | 13 +++++++++++++
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 088f59586..e5a542ed5 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -5,9 +5,6 @@ import subprocess
import sys
import time
-if os.name == 'nt':
- import ctypes
-
from .utils import (
compat_urllib_error,
compat_urllib_request,
@@ -151,16 +148,8 @@ class FileDownloader(object):
def to_stderr(self, message):
self.ydl.to_screen(message)
- def to_cons_title(self, message):
- """Set console/terminal window title to message."""
- if not self.params.get('consoletitle', False):
- return
- if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
- # c_wchar_p() might not be necessary if `message` is
- # already of type unicode()
- ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
- elif 'TERM' in os.environ:
- self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+ def to_console_title(self, message):
+ self.ydl.to_console_title(message)
def trouble(self, *args, **kargs):
self.ydl.trouble(*args, **kargs)
@@ -249,7 +238,7 @@ class FileDownloader(object):
else:
self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' %
(clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
- self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
+ self.to_console_title(u'youtube-dl - %s of %s at %s ETA %s' %
(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
def report_resuming_byte(self, resume_len):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d29e8bec5..6ea865bd9 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -13,6 +13,9 @@ import sys
import time
import traceback
+if os.name == 'nt':
+ import ctypes
+
from .utils import *
from .extractor import get_info_extractor, gen_extractors
from .FileDownloader import FileDownloader
@@ -176,6 +179,16 @@ class YoutubeDL(object):
output = output.encode(preferredencoding())
sys.stderr.write(output)
+ def to_console_title(self, message):
+ if not self.params.get('consoletitle', False):
+ return
+ if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+ # c_wchar_p() might not be necessary if `message` is
+ # already of type unicode()
+ ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
+ elif 'TERM' in os.environ:
+ self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+
def fixed_template(self):
"""Checks if the output template is fixed."""
return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
From ce02ed60f27ea27e66c33af745dc7e716377b46f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 16:47:52 +0100
Subject: [PATCH 125/132] Remove * imports
---
youtube_dl/YoutubeDL.py | 28 ++++++++++++++++++++++++++--
youtube_dl/update.py | 6 +++++-
2 files changed, 31 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6ea865bd9..6e5ae44d3 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -16,7 +16,31 @@ import traceback
if os.name == 'nt':
import ctypes
-from .utils import *
+from .utils import (
+ compat_http_client,
+ compat_print,
+ compat_str,
+ compat_urllib_error,
+ compat_urllib_request,
+ ContentTooShortError,
+ date_from_str,
+ DateRange,
+ determine_ext,
+ DownloadError,
+ encodeFilename,
+ ExtractorError,
+ locked_file,
+ MaxDownloadsReached,
+ PostProcessingError,
+ preferredencoding,
+ SameFileError,
+ sanitize_filename,
+ subtitles_filename,
+ takewhile_inclusive,
+ UnavailableVideoError,
+ write_json_file,
+ write_string,
+)
from .extractor import get_info_extractor, gen_extractors
from .FileDownloader import FileDownloader
@@ -267,7 +291,7 @@ class YoutubeDL(object):
"""Report file has already been fully downloaded."""
try:
self.to_screen(u'[download] %s has already been downloaded' % file_name)
- except (UnicodeEncodeError) as err:
+ except UnicodeEncodeError:
self.to_screen(u'[download] The file has already been downloaded')
def increment_downloads(self):
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 0689a4891..f41b4785a 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -2,11 +2,15 @@ import io
import json
import traceback
import hashlib
+import os
import subprocess
import sys
from zipimport import zipimporter
-from .utils import *
+from .utils import (
+ compat_str,
+ compat_urllib_request,
+)
from .version import __version__
def rsa_verify(message, signature, key):
From 90b6bbc38c7258358e294e7f1bcb3d46cd56ffd9 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sun, 17 Nov 2013 17:42:24 +0100
Subject: [PATCH 126/132] [SouthParkStudiosIE] Also detect urls without http://
or www
---
youtube_dl/extractor/southparkstudios.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py
index b1e96b679..bb0c4b393 100644
--- a/youtube_dl/extractor/southparkstudios.py
+++ b/youtube_dl/extractor/southparkstudios.py
@@ -5,7 +5,7 @@ from .mtv import MTVIE, _media_xml_tag
class SouthParkStudiosIE(MTVIE):
IE_NAME = u'southparkstudios.com'
- _VALID_URL = r'https?://www\.southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$)'
+ _VALID_URL = r'(https?://)?(www\.)?(?P<url>southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
_FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
@@ -31,6 +31,7 @@ class SouthParkStudiosIE(MTVIE):
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
+ url = u'http://www.' + mobj.group(u'url')
video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id)
mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"',
From 1672647ade97e070fa67eeff68370910ae715573 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sun, 17 Nov 2013 17:43:58 +0100
Subject: [PATCH 127/132] [SouthParkStudiosIE] Move from _TEST to _TESTS
---
youtube_dl/extractor/southparkstudios.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py
index bb0c4b393..a75e328a7 100644
--- a/youtube_dl/extractor/southparkstudios.py
+++ b/youtube_dl/extractor/southparkstudios.py
@@ -9,17 +9,15 @@ class SouthParkStudiosIE(MTVIE):
_FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
- _TEST = {
+ # Overwrite MTVIE properties we don't want
+ _TESTS = [{
u'url': u'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured',
u'file': u'a7bff6c2-ed00-11e0-aca6-0026b9414f30.mp4',
u'info_dict': {
u'title': u'Bat Daded',
u'description': u'Randy disqualifies South Park by getting into a fight with Bat Dad.',
},
- }
-
- # Overwrite MTVIE properties we don't want
- _TESTS = []
+ }]
def _get_thumbnail_url(self, uri, itemdoc):
search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))
From 746f491f82cfddd6fafacbf9978205963d7a214d Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sun, 17 Nov 2013 17:54:47 +0100
Subject: [PATCH 128/132] Add support for southpark.de
---
youtube_dl/extractor/__init__.py | 5 ++++-
youtube_dl/extractor/southparkstudios.py | 14 ++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ee3173468..2d1e3cdfd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -116,7 +116,10 @@ from .slashdot import SlashdotIE
from .slideshare import SlideshareIE
from .sohu import SohuIE
from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE
-from .southparkstudios import SouthParkStudiosIE
+from .southparkstudios import (
+ SouthParkStudiosIE,
+ SouthparkDeIE,
+)
from .space import SpaceIE
from .spankwire import SpankwireIE
from .spiegel import SpiegelIE
diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py
index a75e328a7..a711531e6 100644
--- a/youtube_dl/extractor/southparkstudios.py
+++ b/youtube_dl/extractor/southparkstudios.py
@@ -35,3 +35,17 @@ class SouthParkStudiosIE(MTVIE):
mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"',
webpage, u'mgid')
return self._get_videos_info(mgid)
+
+class SouthparkDeIE(SouthParkStudiosIE):
+ IE_NAME = u'southpark.de'
+ _VALID_URL = r'(https?://)?(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
+ _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'
+
+ _TESTS = [{
+ u'url': u'http://www.southpark.de/clips/uygssh/the-government-wont-respect-my-privacy#tab=featured',
+ u'file': u'85487c96-b3b9-4e39-9127-ad88583d9bf2.mp4',
+ u'info_dict': {
+ u'title': u'The Government Won\'t Respect My Privacy',
+ u'description': u'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
+ },
+ }]
From bdde425cbe01329d8c24e18cf0492465abb21411 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 21:05:14 +0100
Subject: [PATCH 129/132] Save and restore console title (Fixes #1782)
---
youtube_dl/YoutubeDL.py | 19 ++++++++
youtube_dl/__init__.py | 97 +++++++++++++++++++++--------------------
2 files changed, 68 insertions(+), 48 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6e5ae44d3..4e28f9120 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -213,6 +213,25 @@ class YoutubeDL(object):
elif 'TERM' in os.environ:
self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+ def save_console_title(self):
+ if not self.params.get('consoletitle', False):
+ return
+ if 'TERM' in os.environ:
+ self.to_screen('\033[22t')
+
+ def restore_console_title(self):
+ if not self.params.get('consoletitle', False):
+ return
+ if 'TERM' in os.environ:
+ self.to_screen('\033[23t')
+
+ def __enter__(self):
+ self.save_console_title()
+ return self
+
+ def __exit__(self, *args):
+ self.restore_console_title()
+
def fixed_template(self):
"""Checks if the output template is fixed."""
return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4dee487ab..af4c9c5c4 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -603,8 +603,7 @@ def _real_main(argv=None):
u' file! Use "%%(ext)s" instead of %r' %
determine_ext(outtmpl, u''))
- # YoutubeDL
- ydl = YoutubeDL({
+ ydl_opts = {
'usenetrc': opts.usenetrc,
'username': opts.username,
'password': opts.password,
@@ -667,61 +666,63 @@ def _real_main(argv=None):
'youtube_print_sig_code': opts.youtube_print_sig_code,
'age_limit': opts.age_limit,
'download_archive': opts.download_archive,
- })
+ }
- if opts.verbose:
- write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
- try:
- sp = subprocess.Popen(
- ['git', 'rev-parse', '--short', 'HEAD'],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- cwd=os.path.dirname(os.path.abspath(__file__)))
- out, err = sp.communicate()
- out = out.decode().strip()
- if re.match('[0-9a-f]+', out):
- write_string(u'[debug] Git HEAD: ' + out + u'\n')
- except:
+ with YoutubeDL(ydl_opts) as ydl:
+ if opts.verbose:
+ write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
try:
- sys.exc_clear()
+ sp = subprocess.Popen(
+ ['git', 'rev-parse', '--short', 'HEAD'],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ cwd=os.path.dirname(os.path.abspath(__file__)))
+ out, err = sp.communicate()
+ out = out.decode().strip()
+ if re.match('[0-9a-f]+', out):
+ write_string(u'[debug] Git HEAD: ' + out + u'\n')
except:
- pass
- write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
+ try:
+ sys.exc_clear()
+ except:
+ pass
+ write_string(u'[debug] Python version %s - %s' %
+ (platform.python_version(), platform_name()) + u'\n')
- proxy_map = {}
- for handler in opener.handlers:
- if hasattr(handler, 'proxies'):
- proxy_map.update(handler.proxies)
- write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
+ proxy_map = {}
+ for handler in opener.handlers:
+ if hasattr(handler, 'proxies'):
+ proxy_map.update(handler.proxies)
+ write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
- ydl.add_default_info_extractors()
+ ydl.add_default_info_extractors()
- # PostProcessors
- # Add the metadata pp first, the other pps will copy it
- if opts.addmetadata:
- ydl.add_post_processor(FFmpegMetadataPP())
- if opts.extractaudio:
- ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
- if opts.recodevideo:
- ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
- if opts.embedsubtitles:
- ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
+ # PostProcessors
+ # Add the metadata pp first, the other pps will copy it
+ if opts.addmetadata:
+ ydl.add_post_processor(FFmpegMetadataPP())
+ if opts.extractaudio:
+ ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
+ if opts.recodevideo:
+ ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
+ if opts.embedsubtitles:
+ ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
- # Update version
- if opts.update_self:
- update_self(ydl.to_screen, opts.verbose)
+ # Update version
+ if opts.update_self:
+ update_self(ydl.to_screen, opts.verbose)
- # Maybe do nothing
- if len(all_urls) < 1:
- if not opts.update_self:
- parser.error(u'you must provide at least one URL')
- else:
- sys.exit()
+ # Maybe do nothing
+ if len(all_urls) < 1:
+ if not opts.update_self:
+ parser.error(u'you must provide at least one URL')
+ else:
+ sys.exit()
- try:
- retcode = ydl.download(all_urls)
- except MaxDownloadsReached:
- ydl.to_screen(u'--max-download limit reached, aborting.')
- retcode = 101
+ try:
+ retcode = ydl.download(all_urls)
+ except MaxDownloadsReached:
+ ydl.to_screen(u'--max-download limit reached, aborting.')
+ retcode = 101
# Dump cookie jar if requested
if opts.cookiefile is not None:
From 749febf4d1513328ca8b9b6c16969d8c70ff4555 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 21:12:50 +0100
Subject: [PATCH 130/132] Allow --console-title when --quiet is given (Fixes
#1783)
---
youtube_dl/YoutubeDL.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4e28f9120..20eed96ca 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -211,19 +211,19 @@ class YoutubeDL(object):
# already of type unicode()
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
elif 'TERM' in os.environ:
- self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+ write_string(u'\033]0;%s\007' % message, self._screen_file)
def save_console_title(self):
if not self.params.get('consoletitle', False):
return
if 'TERM' in os.environ:
- self.to_screen('\033[22t')
+ write_string(u'\033[22t', self._screen_file)
def restore_console_title(self):
if not self.params.get('consoletitle', False):
return
if 'TERM' in os.environ:
- self.to_screen('\033[23t')
+ write_string(u'\033[23t', self._screen_file)
def __enter__(self):
self.save_console_title()
From 63b7b7224a9cbb33856d3e1f8755ba4bdd3a9d58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 22:11:39 +0100
Subject: [PATCH 131/132] [MTVIE] Try with RTMP URL if download fails
This fixes youtube-dl http://www.southpark.de/clips/155251/cartman-vs-the-dog-whisperer
---
youtube_dl/extractor/mtv.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 24a79ae13..3df7f9b85 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -48,7 +48,7 @@ class MTVIE(InfoExtractor):
def _transform_rtmp_url(rtmp_video_url):
m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
if not m:
- raise ExtractorError(u'Cannot transform RTMP url')
+ return rtmp_video_url
base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
return base + m.group('finalid')
From 73c566695fac926e7e9e6922fe4e6d82c64a1850 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 22:14:13 +0100
Subject: [PATCH 132/132] release 2013.11.17
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b04238eb5..110058c79 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
-__version__ = '2013.11.15.1'
+__version__ = '2013.11.17'