Merge 1b92d6fab7 into e3b42d8b1b

Closes #9720
Authored by: bashonly

.github/workflows/build.yml

@@ -254,7 +254,7 @@ jobs:
           # We need to fuse our own universal2 wheels for curl_cffi
           python3 -m pip install -U --user delocate
           mkdir curl_cffi_whls curl_cffi_universal2
-          python3 devscripts/install_deps.py --print -o --include curl_cffi > requirements.txt
+          python3 devscripts/install_deps.py --print -o --include curl-cffi > requirements.txt
           for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
             python3 -m pip download \
               --only-binary=:all: \
@@ -362,7 +362,7 @@ jobs:
       - name: Install Requirements
         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
           python devscripts/install_deps.py -o --include build
-          python devscripts/install_deps.py --include py2exe --include curl_cffi
+          python devscripts/install_deps.py --include py2exe --include curl-cffi
           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
 
       - name: Prepare

README.md

@@ -202,7 +202,7 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
 The following provide support for impersonating browser requests. This may be required for some sites that employ TLS fingerprinting. 
 
 * [**curl_cffi**](https://github.com/yifeikong/curl_cffi) (recommended) - Python binding for [curl-impersonate](https://github.com/lwthiker/curl-impersonate). Provides impersonation targets for Chrome, Edge and Safari. Licensed under [MIT](https://github.com/yifeikong/curl_cffi/blob/main/LICENSE)
-  * Can be installed with the `curl_cffi` group, e.g. `pip install yt-dlp[default,curl_cffi]`
+  * Can be installed with the `curl-cffi` group, e.g. `pip install yt-dlp[default,curl-cffi]`
   * Currently only included in `yt-dlp.exe` and `yt-dlp_macos` builds
 
 

pyproject.toml

@@ -53,7 +53,7 @@ dependencies = [
 
 [project.optional-dependencies]
 default = []
-curl_cffi = ["curl-cffi==0.5.10; implementation_name=='cpython'"]
+curl-cffi = ["curl-cffi==0.5.10; implementation_name=='cpython'"]
 secretstorage = [
     "cffi",
     "secretstorage",

yt_dlp/extractor/_extractors.py

@@ -820,6 +820,7 @@ from .instagram import (
     InstagramUserIE,
     InstagramTagIE,
     InstagramStoryIE,
+    InstagramLiveIE,
 )
 from .internazionale import InternazionaleIE
 from .internetvideoarchive import InternetVideoArchiveIE

yt_dlp/extractor/facebook.py

@@ -560,7 +560,7 @@ class FacebookIE(InfoExtractor):
                     js_data, lambda x: x['jsmods']['instances'], list) or [])
 
         def extract_dash_manifest(video, formats):
-            dash_manifest = video.get('dash_manifest')
+            dash_manifest = traverse_obj(video, 'dash_manifest', 'playlist', expected_type=str)
             if dash_manifest:
                 formats.extend(self._parse_mpd_formats(
                     compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest)),

yt_dlp/extractor/instagram.py

@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
+    UserNotLive,
     decode_base_n,
     encode_base_n,
     filter_dict,
@@ -733,3 +734,59 @@ class InstagramStoryIE(InstagramBaseIE):
                     **filter_dict(highlight_data),
                 })
         return self.playlist_result(info_data, playlist_id=story_id, playlist_title=story_title)
+
+
+class InstagramLiveIE(InstagramBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<user>[^/]+)/live(?:/|/(?P<id>\d+))?'
+    IE_NAME = 'instagram:live'
+
+    _TESTS = [{
+        'url': 'https://www.instagram.com/answer/live/42',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        username, video_id = self._match_valid_url(url).groups()
+        video_id = video_id or username
+
+        metadata_str = self._html_search_regex(
+            r'data-sjs>({.*?ScheduledServerJS.*?target_user_id.*?})</script>',
+            self._download_webpage(url, video_id), 'live metadata', default=None)
+        if not metadata_str:
+            raise UserNotLive(video_id=video_id)
+
+        metadata = self._parse_json(metadata_str, video_id)
+
+        target_user_id = traverse_obj(
+            metadata,
+            ('require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., 'rootView', 'props', 'target_user_id', {str}),
+            ('require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., 'hostableView', 'props', 'target_user_id', {str}),
+            get_all=False)
+        if not target_user_id:
+            raise ExtractorError('Missing "target_user_id" in metadata')
+
+        app_id = traverse_obj(
+            metadata,
+            ('require', ..., ..., ..., '__bbox', 'define', ..., ..., 'customHeaders', 'X-IG-App-ID', {str}),
+            ('require', ..., ..., ..., '__bbox', 'define', ..., ..., 'appId', {int}),
+            ('require', ..., ..., ..., '__bbox', 'define', ..., ..., 'APP_ID', {str}),
+            ('require', ..., ..., ..., '__bbox', 'define', ..., ..., 'app_id', {str}),
+            get_all=False)
+        if not app_id:
+            raise ExtractorError('Missing "app_id" in metadata')
+
+        web_info_json = self._download_json(
+            'https://www.instagram.com/api/v1/live/web_info/', video_id,
+            query={'target_user_id': target_user_id}, headers={'x-ig-app-id': app_id},
+            note='Downloading web_info', errnote='Unable to download web_info')
+
+        video_id = traverse_obj(web_info_json, ('id'))
+
+        return {
+            'id': video_id,
+            'formats': self._extract_mpd_formats(
+                traverse_obj(web_info_json, ('dash_abr_playback_url', {url_or_none})), video_id),
+            'uploader': traverse_obj(web_info_json, ('broadcast_owner', 'full_name')),
+            'uploader_id': username,
+            'live_status': 'is_live',
+        }

yt_dlp/extractor/patreon.py

@@ -1,8 +1,8 @@
 import itertools
+import urllib.parse
 
 from .common import InfoExtractor
 from .vimeo import VimeoIE
-from ..compat import compat_urllib_parse_unquote
 from ..networking.exceptions import HTTPError
 from ..utils import (
     KNOWN_EXTENSIONS,
@@ -14,7 +14,6 @@ from ..utils import (
     parse_iso8601,
     str_or_none,
     traverse_obj,
-    try_get,
     url_or_none,
     urljoin,
 )
@@ -199,6 +198,27 @@ class PatreonIE(PatreonBaseIE):
             'channel_id': '2147162',
             'uploader_url': 'https://www.patreon.com/yaboyroshi',
         },
+    }, {
+        # NSFW vimeo embed URL
+        'url': 'https://www.patreon.com/posts/4k-spiderman-4k-96414599',
+        'info_dict': {
+            'id': '902250943',
+            'ext': 'mp4',
+            'title': '❤️(4K) Spiderman Girl Yeonhwa’s Gift ❤️(4K) 스파이더맨걸 연화의 선물',
+            'description': '❤️(4K) Spiderman Girl Yeonhwa’s Gift \n❤️(4K) 스파이더맨걸 연화의 선물',
+            'uploader': 'Npickyeonhwa',
+            'uploader_id': '90574422',
+            'uploader_url': 'https://www.patreon.com/Yeonhwa726',
+            'channel_id': '10237902',
+            'channel_url': 'https://www.patreon.com/Yeonhwa726',
+            'duration': 70,
+            'timestamp': 1705150153,
+            'upload_date': '20240113',
+            'comment_count': int,
+            'like_count': int,
+            'thumbnail': r're:^https?://.+',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
@@ -268,16 +288,19 @@ class PatreonIE(PatreonBaseIE):
                 })
 
         # handle Vimeo embeds
-        if try_get(attributes, lambda x: x['embed']['provider']) == 'Vimeo':
-            embed_html = try_get(attributes, lambda x: x['embed']['html'])
-            v_url = url_or_none(compat_urllib_parse_unquote(
-                self._search_regex(r'(https(?:%3A%2F%2F|://)player\.vimeo\.com.+app_id(?:=|%3D)+\d+)', embed_html, 'vimeo url', fatal=False)))
-            if v_url:
-                v_url = VimeoIE._smuggle_referrer(v_url, 'https://patreon.com')
-                if self._request_webpage(v_url, video_id, 'Checking Vimeo embed URL', fatal=False, errnote=False):
-                    return self.url_result(v_url, VimeoIE, url_transparent=True, **info)
+        if traverse_obj(attributes, ('embed', 'provider')) == 'Vimeo':
+            v_url = urllib.parse.unquote(self._html_search_regex(
+                r'(https(?:%3A%2F%2F|://)player\.vimeo\.com.+app_id(?:=|%3D)+\d+)',
+                traverse_obj(attributes, ('embed', 'html', {str})), 'vimeo url', fatal=False) or '')
+            if url_or_none(v_url) and self._request_webpage(
+                    v_url, video_id, 'Checking Vimeo embed URL',
+                    headers={'Referer': 'https://patreon.com/'},
+                    fatal=False, errnote=False):
+                return self.url_result(
+                    VimeoIE._smuggle_referrer(v_url, 'https://patreon.com/'),
+                    VimeoIE, url_transparent=True, **info)
 
-        embed_url = try_get(attributes, lambda x: x['embed']['url'])
+        embed_url = traverse_obj(attributes, ('embed', 'url', {url_or_none}))
         if embed_url and self._request_webpage(embed_url, video_id, 'Checking embed URL', fatal=False, errnote=False):
             return self.url_result(embed_url, **info)