[jamendo] fix track extraction(closes #28686)

youtube_dl/extractor/curiositystream.py

@@ -25,12 +25,12 @@ class CuriosityStreamBaseIE(InfoExtractor):
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, error), expected=True)
 
-    def _call_api(self, path, video_id):
+    def _call_api(self, path, video_id, query=None):
         headers = {}
         if self._auth_token:
             headers['X-Auth-Token'] = self._auth_token
         result = self._download_json(
-            self._API_BASE_URL + path, video_id, headers=headers)
+            self._API_BASE_URL + path, video_id, headers=headers, query=query)
         self._handle_errors(result)
         return result['data']
 
@@ -52,62 +52,75 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://app.curiositystream.com/video/2',
-        'md5': '262bb2f257ff301115f1973540de8983',
         'info_dict': {
             'id': '2',
             'ext': 'mp4',
             'title': 'How Did You Develop The Internet?',
             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
-        }
+        },
+        'params': {
+            'format': 'bestvideo',
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        media = self._call_api('media/' + video_id, video_id)
-        title = media['title']
 
         formats = []
-        for encoding in media.get('encodings', []):
+        for encoding_format in ('m3u8', 'mpd'):
-            m3u8_url = encoding.get('master_playlist_url')
+            media = self._call_api('media/' + video_id, video_id, query={
-            if m3u8_url:
+                'encodingsNew': 'true',
-                formats.extend(self._extract_m3u8_formats(
+                'encodingsFormat': encoding_format,
-                    m3u8_url, video_id, 'mp4', 'm3u8_native',
+            })
-                    m3u8_id='hls', fatal=False))
+            for encoding in media.get('encodings', []):
-            encoding_url = encoding.get('url')
+                playlist_url = encoding.get('master_playlist_url')
-            file_url = encoding.get('file_url')
+                if encoding_format == 'm3u8':
-            if not encoding_url and not file_url:
+                    # use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
-                continue
+                    formats.extend(self._extract_m3u8_formats(
-            f = {
+                        playlist_url, video_id, 'mp4',
-                'width': int_or_none(encoding.get('width')),
+                        m3u8_id='hls', fatal=False))
-                'height': int_or_none(encoding.get('height')),
+                elif encoding_format == 'mpd':
-                'vbr': int_or_none(encoding.get('video_bitrate')),
+                    formats.extend(self._extract_mpd_formats(
-                'abr': int_or_none(encoding.get('audio_bitrate')),
+                        playlist_url, video_id, mpd_id='dash', fatal=False))
-                'filesize': int_or_none(encoding.get('size_in_bytes')),
+                encoding_url = encoding.get('url')
-                'vcodec': encoding.get('video_codec'),
+                file_url = encoding.get('file_url')
-                'acodec': encoding.get('audio_codec'),
+                if not encoding_url and not file_url:
-                'container': encoding.get('container_type'),
-            }
-            for f_url in (encoding_url, file_url):
-                if not f_url:
                     continue
-                fmt = f.copy()
+                f = {
-                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
+                    'width': int_or_none(encoding.get('width')),
-                if rtmp:
+                    'height': int_or_none(encoding.get('height')),
-                    fmt.update({
+                    'vbr': int_or_none(encoding.get('video_bitrate')),
-                        'url': rtmp.group('url'),
+                    'abr': int_or_none(encoding.get('audio_bitrate')),
-                        'play_path': rtmp.group('playpath'),
+                    'filesize': int_or_none(encoding.get('size_in_bytes')),
-                        'app': rtmp.group('app'),
+                    'vcodec': encoding.get('video_codec'),
-                        'ext': 'flv',
+                    'acodec': encoding.get('audio_codec'),
-                        'format_id': 'rtmp',
+                    'container': encoding.get('container_type'),
-                    })
+                }
-                else:
+                for f_url in (encoding_url, file_url):
-                    fmt.update({
+                    if not f_url:
-                        'url': f_url,
+                        continue
-                        'format_id': 'http',
+                    fmt = f.copy()
-                    })
+                    rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
-                formats.append(fmt)
+                    if rtmp:
+                        fmt.update({
+                            'url': rtmp.group('url'),
+                            'play_path': rtmp.group('playpath'),
+                            'app': rtmp.group('app'),
+                            'ext': 'flv',
+                            'format_id': 'rtmp',
+                        })
+                    else:
+                        fmt.update({
+                            'url': f_url,
+                            'format_id': 'http',
+                        })
+                    formats.append(fmt)
         self._sort_formats(formats)
 
+        title = media['title']
+
         subtitles = {}
         for closed_caption in media.get('closed_captions', []):
             sub_url = closed_caption.get('file')
@@ -140,7 +153,7 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
             'title': 'Curious Minds: The Internet',
             'description': 'How is the internet shaping our lives in the 21st Century?',
         },
-        'playlist_mincount': 17,
+        'playlist_mincount': 16,
     }, {
         'url': 'https://curiositystream.com/series/2',
         'only_matching': True,

youtube_dl/extractor/jamendo.py

@@ -29,34 +29,51 @@ class JamendoIE(InfoExtractor):
             'id': '196219',
             'display_id': 'stories-from-emona-i',
             'ext': 'flac',
-            'title': 'Maya Filipič - Stories from Emona I',
+            # 'title': 'Maya Filipič - Stories from Emona I',
-            'artist': 'Maya Filipič',
+            'title': 'Stories from Emona I',
+            # 'artist': 'Maya Filipič',
             'track': 'Stories from Emona I',
             'duration': 210,
             'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1217438117,
             'upload_date': '20080730',
+            'license': 'by-nc-nd',
+            'view_count': int,
+            'like_count': int,
+            'average_rating': int,
+            'tags': ['piano', 'peaceful', 'newage', 'strings', 'upbeat'],
         }
     }, {
         'url': 'https://licensing.jamendo.com/en/track/1496667/energetic-rock',
         'only_matching': True,
     }]
 
+    def _call_api(self, resource, resource_id):
+        path = '/api/%ss' % resource
+        rand = compat_str(random.random())
+        return self._download_json(
+            'https://www.jamendo.com' + path, resource_id, query={
+                'id[]': resource_id,
+            }, headers={
+                'X-Jam-Call': '$%s*%s~' % (hashlib.sha1((path + rand).encode()).hexdigest(), rand)
+            })[0]
+
     def _real_extract(self, url):
         track_id, display_id = self._VALID_URL_RE.match(url).groups()
-        webpage = self._download_webpage(
+        # webpage = self._download_webpage(
-            'https://www.jamendo.com/track/' + track_id, track_id)
+        #     'https://www.jamendo.com/track/' + track_id, track_id)
-        models = self._parse_json(self._html_search_regex(
+        # models = self._parse_json(self._html_search_regex(
-            r"data-bundled-models='([^']+)",
+        #     r"data-bundled-models='([^']+)",
-            webpage, 'bundled models'), track_id)
+        #     webpage, 'bundled models'), track_id)
-        track = models['track']['models'][0]
+        # track = models['track']['models'][0]
+        track = self._call_api('track', track_id)
         title = track_name = track['name']
-        get_model = lambda x: try_get(models, lambda y: y[x]['models'][0], dict) or {}
+        # get_model = lambda x: try_get(models, lambda y: y[x]['models'][0], dict) or {}
-        artist = get_model('artist')
+        # artist = get_model('artist')
-        artist_name = artist.get('name')
+        # artist_name = artist.get('name')
-        if artist_name:
+        # if artist_name:
-            title = '%s - %s' % (artist_name, title)
+        #     title = '%s - %s' % (artist_name, title)
-        album = get_model('album')
+        # album = get_model('album')
 
         formats = [{
             'url': 'https://%s.jamendo.com/?trackid=%s&format=%s&from=app-97dab294'
@@ -74,7 +91,7 @@ class JamendoIE(InfoExtractor):
 
         urls = []
         thumbnails = []
-        for _, covers in track.get('cover', {}).items():
+        for covers in (track.get('cover') or {}).values():
             for cover_id, cover_url in covers.items():
                 if not cover_url or cover_url in urls:
                     continue
@@ -88,13 +105,14 @@ class JamendoIE(InfoExtractor):
                 })
 
         tags = []
-        for tag in track.get('tags', []):
+        for tag in (track.get('tags') or []):
             tag_name = tag.get('name')
             if not tag_name:
                 continue
             tags.append(tag_name)
 
         stats = track.get('stats') or {}
+        license = track.get('licenseCC') or []
 
         return {
             'id': track_id,
@@ -103,11 +121,11 @@ class JamendoIE(InfoExtractor):
             'title': title,
             'description': track.get('description'),
             'duration': int_or_none(track.get('duration')),
-            'artist': artist_name,
+            # 'artist': artist_name,
             'track': track_name,
-            'album': album.get('name'),
+            # 'album': album.get('name'),
             'formats': formats,
-            'license': '-'.join(track.get('licenseCC', [])) or None,
+            'license': '-'.join(license) if license else None,
             'timestamp': int_or_none(track.get('dateCreated')),
             'view_count': int_or_none(stats.get('listenedAll')),
             'like_count': int_or_none(stats.get('favorited')),
@@ -116,9 +134,9 @@ class JamendoIE(InfoExtractor):
         }
 
 
-class JamendoAlbumIE(InfoExtractor):
+class JamendoAlbumIE(JamendoIE):
     _VALID_URL = r'https?://(?:www\.)?jamendo\.com/album/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.jamendo.com/album/121486/duck-on-cover',
         'info_dict': {
             'id': '121486',
@@ -151,17 +169,7 @@ class JamendoAlbumIE(InfoExtractor):
         'params': {
             'playlistend': 2
         }
-    }
+    }]
-
-    def _call_api(self, resource, resource_id):
-        path = '/api/%ss' % resource
-        rand = compat_str(random.random())
-        return self._download_json(
-            'https://www.jamendo.com' + path, resource_id, query={
-                'id[]': resource_id,
-            }, headers={
-                'X-Jam-Call': '$%s*%s~' % (hashlib.sha1((path + rand).encode()).hexdigest(), rand)
-            })[0]
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
@@ -169,7 +177,7 @@ class JamendoAlbumIE(InfoExtractor):
         album_name = album.get('name')
 
         entries = []
-        for track in album.get('tracks', []):
+        for track in (album.get('tracks') or []):
             track_id = track.get('id')
             if not track_id:
                 continue