test/test_aes.py

@@ -8,7 +8,7 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text, aes_ecb_encrypt
+from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text
 from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
 import base64
 
@@ -58,13 +58,6 @@ class TestAES(unittest.TestCase):
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 
-    def test_ecb_encrypt(self):
-        data = bytes_to_intlist(self.secret_msg)
-        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key))
-        self.assertEqual(
-            encrypted,
-            b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
-
 
 if __name__ == '__main__':
     unittest.main()

youtube_dl/aes.py

@@ -8,18 +8,6 @@ from .utils import bytes_to_intlist, intlist_to_bytes
 BLOCK_SIZE_BYTES = 16
 
 
-def pkcs7_padding(data):
-    """
-    PKCS#7 padding
-
-    @param {int[]} data        cleartext
-    @returns {int[]}           padding data
-    """
-
-    remaining_length = BLOCK_SIZE_BYTES - len(data) % BLOCK_SIZE_BYTES
-    return data + [remaining_length] * remaining_length
-
-
 def aes_ctr_decrypt(data, key, counter):
     """
     Decrypt with aes in counter mode
@@ -88,7 +76,8 @@ def aes_cbc_encrypt(data, key, iv):
     previous_cipher_block = iv
     for i in range(block_count):
         block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
-        block = pkcs7_padding(block)
+        remaining_length = BLOCK_SIZE_BYTES - len(block)
+        block += [remaining_length] * remaining_length
         mixed_block = xor(block, previous_cipher_block)
 
         encrypted_block = aes_encrypt(mixed_block, expanded_key)
@@ -99,28 +88,6 @@ def aes_cbc_encrypt(data, key, iv):
     return encrypted_data
 
 
-def aes_ecb_encrypt(data, key):
-    """
-    Encrypt with aes in ECB mode. Using PKCS#7 padding
-
-    @param {int[]} data        cleartext
-    @param {int[]} key         16/24/32-Byte cipher key
-    @returns {int[]}           encrypted data
-    """
-    expanded_key = key_expansion(key)
-    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
-
-    encrypted_data = []
-    for i in range(block_count):
-        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
-        block = pkcs7_padding(block)
-
-        encrypted_block = aes_encrypt(block, expanded_key)
-        encrypted_data += encrypted_block
-
-    return encrypted_data
-
-
 def key_expansion(data):
     """
     Generate key schedule

youtube_dl/extractor/common.py

@@ -2844,7 +2844,7 @@ class InfoExtractor(object):
                     'url': source_url,
                     'width': int_or_none(source.get('width')),
                     'height': height,
-                    'tbr': int_or_none(source.get('bitrate'), scale=1000),
+                    'tbr': int_or_none(source.get('bitrate')),
                     'ext': ext,
                 }
                 if source_url.startswith('rtmp'):

youtube_dl/extractor/neteasemusic.py

@@ -1,31 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from base64 import b64encode
-from binascii import hexlify
-from datetime import datetime
 from hashlib import md5
-from random import randint
-import json
+from base64 import b64encode
+from datetime import datetime
 import re
-import time
 
 from .common import InfoExtractor
-from ..aes import aes_ecb_encrypt, pkcs7_padding
 from ..compat import (
     compat_urllib_parse_urlencode,
     compat_str,
     compat_itertools_count,
 )
 from ..utils import (
-    ExtractorError,
-    bytes_to_intlist,
-    float_or_none,
-    int_or_none,
-    intlist_to_bytes,
     sanitized_Request,
-    std_headers,
-    try_get,
+    float_or_none,
 )
 
 
@@ -46,85 +35,32 @@ class NetEaseMusicBaseIE(InfoExtractor):
         result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
-    @classmethod
-    def make_player_api_request_data_and_headers(cls, song_id, bitrate):
-        KEY = b'e82ckenh8dichen8'
-        URL = '/api/song/enhance/player/url'
-        now = int(time.time() * 1000)
-        rand = randint(0, 1000)
-        cookie = {
-            'osver': None,
-            'deviceId': None,
-            'appver': '8.0.0',
-            'versioncode': '140',
-            'mobilename': None,
-            'buildver': '1623435496',
-            'resolution': '1920x1080',
-            '__csrf': '',
-            'os': 'pc',
-            'channel': None,
-            'requestId': '{0}_{1:04}'.format(now, rand),
-        }
-        request_text = json.dumps(
-            {'ids': '[{0}]'.format(song_id), 'br': bitrate, 'header': cookie},
-            separators=(',', ':'))
-        message = 'nobody{0}use{1}md5forencrypt'.format(
-            URL, request_text).encode('latin1')
-        msg_digest = md5(message).hexdigest()
-
-        data = '{0}-36cd479b6b5-{1}-36cd479b6b5-{2}'.format(
-            URL, request_text, msg_digest)
-        data = pkcs7_padding(bytes_to_intlist(data))
-        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, bytes_to_intlist(KEY)))
-        encrypted_params = hexlify(encrypted).decode('ascii').upper()
-
-        cookie = '; '.join(
-            ['{0}={1}'.format(k, v if v is not None else 'undefined')
-             for [k, v] in cookie.items()])
-
-        headers = {
-            'User-Agent': std_headers['User-Agent'],
-            'Content-Type': 'application/x-www-form-urlencoded',
-            'Referer': 'https://music.163.com',
-            'Cookie': cookie,
-        }
-        return ('params={0}'.format(encrypted_params), headers)
-
-    def _call_player_api(self, song_id, bitrate):
-        url = 'https://interface3.music.163.com/eapi/song/enhance/player/url'
-        data, headers = self.make_player_api_request_data_and_headers(song_id, bitrate)
-        try:
-            return self._download_json(
-                url, song_id, data=data.encode('ascii'), headers=headers)
-        except ExtractorError as e:
-            if type(e.cause) in (ValueError, TypeError):
-                # JSON load failure
-                raise
-        except Exception:
-            pass
-        return {}
-
     def extract_formats(self, info):
         formats = []
-        song_id = info['id']
         for song_format in self._FORMATS:
             details = info.get(song_format)
             if not details:
                 continue
+            song_file_path = '/%s/%s.%s' % (
+                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])
 
-            bitrate = int_or_none(details.get('bitrate')) or 999000
-            data = self._call_player_api(song_id, bitrate)
-            for song in try_get(data, lambda x: x['data'], list) or []:
-                song_url = try_get(song, lambda x: x['url'])
+            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
+            # from NetEase's CDN provider that can be used if m5.music.126.net does not
+            # work, especially for users outside of Mainland China
+            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
+            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
+                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
+                song_url = host + song_file_path
                 if self._is_valid_url(song_url, info['id'], 'song'):
                     formats.append({
                         'url': song_url,
                         'ext': details.get('extension'),
-                        'abr': float_or_none(song.get('br'), scale=1000),
+                        'abr': float_or_none(details.get('bitrate'), scale=1000),
                         'format_id': song_format,
-                        'filesize': int_or_none(song.get('size')),
-                        'asr': int_or_none(details.get('sr')),
+                        'filesize': details.get('size'),
+                        'asr': details.get('sr')
                     })
+                    break
         return formats
 
     @classmethod
@@ -143,16 +79,30 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',
-        'md5': '3e909614ce09b1ccef4a3eb205441190',
+        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
         'info_dict': {
             'id': '32102397',
             'ext': 'mp3',
-            'title': 'Bad Blood',
+            'title': 'Bad Blood (feat. Kendrick Lamar)',
             'creator': 'Taylor Swift / Kendrick Lamar',
-            'upload_date': '20150516',
-            'timestamp': 1431792000,
-            'description': 'md5:25fc5f27e47aad975aa6d36382c7833c',
+            'upload_date': '20150517',
+            'timestamp': 1431878400,
+            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
         },
+        'skip': 'Blocked outside Mainland China',
+    }, {
+        'note': 'No lyrics translation.',
+        'url': 'http://music.163.com/#/song?id=29822014',
+        'info_dict': {
+            'id': '29822014',
+            'ext': 'mp3',
+            'title': '听见下雨的声音',
+            'creator': '周杰伦',
+            'upload_date': '20141225',
+            'timestamp': 1419523200,
+            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
+        },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'No lyrics.',
         'url': 'http://music.163.com/song?id=17241424',
@@ -162,9 +112,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': 'Opus 28',
             'creator': 'Dustin O\'Halloran',
             'upload_date': '20080211',
-            'description': 'md5:f12945b0f6e0365e3b73c5032e1b0ff4',
             'timestamp': 1202745600,
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Has translated name.',
         'url': 'http://music.163.com/#/song?id=22735043',
@@ -178,6 +128,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1264608000,
             'alt_title': '说出愿望吧(Genie)',
         },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _process_lyrics(self, lyrics_info):

youtube_dl/utils.py

@@ -33,7 +33,6 @@ import sys
 import tempfile
 import time
 import traceback
-import unicodedata
 import xml.etree.ElementTree
 import zlib
 
@@ -2119,9 +2118,6 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
-    # Replace look-alike Unicode glyphs
-    if restricted and not is_id:
-        s = unicodedata.normalize('NFKC', s)
     # Handle timestamps
     s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
     result = ''.join(map(replace_insane, s))