test_YoutubeDL.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924
  1. #!/usr/bin/env python
  2. # coding: utf-8
  3. from __future__ import unicode_literals
  4. # Allow direct execution
  5. import os
  6. import sys
  7. import unittest
  8. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  9. import copy
  10. from test.helper import FakeYDL, assertRegexpMatches
  11. from youtube_dl import YoutubeDL
  12. from youtube_dl.compat import compat_str, compat_urllib_error
  13. from youtube_dl.extractor import YoutubeIE
  14. from youtube_dl.extractor.common import InfoExtractor
  15. from youtube_dl.postprocessor.common import PostProcessor
  16. from youtube_dl.utils import ExtractorError, match_filter_func
  17. TEST_URL = 'http://localhost/sample.mp4'
  18. class YDL(FakeYDL):
  19. def __init__(self, *args, **kwargs):
  20. super(YDL, self).__init__(*args, **kwargs)
  21. self.downloaded_info_dicts = []
  22. self.msgs = []
  23. def process_info(self, info_dict):
  24. self.downloaded_info_dicts.append(info_dict)
  25. def to_screen(self, msg):
  26. self.msgs.append(msg)
  27. def _make_result(formats, **kwargs):
  28. res = {
  29. 'formats': formats,
  30. 'id': 'testid',
  31. 'title': 'testttitle',
  32. 'extractor': 'testex',
  33. 'extractor_key': 'TestEx',
  34. }
  35. res.update(**kwargs)
  36. return res
  37. class TestFormatSelection(unittest.TestCase):
  38. def test_prefer_free_formats(self):
  39. # Same resolution => download webm
  40. ydl = YDL()
  41. ydl.params['prefer_free_formats'] = True
  42. formats = [
  43. {'ext': 'webm', 'height': 460, 'url': TEST_URL},
  44. {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
  45. ]
  46. info_dict = _make_result(formats)
  47. yie = YoutubeIE(ydl)
  48. yie._sort_formats(info_dict['formats'])
  49. ydl.process_ie_result(info_dict)
  50. downloaded = ydl.downloaded_info_dicts[0]
  51. self.assertEqual(downloaded['ext'], 'webm')
  52. # Different resolution => download best quality (mp4)
  53. ydl = YDL()
  54. ydl.params['prefer_free_formats'] = True
  55. formats = [
  56. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  57. {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
  58. ]
  59. info_dict['formats'] = formats
  60. yie = YoutubeIE(ydl)
  61. yie._sort_formats(info_dict['formats'])
  62. ydl.process_ie_result(info_dict)
  63. downloaded = ydl.downloaded_info_dicts[0]
  64. self.assertEqual(downloaded['ext'], 'mp4')
  65. # No prefer_free_formats => prefer mp4 and flv for greater compatibility
  66. ydl = YDL()
  67. ydl.params['prefer_free_formats'] = False
  68. formats = [
  69. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  70. {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
  71. {'ext': 'flv', 'height': 720, 'url': TEST_URL},
  72. ]
  73. info_dict['formats'] = formats
  74. yie = YoutubeIE(ydl)
  75. yie._sort_formats(info_dict['formats'])
  76. ydl.process_ie_result(info_dict)
  77. downloaded = ydl.downloaded_info_dicts[0]
  78. self.assertEqual(downloaded['ext'], 'mp4')
  79. ydl = YDL()
  80. ydl.params['prefer_free_formats'] = False
  81. formats = [
  82. {'ext': 'flv', 'height': 720, 'url': TEST_URL},
  83. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  84. ]
  85. info_dict['formats'] = formats
  86. yie = YoutubeIE(ydl)
  87. yie._sort_formats(info_dict['formats'])
  88. ydl.process_ie_result(info_dict)
  89. downloaded = ydl.downloaded_info_dicts[0]
  90. self.assertEqual(downloaded['ext'], 'flv')
  91. def test_format_selection(self):
  92. formats = [
  93. {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
  94. {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
  95. {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
  96. {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
  97. {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
  98. ]
  99. info_dict = _make_result(formats)
  100. ydl = YDL({'format': '20/47'})
  101. ydl.process_ie_result(info_dict.copy())
  102. downloaded = ydl.downloaded_info_dicts[0]
  103. self.assertEqual(downloaded['format_id'], '47')
  104. ydl = YDL({'format': '20/71/worst'})
  105. ydl.process_ie_result(info_dict.copy())
  106. downloaded = ydl.downloaded_info_dicts[0]
  107. self.assertEqual(downloaded['format_id'], '35')
  108. ydl = YDL()
  109. ydl.process_ie_result(info_dict.copy())
  110. downloaded = ydl.downloaded_info_dicts[0]
  111. self.assertEqual(downloaded['format_id'], '2')
  112. ydl = YDL({'format': 'webm/mp4'})
  113. ydl.process_ie_result(info_dict.copy())
  114. downloaded = ydl.downloaded_info_dicts[0]
  115. self.assertEqual(downloaded['format_id'], '47')
  116. ydl = YDL({'format': '3gp/40/mp4'})
  117. ydl.process_ie_result(info_dict.copy())
  118. downloaded = ydl.downloaded_info_dicts[0]
  119. self.assertEqual(downloaded['format_id'], '35')
  120. ydl = YDL({'format': 'example-with-dashes'})
  121. ydl.process_ie_result(info_dict.copy())
  122. downloaded = ydl.downloaded_info_dicts[0]
  123. self.assertEqual(downloaded['format_id'], 'example-with-dashes')
  124. def test_format_selection_audio(self):
  125. formats = [
  126. {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
  127. {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
  128. {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
  129. {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
  130. ]
  131. info_dict = _make_result(formats)
  132. ydl = YDL({'format': 'bestaudio'})
  133. ydl.process_ie_result(info_dict.copy())
  134. downloaded = ydl.downloaded_info_dicts[0]
  135. self.assertEqual(downloaded['format_id'], 'audio-high')
  136. ydl = YDL({'format': 'worstaudio'})
  137. ydl.process_ie_result(info_dict.copy())
  138. downloaded = ydl.downloaded_info_dicts[0]
  139. self.assertEqual(downloaded['format_id'], 'audio-low')
  140. formats = [
  141. {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
  142. {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
  143. ]
  144. info_dict = _make_result(formats)
  145. ydl = YDL({'format': 'bestaudio/worstaudio/best'})
  146. ydl.process_ie_result(info_dict.copy())
  147. downloaded = ydl.downloaded_info_dicts[0]
  148. self.assertEqual(downloaded['format_id'], 'vid-high')
  149. def test_format_selection_audio_exts(self):
  150. formats = [
  151. {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  152. {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  153. {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  154. {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
  155. {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
  156. ]
  157. info_dict = _make_result(formats)
  158. ydl = YDL({'format': 'best'})
  159. ie = YoutubeIE(ydl)
  160. ie._sort_formats(info_dict['formats'])
  161. ydl.process_ie_result(copy.deepcopy(info_dict))
  162. downloaded = ydl.downloaded_info_dicts[0]
  163. self.assertEqual(downloaded['format_id'], 'aac-64')
  164. ydl = YDL({'format': 'mp3'})
  165. ie = YoutubeIE(ydl)
  166. ie._sort_formats(info_dict['formats'])
  167. ydl.process_ie_result(copy.deepcopy(info_dict))
  168. downloaded = ydl.downloaded_info_dicts[0]
  169. self.assertEqual(downloaded['format_id'], 'mp3-64')
  170. ydl = YDL({'prefer_free_formats': True})
  171. ie = YoutubeIE(ydl)
  172. ie._sort_formats(info_dict['formats'])
  173. ydl.process_ie_result(copy.deepcopy(info_dict))
  174. downloaded = ydl.downloaded_info_dicts[0]
  175. self.assertEqual(downloaded['format_id'], 'ogg-64')
  176. def test_format_selection_video(self):
  177. formats = [
  178. {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
  179. {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
  180. {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
  181. ]
  182. info_dict = _make_result(formats)
  183. ydl = YDL({'format': 'bestvideo'})
  184. ydl.process_ie_result(info_dict.copy())
  185. downloaded = ydl.downloaded_info_dicts[0]
  186. self.assertEqual(downloaded['format_id'], 'dash-video-high')
  187. ydl = YDL({'format': 'worstvideo'})
  188. ydl.process_ie_result(info_dict.copy())
  189. downloaded = ydl.downloaded_info_dicts[0]
  190. self.assertEqual(downloaded['format_id'], 'dash-video-low')
  191. ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
  192. ydl.process_ie_result(info_dict.copy())
  193. downloaded = ydl.downloaded_info_dicts[0]
  194. self.assertEqual(downloaded['format_id'], 'dash-video-low')
  195. formats = [
  196. {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
  197. ]
  198. info_dict = _make_result(formats)
  199. ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
  200. ydl.process_ie_result(info_dict.copy())
  201. downloaded = ydl.downloaded_info_dicts[0]
  202. self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
  203. def test_format_selection_string_ops(self):
  204. formats = [
  205. {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
  206. {'format_id': 'zxc-cxz', 'ext': 'webm', 'url': TEST_URL},
  207. ]
  208. info_dict = _make_result(formats)
  209. # equals (=)
  210. ydl = YDL({'format': '[format_id=abc-cba]'})
  211. ydl.process_ie_result(info_dict.copy())
  212. downloaded = ydl.downloaded_info_dicts[0]
  213. self.assertEqual(downloaded['format_id'], 'abc-cba')
  214. # does not equal (!=)
  215. ydl = YDL({'format': '[format_id!=abc-cba]'})
  216. ydl.process_ie_result(info_dict.copy())
  217. downloaded = ydl.downloaded_info_dicts[0]
  218. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  219. ydl = YDL({'format': '[format_id!=abc-cba][format_id!=zxc-cxz]'})
  220. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  221. # starts with (^=)
  222. ydl = YDL({'format': '[format_id^=abc]'})
  223. ydl.process_ie_result(info_dict.copy())
  224. downloaded = ydl.downloaded_info_dicts[0]
  225. self.assertEqual(downloaded['format_id'], 'abc-cba')
  226. # does not start with (!^=)
  227. ydl = YDL({'format': '[format_id!^=abc]'})
  228. ydl.process_ie_result(info_dict.copy())
  229. downloaded = ydl.downloaded_info_dicts[0]
  230. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  231. ydl = YDL({'format': '[format_id!^=abc][format_id!^=zxc]'})
  232. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  233. # ends with ($=)
  234. ydl = YDL({'format': '[format_id$=cba]'})
  235. ydl.process_ie_result(info_dict.copy())
  236. downloaded = ydl.downloaded_info_dicts[0]
  237. self.assertEqual(downloaded['format_id'], 'abc-cba')
  238. # does not end with (!$=)
  239. ydl = YDL({'format': '[format_id!$=cba]'})
  240. ydl.process_ie_result(info_dict.copy())
  241. downloaded = ydl.downloaded_info_dicts[0]
  242. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  243. ydl = YDL({'format': '[format_id!$=cba][format_id!$=cxz]'})
  244. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  245. # contains (*=)
  246. ydl = YDL({'format': '[format_id*=bc-cb]'})
  247. ydl.process_ie_result(info_dict.copy())
  248. downloaded = ydl.downloaded_info_dicts[0]
  249. self.assertEqual(downloaded['format_id'], 'abc-cba')
  250. # does not contain (!*=)
  251. ydl = YDL({'format': '[format_id!*=bc-cb]'})
  252. ydl.process_ie_result(info_dict.copy())
  253. downloaded = ydl.downloaded_info_dicts[0]
  254. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  255. ydl = YDL({'format': '[format_id!*=abc][format_id!*=zxc]'})
  256. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  257. ydl = YDL({'format': '[format_id!*=-]'})
  258. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  259. def test_youtube_format_selection(self):
  260. order = [
  261. '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
  262. # Apple HTTP Live Streaming
  263. '96', '95', '94', '93', '92', '132', '151',
  264. # 3D
  265. '85', '84', '102', '83', '101', '82', '100',
  266. # Dash video
  267. '137', '248', '136', '247', '135', '246',
  268. '245', '244', '134', '243', '133', '242', '160',
  269. # Dash audio
  270. '141', '172', '140', '171', '139',
  271. ]
  272. def format_info(f_id):
  273. info = YoutubeIE._formats[f_id].copy()
  274. # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
  275. # and 'vcodec', while in tests such information is incomplete since
  276. # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
  277. # test_YoutubeDL.test_youtube_format_selection is broken without
  278. # this fix
  279. if 'acodec' in info and 'vcodec' not in info:
  280. info['vcodec'] = 'none'
  281. elif 'vcodec' in info and 'acodec' not in info:
  282. info['acodec'] = 'none'
  283. info['format_id'] = f_id
  284. info['url'] = 'url:' + f_id
  285. return info
  286. formats_order = [format_info(f_id) for f_id in order]
  287. info_dict = _make_result(list(formats_order), extractor='youtube')
  288. ydl = YDL({'format': 'bestvideo+bestaudio'})
  289. yie = YoutubeIE(ydl)
  290. yie._sort_formats(info_dict['formats'])
  291. ydl.process_ie_result(info_dict)
  292. downloaded = ydl.downloaded_info_dicts[0]
  293. self.assertEqual(downloaded['format_id'], '137+141')
  294. self.assertEqual(downloaded['ext'], 'mp4')
  295. info_dict = _make_result(list(formats_order), extractor='youtube')
  296. ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
  297. yie = YoutubeIE(ydl)
  298. yie._sort_formats(info_dict['formats'])
  299. ydl.process_ie_result(info_dict)
  300. downloaded = ydl.downloaded_info_dicts[0]
  301. self.assertEqual(downloaded['format_id'], '38')
  302. info_dict = _make_result(list(formats_order), extractor='youtube')
  303. ydl = YDL({'format': 'bestvideo/best,bestaudio'})
  304. yie = YoutubeIE(ydl)
  305. yie._sort_formats(info_dict['formats'])
  306. ydl.process_ie_result(info_dict)
  307. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  308. self.assertEqual(downloaded_ids, ['137', '141'])
  309. info_dict = _make_result(list(formats_order), extractor='youtube')
  310. ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
  311. yie = YoutubeIE(ydl)
  312. yie._sort_formats(info_dict['formats'])
  313. ydl.process_ie_result(info_dict)
  314. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  315. self.assertEqual(downloaded_ids, ['137+141', '248+141'])
  316. info_dict = _make_result(list(formats_order), extractor='youtube')
  317. ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
  318. yie = YoutubeIE(ydl)
  319. yie._sort_formats(info_dict['formats'])
  320. ydl.process_ie_result(info_dict)
  321. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  322. self.assertEqual(downloaded_ids, ['136+141', '247+141'])
  323. info_dict = _make_result(list(formats_order), extractor='youtube')
  324. ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
  325. yie = YoutubeIE(ydl)
  326. yie._sort_formats(info_dict['formats'])
  327. ydl.process_ie_result(info_dict)
  328. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  329. self.assertEqual(downloaded_ids, ['248+141'])
  330. for f1, f2 in zip(formats_order, formats_order[1:]):
  331. info_dict = _make_result([f1, f2], extractor='youtube')
  332. ydl = YDL({'format': 'best/bestvideo'})
  333. yie = YoutubeIE(ydl)
  334. yie._sort_formats(info_dict['formats'])
  335. ydl.process_ie_result(info_dict)
  336. downloaded = ydl.downloaded_info_dicts[0]
  337. self.assertEqual(downloaded['format_id'], f1['format_id'])
  338. info_dict = _make_result([f2, f1], extractor='youtube')
  339. ydl = YDL({'format': 'best/bestvideo'})
  340. yie = YoutubeIE(ydl)
  341. yie._sort_formats(info_dict['formats'])
  342. ydl.process_ie_result(info_dict)
  343. downloaded = ydl.downloaded_info_dicts[0]
  344. self.assertEqual(downloaded['format_id'], f1['format_id'])
  345. def test_audio_only_extractor_format_selection(self):
  346. # For extractors with incomplete formats (all formats are audio-only or
  347. # video-only) best and worst should fallback to corresponding best/worst
  348. # video-only or audio-only formats (as per
  349. # https://github.com/ytdl-org/youtube-dl/pull/5556)
  350. formats = [
  351. {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
  352. {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
  353. ]
  354. info_dict = _make_result(formats)
  355. ydl = YDL({'format': 'best'})
  356. ydl.process_ie_result(info_dict.copy())
  357. downloaded = ydl.downloaded_info_dicts[0]
  358. self.assertEqual(downloaded['format_id'], 'high')
  359. ydl = YDL({'format': 'worst'})
  360. ydl.process_ie_result(info_dict.copy())
  361. downloaded = ydl.downloaded_info_dicts[0]
  362. self.assertEqual(downloaded['format_id'], 'low')
  363. def test_format_not_available(self):
  364. formats = [
  365. {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
  366. {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
  367. ]
  368. info_dict = _make_result(formats)
  369. # This must fail since complete video-audio format does not match filter
  370. # and extractor does not provide incomplete only formats (i.e. only
  371. # video-only or audio-only).
  372. ydl = YDL({'format': 'best[height>360]'})
  373. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  374. def test_format_selection_issue_10083(self):
  375. # See https://github.com/ytdl-org/youtube-dl/issues/10083
  376. formats = [
  377. {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
  378. {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
  379. {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
  380. ]
  381. info_dict = _make_result(formats)
  382. ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
  383. ydl.process_ie_result(info_dict.copy())
  384. self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
  385. def test_invalid_format_specs(self):
  386. def assert_syntax_error(format_spec):
  387. ydl = YDL({'format': format_spec})
  388. info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
  389. self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
  390. assert_syntax_error('bestvideo,,best')
  391. assert_syntax_error('+bestaudio')
  392. assert_syntax_error('bestvideo+')
  393. assert_syntax_error('/')
  394. def test_format_filtering(self):
  395. formats = [
  396. {'format_id': 'A', 'filesize': 500, 'width': 1000},
  397. {'format_id': 'B', 'filesize': 1000, 'width': 500},
  398. {'format_id': 'C', 'filesize': 1000, 'width': 400},
  399. {'format_id': 'D', 'filesize': 2000, 'width': 600},
  400. {'format_id': 'E', 'filesize': 3000},
  401. {'format_id': 'F'},
  402. {'format_id': 'G', 'filesize': 1000000},
  403. ]
  404. for f in formats:
  405. f['url'] = 'http://_/'
  406. f['ext'] = 'unknown'
  407. info_dict = _make_result(formats)
  408. ydl = YDL({'format': 'best[filesize<3000]'})
  409. ydl.process_ie_result(info_dict)
  410. downloaded = ydl.downloaded_info_dicts[0]
  411. self.assertEqual(downloaded['format_id'], 'D')
  412. ydl = YDL({'format': 'best[filesize<=3000]'})
  413. ydl.process_ie_result(info_dict)
  414. downloaded = ydl.downloaded_info_dicts[0]
  415. self.assertEqual(downloaded['format_id'], 'E')
  416. ydl = YDL({'format': 'best[filesize <= ? 3000]'})
  417. ydl.process_ie_result(info_dict)
  418. downloaded = ydl.downloaded_info_dicts[0]
  419. self.assertEqual(downloaded['format_id'], 'F')
  420. ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
  421. ydl.process_ie_result(info_dict)
  422. downloaded = ydl.downloaded_info_dicts[0]
  423. self.assertEqual(downloaded['format_id'], 'B')
  424. ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
  425. ydl.process_ie_result(info_dict)
  426. downloaded = ydl.downloaded_info_dicts[0]
  427. self.assertEqual(downloaded['format_id'], 'C')
  428. ydl = YDL({'format': '[filesize>?1]'})
  429. ydl.process_ie_result(info_dict)
  430. downloaded = ydl.downloaded_info_dicts[0]
  431. self.assertEqual(downloaded['format_id'], 'G')
  432. ydl = YDL({'format': '[filesize<1M]'})
  433. ydl.process_ie_result(info_dict)
  434. downloaded = ydl.downloaded_info_dicts[0]
  435. self.assertEqual(downloaded['format_id'], 'E')
  436. ydl = YDL({'format': '[filesize<1MiB]'})
  437. ydl.process_ie_result(info_dict)
  438. downloaded = ydl.downloaded_info_dicts[0]
  439. self.assertEqual(downloaded['format_id'], 'G')
  440. ydl = YDL({'format': 'all[width>=400][width<=600]'})
  441. ydl.process_ie_result(info_dict)
  442. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  443. self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
  444. ydl = YDL({'format': 'best[height<40]'})
  445. try:
  446. ydl.process_ie_result(info_dict)
  447. except ExtractorError:
  448. pass
  449. self.assertEqual(ydl.downloaded_info_dicts, [])
  450. def test_default_format_spec(self):
  451. ydl = YDL({'simulate': True})
  452. self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
  453. ydl = YDL({})
  454. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
  455. ydl = YDL({'simulate': True})
  456. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
  457. ydl = YDL({'outtmpl': '-'})
  458. self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
  459. ydl = YDL({})
  460. self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
  461. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
  462. class TestYoutubeDL(unittest.TestCase):
  463. def test_subtitles(self):
  464. def s_formats(lang, autocaption=False):
  465. return [{
  466. 'ext': ext,
  467. 'url': 'http://localhost/video.%s.%s' % (lang, ext),
  468. '_auto': autocaption,
  469. } for ext in ['vtt', 'srt', 'ass']]
  470. subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
  471. auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
  472. info_dict = {
  473. 'id': 'test',
  474. 'title': 'Test',
  475. 'url': 'http://localhost/video.mp4',
  476. 'subtitles': subtitles,
  477. 'automatic_captions': auto_captions,
  478. 'extractor': 'TEST',
  479. }
  480. def get_info(params={}):
  481. params.setdefault('simulate', True)
  482. ydl = YDL(params)
  483. ydl.report_warning = lambda *args, **kargs: None
  484. return ydl.process_video_result(info_dict, download=False)
  485. result = get_info()
  486. self.assertFalse(result.get('requested_subtitles'))
  487. self.assertEqual(result['subtitles'], subtitles)
  488. self.assertEqual(result['automatic_captions'], auto_captions)
  489. result = get_info({'writesubtitles': True})
  490. subs = result['requested_subtitles']
  491. self.assertTrue(subs)
  492. self.assertEqual(set(subs.keys()), set(['en']))
  493. self.assertTrue(subs['en'].get('data') is None)
  494. self.assertEqual(subs['en']['ext'], 'ass')
  495. result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
  496. subs = result['requested_subtitles']
  497. self.assertEqual(subs['en']['ext'], 'srt')
  498. result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
  499. subs = result['requested_subtitles']
  500. self.assertTrue(subs)
  501. self.assertEqual(set(subs.keys()), set(['es', 'fr']))
  502. result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
  503. subs = result['requested_subtitles']
  504. self.assertTrue(subs)
  505. self.assertEqual(set(subs.keys()), set(['es', 'pt']))
  506. self.assertFalse(subs['es']['_auto'])
  507. self.assertTrue(subs['pt']['_auto'])
  508. result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
  509. subs = result['requested_subtitles']
  510. self.assertTrue(subs)
  511. self.assertEqual(set(subs.keys()), set(['es', 'pt']))
  512. self.assertTrue(subs['es']['_auto'])
  513. self.assertTrue(subs['pt']['_auto'])
  514. def test_add_extra_info(self):
  515. test_dict = {
  516. 'extractor': 'Foo',
  517. }
  518. extra_info = {
  519. 'extractor': 'Bar',
  520. 'playlist': 'funny videos',
  521. }
  522. YDL.add_extra_info(test_dict, extra_info)
  523. self.assertEqual(test_dict['extractor'], 'Foo')
  524. self.assertEqual(test_dict['playlist'], 'funny videos')
  525. def test_prepare_filename(self):
  526. info = {
  527. 'id': '1234',
  528. 'ext': 'mp4',
  529. 'width': None,
  530. 'height': 1080,
  531. 'title1': '$PATH',
  532. 'title2': '%PATH%',
  533. }
  534. def fname(templ):
  535. ydl = YoutubeDL({'outtmpl': templ})
  536. return ydl.prepare_filename(info)
  537. self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
  538. self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
  539. # Replace missing fields with 'NA'
  540. self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
  541. self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
  542. self.assertEqual(fname('%(height)6d.%(ext)s'), ' 1080.mp4')
  543. self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080 .mp4')
  544. self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
  545. self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
  546. self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
  547. self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
  548. self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
  549. self.assertEqual(fname('%(height) 0 6d.%(ext)s'), ' 01080.mp4')
  550. self.assertEqual(fname('%%'), '%')
  551. self.assertEqual(fname('%%%%'), '%%')
  552. self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
  553. self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
  554. self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
  555. self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
  556. self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
  557. self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
  558. def test_format_note(self):
  559. ydl = YoutubeDL()
  560. self.assertEqual(ydl._format_note({}), '')
  561. assertRegexpMatches(self, ydl._format_note({
  562. 'vbr': 10,
  563. }), r'^\s*10k$')
  564. assertRegexpMatches(self, ydl._format_note({
  565. 'fps': 30,
  566. }), r'^30fps$')
  567. def test_postprocessors(self):
  568. filename = 'post-processor-testfile.mp4'
  569. audiofile = filename + '.mp3'
  570. class SimplePP(PostProcessor):
  571. def run(self, info):
  572. with open(audiofile, 'wt') as f:
  573. f.write('EXAMPLE')
  574. return [info['filepath']], info
  575. def run_pp(params, PP):
  576. with open(filename, 'wt') as f:
  577. f.write('EXAMPLE')
  578. ydl = YoutubeDL(params)
  579. ydl.add_post_processor(PP())
  580. ydl.post_process(filename, {'filepath': filename})
  581. run_pp({'keepvideo': True}, SimplePP)
  582. self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
  583. self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
  584. os.unlink(filename)
  585. os.unlink(audiofile)
  586. run_pp({'keepvideo': False}, SimplePP)
  587. self.assertFalse(os.path.exists(filename), '%s exists' % filename)
  588. self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
  589. os.unlink(audiofile)
  590. class ModifierPP(PostProcessor):
  591. def run(self, info):
  592. with open(info['filepath'], 'wt') as f:
  593. f.write('MODIFIED')
  594. return [], info
  595. run_pp({'keepvideo': False}, ModifierPP)
  596. self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
  597. os.unlink(filename)
  598. def test_match_filter(self):
  599. class FilterYDL(YDL):
  600. def __init__(self, *args, **kwargs):
  601. super(FilterYDL, self).__init__(*args, **kwargs)
  602. self.params['simulate'] = True
  603. def process_info(self, info_dict):
  604. super(YDL, self).process_info(info_dict)
  605. def _match_entry(self, info_dict, incomplete):
  606. res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
  607. if res is None:
  608. self.downloaded_info_dicts.append(info_dict)
  609. return res
  610. first = {
  611. 'id': '1',
  612. 'url': TEST_URL,
  613. 'title': 'one',
  614. 'extractor': 'TEST',
  615. 'duration': 30,
  616. 'filesize': 10 * 1024,
  617. 'playlist_id': '42',
  618. 'uploader': "變態妍字幕版 太妍 тест",
  619. 'creator': "тест ' 123 ' тест--",
  620. }
  621. second = {
  622. 'id': '2',
  623. 'url': TEST_URL,
  624. 'title': 'two',
  625. 'extractor': 'TEST',
  626. 'duration': 10,
  627. 'description': 'foo',
  628. 'filesize': 5 * 1024,
  629. 'playlist_id': '43',
  630. 'uploader': "тест 123",
  631. }
  632. videos = [first, second]
  633. def get_videos(filter_=None):
  634. ydl = FilterYDL({'match_filter': filter_})
  635. for v in videos:
  636. ydl.process_ie_result(v, download=True)
  637. return [v['id'] for v in ydl.downloaded_info_dicts]
  638. res = get_videos()
  639. self.assertEqual(res, ['1', '2'])
  640. def f(v):
  641. if v['id'] == '1':
  642. return None
  643. else:
  644. return 'Video id is not 1'
  645. res = get_videos(f)
  646. self.assertEqual(res, ['1'])
  647. f = match_filter_func('duration < 30')
  648. res = get_videos(f)
  649. self.assertEqual(res, ['2'])
  650. f = match_filter_func('description = foo')
  651. res = get_videos(f)
  652. self.assertEqual(res, ['2'])
  653. f = match_filter_func('description =? foo')
  654. res = get_videos(f)
  655. self.assertEqual(res, ['1', '2'])
  656. f = match_filter_func('filesize > 5KiB')
  657. res = get_videos(f)
  658. self.assertEqual(res, ['1'])
  659. f = match_filter_func('playlist_id = 42')
  660. res = get_videos(f)
  661. self.assertEqual(res, ['1'])
  662. f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
  663. res = get_videos(f)
  664. self.assertEqual(res, ['1'])
  665. f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
  666. res = get_videos(f)
  667. self.assertEqual(res, ['2'])
  668. f = match_filter_func('creator = "тест \' 123 \' тест--"')
  669. res = get_videos(f)
  670. self.assertEqual(res, ['1'])
  671. f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
  672. res = get_videos(f)
  673. self.assertEqual(res, ['1'])
  674. f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
  675. res = get_videos(f)
  676. self.assertEqual(res, [])
  677. def test_playlist_items_selection(self):
  678. entries = [{
  679. 'id': compat_str(i),
  680. 'title': compat_str(i),
  681. 'url': TEST_URL,
  682. } for i in range(1, 5)]
  683. playlist = {
  684. '_type': 'playlist',
  685. 'id': 'test',
  686. 'entries': entries,
  687. 'extractor': 'test:playlist',
  688. 'extractor_key': 'test:playlist',
  689. 'webpage_url': 'http://example.com',
  690. }
  691. def get_downloaded_info_dicts(params):
  692. ydl = YDL(params)
  693. # make a deep copy because the dictionary and nested entries
  694. # can be modified
  695. ydl.process_ie_result(copy.deepcopy(playlist))
  696. return ydl.downloaded_info_dicts
  697. def get_ids(params):
  698. return [int(v['id']) for v in get_downloaded_info_dicts(params)]
  699. result = get_ids({})
  700. self.assertEqual(result, [1, 2, 3, 4])
  701. result = get_ids({'playlistend': 10})
  702. self.assertEqual(result, [1, 2, 3, 4])
  703. result = get_ids({'playlistend': 2})
  704. self.assertEqual(result, [1, 2])
  705. result = get_ids({'playliststart': 10})
  706. self.assertEqual(result, [])
  707. result = get_ids({'playliststart': 2})
  708. self.assertEqual(result, [2, 3, 4])
  709. result = get_ids({'playlist_items': '2-4'})
  710. self.assertEqual(result, [2, 3, 4])
  711. result = get_ids({'playlist_items': '2,4'})
  712. self.assertEqual(result, [2, 4])
  713. result = get_ids({'playlist_items': '10'})
  714. self.assertEqual(result, [])
  715. result = get_ids({'playlist_items': '3-10'})
  716. self.assertEqual(result, [3, 4])
  717. result = get_ids({'playlist_items': '2-4,3-4,3'})
  718. self.assertEqual(result, [2, 3, 4])
  719. # Tests for https://github.com/ytdl-org/youtube-dl/issues/10591
  720. # @{
  721. result = get_downloaded_info_dicts({'playlist_items': '2-4,3-4,3'})
  722. self.assertEqual(result[0]['playlist_index'], 2)
  723. self.assertEqual(result[1]['playlist_index'], 3)
  724. result = get_downloaded_info_dicts({'playlist_items': '2-4,3-4,3'})
  725. self.assertEqual(result[0]['playlist_index'], 2)
  726. self.assertEqual(result[1]['playlist_index'], 3)
  727. self.assertEqual(result[2]['playlist_index'], 4)
  728. result = get_downloaded_info_dicts({'playlist_items': '4,2'})
  729. self.assertEqual(result[0]['playlist_index'], 4)
  730. self.assertEqual(result[1]['playlist_index'], 2)
  731. # @}
  732. def test_urlopen_no_file_protocol(self):
  733. # see https://github.com/ytdl-org/youtube-dl/issues/8227
  734. ydl = YDL()
  735. self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
  736. def test_do_not_override_ie_key_in_url_transparent(self):
  737. ydl = YDL()
  738. class Foo1IE(InfoExtractor):
  739. _VALID_URL = r'foo1:'
  740. def _real_extract(self, url):
  741. return {
  742. '_type': 'url_transparent',
  743. 'url': 'foo2:',
  744. 'ie_key': 'Foo2',
  745. 'title': 'foo1 title',
  746. 'id': 'foo1_id',
  747. }
  748. class Foo2IE(InfoExtractor):
  749. _VALID_URL = r'foo2:'
  750. def _real_extract(self, url):
  751. return {
  752. '_type': 'url',
  753. 'url': 'foo3:',
  754. 'ie_key': 'Foo3',
  755. }
  756. class Foo3IE(InfoExtractor):
  757. _VALID_URL = r'foo3:'
  758. def _real_extract(self, url):
  759. return _make_result([{'url': TEST_URL}], title='foo3 title')
  760. ydl.add_info_extractor(Foo1IE(ydl))
  761. ydl.add_info_extractor(Foo2IE(ydl))
  762. ydl.add_info_extractor(Foo3IE(ydl))
  763. ydl.extract_info('foo1:')
  764. downloaded = ydl.downloaded_info_dicts[0]
  765. self.assertEqual(downloaded['url'], TEST_URL)
  766. self.assertEqual(downloaded['title'], 'foo1 title')
  767. self.assertEqual(downloaded['id'], 'testid')
  768. self.assertEqual(downloaded['extractor'], 'testex')
  769. self.assertEqual(downloaded['extractor_key'], 'TestEx')
  770. if __name__ == '__main__':
  771. unittest.main()