[version] update

Created by: pukkandan

:ci skip all

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -11,9 +11,9 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.04.08** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.05.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
@@ -51,12 +51,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.04.08 (exe)
+        [debug] yt-dlp version 2022.05.18 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.04.08)
+        yt-dlp is up to date (2022.05.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,9 +11,9 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.04.08** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.05.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge
           required: true
@@ -62,12 +62,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.04.08 (exe)
+        [debug] yt-dlp version 2022.05.18 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.04.08)
+        yt-dlp is up to date (2022.05.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,9 +11,9 @@ body:
       options:
         - label: I'm reporting a site feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.04.08** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.05.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true
@@ -60,12 +60,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.04.08 (exe)
+        [debug] yt-dlp version 2022.05.18 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.04.08)
+        yt-dlp is up to date (2022.05.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,9 +11,9 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.04.08** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.05.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
@@ -45,12 +45,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.04.08 (exe)
+        [debug] yt-dlp version 2022.05.18 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.04.08)
+        yt-dlp is up to date (2022.05.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.04.08** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.05.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge
           required: true

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've checked that all provided URLs are alive and playable in a browser
+        - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,28 +1,29 @@
-## Please follow the guide below
+<!--
+# Please follow the guide below
 
 - You will be asked some questions, please read them **carefully** and answer honestly
-- Put an `x` into all the boxes [ ] relevant to your *pull request* (like that [x])
+- Put an `x` into all the boxes `[ ]` relevant to your *pull request* (like [x])
 - Use *Preview* tab to see how your *pull request* will actually look like
 
----
+-->
 
 ### Before submitting a *pull request* make sure you have:
 - [ ] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions)
 - [ ] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
-- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)
+- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) and [ran relevant tests](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions)
 
 ### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
 - [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
 - [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)
 
 ### What is the purpose of your *pull request*?
-- [ ] Bug fix
-- [ ] Improvement
-- [ ] New extractor
-- [ ] New feature
+- [ ] Fix or improvement to an extractor (Make sure to add/update tests)
+- [ ] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy))
+- [ ] Core bug fix/improvement
+- [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))
 
 ---
 
 ### Description of your *pull request* and other information
 
-Explanation of your *pull request* in arbitrary form goes here. Please make sure the description explains the purpose and effect of your *pull request* and is worded well enough to be understood. Provide as much context and examples as possible.
+Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible.

.github/workflows/core.yml

@@ -8,14 +8,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04]
-        # py3.9 is in quick-test
-        python-version: [3.7, 3.8, 3.10-dev, pypy-3.6, pypy-3.7]
+        os: [ubuntu-latest]
+        # CPython 3.9 is in quick-test
+        python-version: ['3.6', '3.7', '3.10', 3.11-dev, pypy-3.6, pypy-3.7, pypy-3.8, pypy-3.9]
         run-tests-ext: [sh]
         include:
         # atleast one of the tests must be in windows
         - os: windows-latest
-          python-version: 3.6
+          python-version: 3.8
           run-tests-ext: bat
     steps:
     - uses: actions/checkout@v2

.github/workflows/download.yml

@@ -8,12 +8,12 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-18.04]
-        python-version: [3.7, 3.8, 3.9, 3.10-dev, pypy-3.6, pypy-3.7]
+        os: [ubuntu-latest]
+        python-version: ['3.6', '3.7', '3.9', '3.10', 3.11-dev, pypy-3.6, pypy-3.7, pypy-3.8, pypy-3.9]
         run-tests-ext: [sh]
         include:
         - os: windows-latest
-          python-version: 3.6
+          python-version: 3.8
           run-tests-ext: bat
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -82,6 +82,7 @@ updates_key.pem
 *.egg-info
 .tox
 *.class
+*.isorted
 
 # Generated
 AUTHORS

CONTRIBUTING.md

@@ -178,7 +178,6 @@ After you have ensured this site is distributing its content legally, you can fo
 1. Start with this simple template and save it to `yt_dlp/extractor/yourextractor.py`:
 
     ```python
-    # coding: utf-8
     from .common import InfoExtractor
     
     
@@ -375,21 +374,21 @@ When extracting metadata try to do so from multiple sources. For example if `tit
 
 #### Example
 
-Say `meta` from the previous example has a `title` and you are about to extract it. Since `title` is a mandatory meta field you should end up with something like:
+Say `meta` from the previous example has a `title` and you are about to extract it like:
 
 ```python
-title = meta['title']
+title = meta.get('title')
 ```
 
-If `title` disappears from `meta` in future due to some changes on the hoster's side the extraction would fail since `title` is mandatory. That's expected.
+If `title` disappears from `meta` in future due to some changes on the hoster's side the title extraction would fail.
 
-Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback scenario:
+Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback like:
 
 ```python
 title = meta.get('title') or self._og_search_title(webpage)
 ```
 
-This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
+This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`, making the extractor more robust.
 
 
 ### Regular expressions

CONTRIBUTORS

@@ -231,3 +231,20 @@ Fam0r
 bohwaz
 dodrian
 vvto33
+ca-za
+connercsbn
+diegorodriguezv
+ekangmonyet
+elyse0
+evansp
+GiedriusS
+HE7086
+JordanWeatherby
+m4tu4g
+MarwenDallel
+nevack
+putnam
+rand-net
+vertan
+Wikidepia
+Yipten

Changelog.md

@@ -11,11 +11,129 @@
 -->
 
 
+### 2022.05.18
+
+* Add support for SSL client certificate authentication by [coletdjnz](https://github.com/coletdjnz), [dirkf](https://github.com/dirkf)
+    * Adds `--client-certificate`, `--client-certificate-key`, `--client-certificate-password`
+* Add `--match-filter -` to interactively ask for each video
+* `--max-downloads` should obey `--break-per-input`
+* Allow use of weaker ciphers with `--legacy-server-connect`
+* Don't imply `-s` for later stages of `-O`
+* Fix `--date today`
+* Fix `--skip-unavailable-fragments`
+* Fix color in `-q -F`
+* Fix redirect HTTP method handling by [coletdjnz](https://github.com/coletdjnz)
+* Improve `--clean-infojson`
+* Remove warning for videos with an empty title
+* Run `FFmpegFixupM3u8PP` for live-streams if needed
+* Show name of downloader in verbose log
+* [cookies] Allow `cookiefile` to be a text stream
+* [cookies] Report progress when importing cookies
+* [downloader/ffmpeg] Specify headers for each URL by [elyse0](https://github.com/elyse0)
+* [fragment] Do not change chunk-size when `--test`
+* [fragment] Make single thread download work for `--live-from-start` by [Lesmiscore](https://github.com/Lesmiscore)
+* [hls] Fix `byte_range` for `EXT-X-MAP` fragment by [fstirlitz](https://github.com/fstirlitz)
+* [http] Fix retrying on read timeout by [coletdjnz](https://github.com/coletdjnz)
+* [ffmpeg] Fix features detection
+* [EmbedSubtitle] Enable for more video extensions
+* [EmbedThumbnail] Disable thumbnail conversion for mkv by [evansp](https://github.com/evansp)
+* [EmbedThumbnail] Do not obey `-k`
+* [EmbedThumbnail] Do not remove id3v1 tags
+* [FFmpegMetadata] Remove `\0` from metadata
+* [FFmpegMetadata] Remove filename from attached info-json
+* [FixupM3u8] Obey `--hls-prefer-mpegts`
+* [Sponsorblock] Don't crash when duration is unknown
+* [XAttrMetadata] Refactor and document dependencies
+* [extractor] Document netrc machines
+* [extractor] Update `manifest_url`s after redirect by [elyse0](https://github.com/elyse0)
+* [extractor] Update dash `manifest_url` after redirects by [elyse0](https://github.com/elyse0)
+* [extractor] Use `classmethod`/`property` where possible
+* [generic] Refactor `_extract_rss`
+* [utils] `is_html`: Handle double BOM
+* [utils] `locked_file`: Ignore illegal seek on `truncate` by [jakeogh](https://github.com/jakeogh)
+* [utils] `sanitize_path`: Fix when path is empty string
+* [utils] `write_string`: Workaround newline issue in `conhost`
+* [utils] `certifi`: Make sure the pem file exists
+* [utils] Fix `WebSocketsWrapper`
+* [utils] `locked_file`: Do not give executable bits for newly created files by [Lesmiscore](https://github.com/Lesmiscore)
+* [utils] `YoutubeDLCookieJar`: Detect and reject JSON file by [Lesmiscore](https://github.com/Lesmiscore)
+* [test] Convert warnings into errors and fix some existing warnings by [fstirlitz](https://github.com/fstirlitz)
+* [dependencies] Create module with all dependency imports
+* [compat] Split into sub-modules by [fstirlitz](https://github.com/fstirlitz), [pukkandan](https://github.com/pukkandan)
+* [compat] Implement `compat.imghdr`
+* [build] Add `make uninstall` by [MrRawes](https://github.com/MrRawes)
+* [build] Avoid use of `install -D`
+* [build] Fix `Makefile` by [putnam](https://github.com/putnam)
+* [build] Fix `--onedir` on macOS
+* [build] Add more test-runners
+* [cleanup] Deprecate some compat vars by [fstirlitz](https://github.com/fstirlitz), [pukkandan](https://github.com/pukkandan)
+* [cleanup] Remove unused code paths, extractors, scripts and tests by [fstirlitz](https://github.com/fstirlitz)
+* [cleanup] Upgrade syntax (`pyupgrade`) and sort imports (`isort`)
+* [cleanup, docs, build] Misc fixes
+* [BilibiliLive] Add extractor by [HE7086](https://github.com/HE7086), [pukkandan](https://github.com/pukkandan)
+* [Fifa] Add Extractor by [Bricio](https://github.com/Bricio)
+* [goodgame] Add extractor by [nevack](https://github.com/nevack)
+* [gronkh] Add playlist extractors by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [icareus] Add extractor by [tpikonen](https://github.com/tpikonen), [pukkandan](https://github.com/pukkandan)
+* [iwara] Add playlist extractors by [i6t](https://github.com/i6t)
+* [Likee] Add extractor by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [masters] Add extractor by [m4tu4g](https://github.com/m4tu4g)
+* [nebula] Add support for subscriptions by [hheimbuerger](https://github.com/hheimbuerger)
+* [Podchaser] Add extractors by [connercsbn](https://github.com/connercsbn)
+* [rokfin:search] Add extractor by [P-reducible](https://github.com/P-reducible), [pukkandan](https://github.com/pukkandan)
+* [youtube] Add `:ytnotifications` extractor by [krichbanana](https://github.com/krichbanana)
+* [youtube] Add YoutubeStoriesIE (`ytstories:<channel UCID>`) by [coletdjnz](https://github.com/coletdjnz)
+* [ZingMp3] Add chart and user extractors by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [adn] Update AES key by [elyse0](https://github.com/elyse0)
+* [adobepass] Allow cookies for authenticating MSO
+* [bandcamp] Exclude merch links by [Yipten](https://github.com/Yipten)
+* [chingari] Fix archiving and tests
+* [DRTV] Improve `_VALID_URL` by [vertan](https://github.com/vertan)
+* [facebook] Improve thumbnail extraction by [Wikidepia](https://github.com/Wikidepia)
+* [fc2] Stop heatbeating once FFmpeg finishes by [Lesmiscore](https://github.com/Lesmiscore)
+* [Gofile] Fix extraction and support password-protected links by [mehq](https://github.com/mehq)
+* [hotstar, cleanup] Refactor extractors
+* [InfoQ] Don't fail on missing audio format by [evansp](https://github.com/evansp)
+* [Jamendo] Extract more metadata by [evansp](https://github.com/evansp)
+* [kaltura] Update API calls by [flashdagger](https://github.com/flashdagger)
+* [KhanAcademy] Fix extractor by [rand-net](https://github.com/rand-net)
+* [LCI] Fix extractor by [MarwenDallel](https://github.com/MarwenDallel)
+* [lrt] Support livestreams by [GiedriusS](https://github.com/GiedriusS)
+* [niconico] Set `expected_protocol` to a public field
+* [Niconico] Support 2FA by [ekangmonyet](https://github.com/ekangmonyet)
+* [Olympics] Fix format extension
+* [openrec:movie] Enable fallback for /movie/ URLs
+* [PearVideo] Add fallback for formats by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [radiko] Fix extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [rai] Add `release_year`
+* [reddit] Prevent infinite loop
+* [rokfin] Implement login by [P-reducible](https://github.com/P-reducible), [pukkandan](https://github.com/pukkandan)
+* [ruutu] Support hs.fi embeds by [tpikonen](https://github.com/tpikonen), [pukkandan](https://github.com/pukkandan)
+* [spotify] Detect iframe embeds by [fstirlitz](https://github.com/fstirlitz)
+* [telegram] Fix metadata extraction
+* [tmz, cleanup] Update tests by [diegorodriguezv](https://github.com/diegorodriguezv)
+* [toggo] Fix `_VALID_URL` by [ca-za](https://github.com/ca-za)
+* [trovo] Update to new API by [nyuszika7h](https://github.com/nyuszika7h)
+* [TVer] Improve extraction by [Lesmiscore](https://github.com/Lesmiscore)
+* [twitcasting] Pass headers for each formats by [Lesmiscore](https://github.com/Lesmiscore)
+* [VideocampusSachsen] Improve extractor by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel)
+* [vimeo] Fix extractors
+* [wat] Fix extraction of multi-language videos and subtitles by [elyse0](https://github.com/elyse0)
+* [wistia] Fix `_VALID_URL` by [dirkf](https://github.com/dirkf)
+* [youtube, cleanup] Minor refactoring by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* [youtube] Added piped instance urls by [JordanWeatherby](https://github.com/JordanWeatherby)
+* [youtube] Deprioritize auto-generated thumbnails
+* [youtube] Deprioritize format 22 (often damaged)
+* [youtube] Fix episode metadata extraction
+* [zee5] Fix extractor by [Ashish0804](https://github.com/Ashish0804)
+* [zingmp3, cleanup] Refactor extractors
+
+
 ### 2022.04.08
 
 * Use certificates from `certifi` if installed by [coletdjnz](https://github.com/coletdjnz)
 * Treat multiple `--match-filters` as OR
-* File locking improvevemnts:
+* File locking improvements:
     * Do not lock downloading file on Windows
     * Do not prevent download if locking is unsupported
     * Do not truncate files before locking by [jakeogh](https://github.com/jakeogh), [pukkandan](https://github.com/pukkandan)
@@ -785,7 +903,7 @@
 * [build] Improvements
     * Build standalone MacOS packages by [smplayer-dev](https://github.com/smplayer-dev)
     * Release windows exe built with `py2exe`
-    * Enable lazy-extractors in releases. 
+    * Enable lazy-extractors in releases
         * Set env var `YTDLP_NO_LAZY_EXTRACTORS` to forcefully disable this (experimental)
     * Clean up error reporting in update
     * Refactor `pyinst.py`, misc cleanup and improve docs
@@ -1393,7 +1511,7 @@
 * [youtube] Non-fatal alert reporting for unavailable videos page by [coletdjnz](https://github.com/coletdjnz)
 * [twitcasting] Websocket support by [nao20010128nao](https://github.com/nao20010128nao)
 * [mediasite] Extract slides by [fstirlitz](https://github.com/fstirlitz)
-* [funimation] Extract subtitles 
+* [funimation] Extract subtitles
 * [pornhub] Extract `cast`
 * [hotstar] Use server time for authentication instead of local time
 * [EmbedThumbnail] Fix for already downloaded thumbnail
@@ -1489,7 +1607,7 @@
 
 ### 2021.05.20
 
-* **Youtube improvements**: 
+* **Youtube improvements**:
     * Support youtube music `MP`, `VL` and `browse` pages
     * Extract more formats for youtube music by [craftingmod](https://github.com/craftingmod), [coletdjnz](https://github.com/coletdjnz) and [pukkandan](https://github.com/pukkandan)
     * Extract multiple subtitles in same language by [pukkandan](https://github.com/pukkandan) and [tpikonen](https://github.com/tpikonen)
@@ -2031,7 +2149,7 @@
 * **Format Sort:** Added `--format-sort` (`-S`), `--format-sort-force` (`--S-force`) - See [Sorting Formats](README.md#sorting-formats) for details
 * **Format Selection:** See [Format Selection](README.md#format-selection) for details
     * New format selectors: `best*`, `worst*`, `bestvideo*`, `bestaudio*`, `worstvideo*`, `worstaudio*`
-    * Changed video format sorting to show video only files and video+audio files together.
+    * Changed video format sorting to show video only files and video+audio files together
     * Added `--video-multistreams`, `--no-video-multistreams`, `--audio-multistreams`, `--no-audio-multistreams`
     * Added `b`,`w`,`v`,`a` as alias for `best`, `worst`, `video` and `audio` respectively
 * Shortcut Options: Added `--write-link`, `--write-url-link`, `--write-webloc-link`, `--write-desktop-link` by [h-h-h-h](https://github.com/h-h-h-h) - See [Internet Shortcut Options](README.md#internet-shortcut-options) for details

Makefile

@@ -22,7 +22,9 @@ clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
 	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
 clean-cache:
-	find . \( -name "*.pyc" -o -name "*.class" \) -delete
+	find . \( \
+		-type d -name .pytest_cache -o -type d -name __pycache__ -o -name "*.pyc" -o -name "*.class" \
+	\) -prune -exec rm -rf {} \;
 
 completion-bash: completions/bash/yt-dlp
 completion-fish: completions/fish/yt-dlp.fish
@@ -40,14 +42,26 @@ PYTHON ?= /usr/bin/env python3
 SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi)
 
 # set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2
-MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi)
+MARKDOWN = $(shell if [ "$(pandoc -v | head -n1 | cut -d" " -f2 | head -c1)" = "2" ]; then echo markdown-smart; else echo markdown; fi)
 
 install: lazy-extractors yt-dlp yt-dlp.1 completions
-	install -Dm755 yt-dlp $(DESTDIR)$(BINDIR)/yt-dlp
-	install -Dm644 yt-dlp.1 $(DESTDIR)$(MANDIR)/man1/yt-dlp.1
-	install -Dm644 completions/bash/yt-dlp $(DESTDIR)$(SHAREDIR)/bash-completion/completions/yt-dlp
-	install -Dm644 completions/zsh/_yt-dlp $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_yt-dlp
-	install -Dm644 completions/fish/yt-dlp.fish $(DESTDIR)$(SHAREDIR)/fish/vendor_completions.d/yt-dlp.fish
+	mkdir -p $(DESTDIR)$(BINDIR)
+	install -m755 yt-dlp $(DESTDIR)$(BINDIR)/yt-dlp
+	mkdir -p $(DESTDIR)$(MANDIR)/man1
+	install -m644 yt-dlp.1 $(DESTDIR)$(MANDIR)/man1/yt-dlp.1
+	mkdir -p $(DESTDIR)$(SHAREDIR)/bash-completion/completions
+	install -m644 completions/bash/yt-dlp $(DESTDIR)$(SHAREDIR)/bash-completion/completions/yt-dlp
+	mkdir -p $(DESTDIR)$(SHAREDIR)/zsh/site-functions
+	install -m644 completions/zsh/_yt-dlp $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_yt-dlp
+	mkdir -p $(DESTDIR)$(SHAREDIR)/fish/vendor_completions.d
+	install -m644 completions/fish/yt-dlp.fish $(DESTDIR)$(SHAREDIR)/fish/vendor_completions.d/yt-dlp.fish
+
+uninstall:
+	rm -f $(DESTDIR)$(BINDIR)/yt-dlp
+	rm -f $(DESTDIR)$(MANDIR)/man1/yt-dlp.1
+	rm -f $(DESTDIR)$(SHAREDIR)/bash-completion/completions/yt-dlp
+	rm -f $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_yt-dlp
+	rm -f $(DESTDIR)$(SHAREDIR)/fish/vendor_completions.d/yt-dlp.fish
 
 codetest:
 	flake8 .
@@ -59,15 +73,18 @@ test:
 offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
+# XXX: This is hard to maintain
+CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat \
+               yt_dlp/extractor/anvato_token_generator
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
 	mkdir -p zip
-	for d in yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor ; do \
+	for d in $(CODE_FOLDERS) ; do \
 	  mkdir -p zip/$$d ;\
 	  cp -pPR $$d/*.py zip/$$d/ ;\
 	done
-	touch -t 200001010101 zip/yt_dlp/*.py zip/yt_dlp/*/*.py
+	touch -t 200001010101 zip/yt_dlp/*.py zip/yt_dlp/*/*.py zip/yt_dlp/*/*/*.py
 	mv zip/yt_dlp/__main__.py zip/
-	cd zip ; zip -q ../yt-dlp yt_dlp/*.py yt_dlp/*/*.py __main__.py
+	cd zip ; zip -q ../yt-dlp yt_dlp/*.py yt_dlp/*/*.py yt_dlp/*/*/*.py __main__.py
 	rm -rf zip
 	echo '#!$(PYTHON)' > yt-dlp
 	cat yt-dlp.zip >> yt-dlp
@@ -123,6 +140,7 @@ yt-dlp.tar.gz: all
 		--exclude '*.pyo' \
 		--exclude '*~' \
 		--exclude '__pycache__' \
+		--exclude '.pytest_cache' \
 		--exclude '.git' \
 		-- \
 		README.md supportedsites.md Changelog.md LICENSE \

README.md

@@ -60,6 +60,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 * [EXTRACTOR ARGUMENTS](#extractor-arguments)
 * [PLUGINS](#plugins)
 * [EMBEDDING YT-DLP](#embedding-yt-dlp)
+    * [Embedding examples](#embedding-examples)
 * [DEPRECATED OPTIONS](#deprecated-options)
 * [CONTRIBUTING](CONTRIBUTING.md#contributing-to-yt-dlp)
     * [Opening an Issue](CONTRIBUTING.md#opening-an-issue)
@@ -79,7 +80,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that the NicoNico livestreams are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details.
 
 * **Youtube improvements**:
-    * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`) and private playlists supports downloading multiple pages of content
+    * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`) and private playlists supports downloading multiple pages of content
     * Search (`ytsearch:`, `ytsearchdate:`), search URLs and in-channel search works
     * Mixes supports downloading multiple pages of content
     * Some (but not all) age-gated content can be downloaded without cookies
@@ -88,6 +89,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
     * `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
     * Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
     * Download livestreams from the start using `--live-from-start` (experimental)
+    * Support for downloading stories (`ytstories:<channel UCID>`) 
 
 * **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE]`
 
@@ -148,6 +150,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * youtube-dl tries to remove some superfluous punctuations from filenames. While this can sometimes be helpfull, it is often undesirable. So yt-dlp tries to keep the fields in the filenames as close to their original values as possible. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 
 For ease of use, a few more compat options are available:
+
 * `--compat-options all`: Use all compat options
 * `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams`
 * `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect`
@@ -163,10 +166,11 @@ You can simply download the [correct binary file](#release-files) for your OS
 
 <!-- MANPAGE: BEGIN EXCLUDED SECTION -->
 [![Windows](https://img.shields.io/badge/-Windows_x64-blue.svg?style=for-the-badge&logo=windows)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)
-[![Linux](https://img.shields.io/badge/-Linux/MacOS/BSD-red.svg?style=for-the-badge&logo=linux)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)
+[![Linux](https://img.shields.io/badge/-Linux/BSD-red.svg?style=for-the-badge&logo=linux)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)
+[![MacOS](https://img.shields.io/badge/-MacOS-lightblue.svg?style=for-the-badge&logo=apple)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)
 [![Source Tarball](https://img.shields.io/badge/-Source_tar-green.svg?style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
 [![Other variants](https://img.shields.io/badge/-Other-grey.svg?style=for-the-badge)](#release-files)
-[![ALl versions](https://img.shields.io/badge/-All_Versions-lightgrey.svg?style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases)
+[![All versions](https://img.shields.io/badge/-All_Versions-lightgrey.svg?style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases)
 <!-- MANPAGE: END EXCLUDED SECTION -->
 
 Note: The manpages, shell completion files etc. are available in the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
@@ -241,7 +245,7 @@ File|Description
 File|Description
 :---|:---
 [yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Vista SP2+) standalone x86 (32-bit) binary
-[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`.<br/> Does not contain `pycryptodomex`, needs VC++14
+[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`.<br/> Does not contain `pycryptodomex`/`certifi`, needs VC++14
 [yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
 [yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update)
 
@@ -268,9 +272,10 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
 * [**mutagen**](https://github.com/quodlibet/mutagen)\* - For embedding thumbnail in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
 * [**pycryptodomex**](https://github.com/Legrandin/pycryptodome)\* - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD-2-Clause](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
 * [**websockets**](https://github.com/aaugustin/websockets)\* - For downloading over websocket. Licensed under [BSD-3-Clause](https://github.com/aaugustin/websockets/blob/main/LICENSE)
-* [**secretstorage**](https://github.com/mitya57/secretstorage)\* - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
+* [**secretstorage**](https://github.com/mitya57/secretstorage) - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
 * [**brotli**](https://github.com/google/brotli)\* or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT <sup>[1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) </sup>
 * [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE)
+* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata on Linux. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
 * [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen/ffmpeg cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
 * [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
 * [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
@@ -280,7 +285,7 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
 
 To use or redistribute the dependencies, you must agree to their respective licensing terms.
 
-The Windows and MacOS standalone release binaries are already built with the python interpreter and all optional python packages (marked with \*) included.
+The Windows and MacOS standalone release binaries are built with the Python interpreter and the packages marked with \* included.
 
 <!-- TODO: ffmpeg has merged this patch. Remove this note once there is new release -->
 **Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds#ffmpeg-static-auto-builds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds
@@ -317,9 +322,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
 ## General Options:
     -h, --help                       Print this help text and exit
     --version                        Print program version and exit
-    -U, --update                     Update this program to latest version. Make
-                                     sure that you have sufficient permissions
-                                     (run with sudo if needed)
+    -U, --update                     Update this program to latest version
     -i, --ignore-errors              Ignore download and postprocessing errors.
                                      The download will be considered successful
                                      even if the postprocessing fails
@@ -425,7 +428,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      (e.g. 50k or 44.6m)
     --date DATE                      Download only videos uploaded on this date.
                                      The date can be "YYYYMMDD" or in the format
-                                     "(now|today)[+-][0-9](day|week|month|year)(s)?"
+                                     [now|today|yesterday][-N[day|week|month|year]].
+                                     Eg: --date today-2weeks
     --datebefore DATE                Download only videos uploaded on or before
                                      this date. The date formats accepted is the
                                      same as --date
@@ -449,7 +453,9 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      those that have a like count more than 100
                                      (or the like field is not available) and
                                      also has a description that contains the
-                                     phrase "cats & dogs" (ignoring case)
+                                     phrase "cats & dogs" (ignoring case). Use
+                                     "--match-filter -" to interactively ask
+                                     whether to download each video
     --no-match-filter                Do not use generic video filter (default)
     --no-playlist                    Download only the video, if the URL refers
                                      to a video and a playlist
@@ -466,9 +472,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      a file that is in the archive
     --break-on-reject                Stop the download process when encountering
                                      a file that has been filtered out
-    --break-per-input                Make --break-on-existing and --break-on-
-                                     reject act only on the current input URL
-    --no-break-per-input             --break-on-existing and --break-on-reject
+    --break-per-input                Make --break-on-existing, --break-on-reject
+                                     and --max-downloads act only on the current
+                                     input URL
+    --no-break-per-input             --break-on-existing and similar options
                                      terminates the entire download queue
     --skip-playlist-after-errors N   Number of allowed failures until the rest
                                      of the playlist is skipped
@@ -485,7 +492,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     -R, --retries RETRIES            Number of retries (default is 10), or
                                      "infinite"
     --file-access-retries RETRIES    Number of times to retry on file access
-                                     error (default is 10), or "infinite"
+                                     error (default is 3), or "infinite"
     --fragment-retries RETRIES       Number of retries for a fragment (default
                                      is 10), or "infinite" (DASH, hlsnative and
                                      ISM)
@@ -529,10 +536,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      (http, ftp, m3u8, dash, rstp, rtmp, mms) to
                                      use it for. Currently supports native,
                                      aria2c, avconv, axel, curl, ffmpeg, httpie,
-                                     wget (Recommended: aria2c). You can use
-                                     this option multiple times to set different
-                                     downloaders for different protocols. For
-                                     example, --downloader aria2c --downloader
+                                     wget. You can use this option multiple
+                                     times to set different downloaders for
+                                     different protocols. For example,
+                                     --downloader aria2c --downloader
                                      "dash,m3u8:native" will use aria2c for
                                      http/ftp downloads, and the native
                                      downloader for dash/m3u8 downloads (Alias:
@@ -688,9 +695,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      print it, separated by a ":". Supported
                                      values of "WHEN" are the same as that of
                                      --use-postprocessor, and "video" (default).
-                                     Implies --quiet and --simulate (unless
-                                     --no-simulate is used). This option can be
-                                     used multiple times
+                                     Implies --quiet. Implies --simulate unless
+                                     --no-simulate or later stages of WHEN are
+                                     used. This option can be used multiple
+                                     times
     --print-to-file [WHEN:]TEMPLATE FILE
                                      Append given template to the file. The
                                      values of WHEN and TEMPLATE are same as
@@ -835,6 +843,15 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      interactively
     --ap-list-mso                    List all supported multiple-system
                                      operators
+    --client-certificate CERTFILE    Path to client certificate file in PEM
+                                     format. May include the private key
+    --client-certificate-key KEYFILE Path to private key file for client
+                                     certificate
+    --client-certificate-password PASSWORD
+                                     Password for client certificate private
+                                     key, if encrypted. If not provided and the
+                                     key is encrypted, yt-dlp will ask
+                                     interactively
 
 ## Post-Processing Options:
     -x, --extract-audio              Convert video files to audio-only files
@@ -925,8 +942,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      same codecs and number of streams to be
                                      concatable. The "pl_video:" prefix can be
                                      used with "--paths" and "--output" to set
-                                     the output filename for the split files.
-                                     See "OUTPUT TEMPLATE" for details
+                                     the output filename for the concatenated
+                                     files. See "OUTPUT TEMPLATE" for details
     --fixup POLICY                   Automatically correct known faults of the
                                      file. One of never (do nothing), warn (only
                                      emit a warning), detect_or_warn (the
@@ -1063,8 +1080,9 @@ You can configure yt-dlp by placing any supported command line option to a confi
     * `%APPDATA%/yt-dlp/config.txt`
     * `~/yt-dlp.conf`
     * `~/yt-dlp.conf.txt`
-
+    
     `%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `%APPDATA%` generally points to `C:\Users\<user name>\AppData\Roaming` and `~` points to `%HOME%` if present, `%USERPROFILE%` (generally `C:\Users\<user name>`), or `%HOMEDRIVE%%HOMEPATH%`
+
 1. **System Configuration**: `/etc/yt-dlp.conf`
 
 For example, with the following configuration file yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
@@ -1121,6 +1139,7 @@ The simplest usage of `-o` is not to set any template arguments when downloading
 It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
 
 The field names themselves (the part inside the parenthesis) can also have some special formatting:
+
 1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)s`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. `%()s` refers to the entire infodict. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
 
 1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. Eg: `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
@@ -1601,7 +1620,9 @@ The general syntax of `--parse-metadata FROM:TO` is to give the name of a field
 Note that any field created by this can be used in the [output template](#output-template) and will also affect the media file's metadata added when using `--add-metadata`.
 
 This option also has a few special uses:
+
 * You can download an additional URL based on the metadata of the currently downloaded video. To do this, set the field `additional_urls` to the URL that you want to download. Eg: `--parse-metadata "description:(?P<additional_urls>https?://www\.vimeo\.com/\d+)` will download the first vimeo video found in the description
+
 * You can use this to change the metadata that is embedded in the media file. To do this, set the value of the corresponding field with a `meta_` prefix. For example, any value you set to `meta_description` field will be added to the `description` field in the file. For example, you can use this to set a different "description" and "synopsis". To modify the metadata of individual streams, use the `meta<n>_` prefix (Eg: `meta1_language`). Any value set to the `meta_` field will overwrite all default values.
 
 **Note**: Metadata modification happens before format selection, post-extraction and other post-processing operations. Some fields may be added or changed during these steps, overriding your changes.
@@ -1743,19 +1764,94 @@ From a Python program, you can embed yt-dlp in a more powerful fashion, like thi
 ```python
 from yt_dlp import YoutubeDL
 
-ydl_opts = {'format': 'bestaudio'}
-with YoutubeDL(ydl_opts) as ydl:
-    ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
+URLS = ['https://www.youtube.com/watch?v=BaW_jenozKc']
+with YoutubeDL() as ydl:
+    ydl.download(URLS)
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L197).
+Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L181).
 
-Here's a more complete example demonstrating various functionality:
+**Tip**: If you are porting your code from youtube-dl to yt-dlp, one important point to look out for is that we do not guarantee the return value of `YoutubeDL.extract_info` to be json serializable, or even be a dictionary. It will be dictionary-like, but if you want to ensure it is a serializable dictionary, pass it through `YoutubeDL.sanitize_info` as shown in the [example below](#extracting-information)
+
+## Embedding examples
+
+#### Extracting information
 
 ```python
 import json
 import yt_dlp
 
+URL = 'https://www.youtube.com/watch?v=BaW_jenozKc'
+
+# ℹ️ See help(yt_dlp.YoutubeDL) for a list of available options and public functions
+ydl_opts = {}
+with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+    info = ydl.extract_info(URL, download=False)
+
+    # ℹ️ ydl.sanitize_info makes the info json-serializable
+    print(json.dumps(ydl.sanitize_info(info)))
+```
+#### Download using an info-json
+
+```python
+import yt_dlp
+
+INFO_FILE = 'path/to/video.info.json'
+
+with yt_dlp.YoutubeDL() as ydl:
+    error_code = ydl.download_with_info_file(INFO_FILE)
+
+print('Some videos failed to download' if error_code
+      else 'All videos successfully downloaded')
+```
+
+#### Extract audio
+
+```python
+import yt_dlp
+
+URLS = ['https://www.youtube.com/watch?v=BaW_jenozKc']
+
+ydl_opts = {
+    'format': 'm4a/bestaudio/best',
+    # ℹ️ See help(yt_dlp.postprocessor) for a list of available Postprocessors and their arguments
+    'postprocessors': [{  # Extract audio using ffmpeg
+        'key': 'FFmpegExtractAudio',
+        'preferredcodec': 'm4a',
+    }]
+}
+
+with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+    error_code = ydl.download(URLS)
+```
+
+#### Filter videos
+
+```python
+import yt_dlp
+
+URLS = ['https://www.youtube.com/watch?v=BaW_jenozKc']
+
+def longer_than_a_minute(info, *, incomplete):
+    """Download only videos longer than a minute (or with unknown duration)"""
+    duration = info.get('duration')
+    if duration and duration < 60:
+        return 'The video is too short'
+
+ydl_opts = {
+    'match_filter': longer_than_a_minute,
+}
+
+with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+    error_code = ydl.download(URLS)
+```
+
+#### Adding logger and progress hook
+
+```python
+import yt_dlp
+
+URLS = ['https://www.youtube.com/watch?v=BaW_jenozKc']
 
 class MyLogger:
     def debug(self, msg):
@@ -1776,23 +1872,51 @@ class MyLogger:
         print(msg)
 
 
-# ℹ️ See the docstring of yt_dlp.postprocessor.common.PostProcessor
+# ℹ️ See "progress_hooks" in help(yt_dlp.YoutubeDL)
+def my_hook(d):
+    if d['status'] == 'finished':
+        print('Done downloading, now post-processing ...')
+
+
+ydl_opts = {
+    'logger': MyLogger(),
+    'progress_hooks': [my_hook],
+}
+
+with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+    ydl.download(URLS)
+```
+
+#### Add a custom PostProcessor
+
+```python
+import yt_dlp
+
+URLS = ['https://www.youtube.com/watch?v=BaW_jenozKc']
+
+# ℹ️ See help(yt_dlp.postprocessor.PostProcessor)
 class MyCustomPP(yt_dlp.postprocessor.PostProcessor):
-    # ℹ️ See docstring of yt_dlp.postprocessor.common.PostProcessor.run
     def run(self, info):
         self.to_screen('Doing stuff')
         return [], info
 
 
-# ℹ️ See "progress_hooks" in the docstring of yt_dlp.YoutubeDL
-def my_hook(d):
-    if d['status'] == 'finished':
-        print('Done downloading, now converting ...')
+with yt_dlp.YoutubeDL() as ydl:
+    ydl.add_post_processor(MyCustomPP())
+    ydl.download(URLS)
+```
 
 
+#### Use a custom format selector
+
+```python
+import yt_dlp
+
+URL = ['https://www.youtube.com/watch?v=BaW_jenozKc']
+
 def format_selector(ctx):
     """ Select the best video and the best audio that won't result in an mkv.
-    This is just an example and does not handle all cases """
+    NOTE: This is just an example and does not handle all cases """
 
     # formats are already sorted worst to best
     formats = ctx.get('formats')[::-1]
@@ -1807,8 +1931,8 @@ def format_selector(ctx):
     best_audio = next(f for f in formats if (
         f['acodec'] != 'none' and f['vcodec'] == 'none' and f['ext'] == audio_ext))
 
+    # These are the minimum required fields for a merged format
     yield {
-        # These are the minimum required fields for a merged format
         'format_id': f'{best_video["format_id"]}+{best_audio["format_id"]}',
         'ext': best_video['ext'],
         'requested_formats': [best_video, best_audio],
@@ -1817,36 +1941,14 @@ def format_selector(ctx):
     }
 
 
-# ℹ️ See docstring of yt_dlp.YoutubeDL for a description of the options
 ydl_opts = {
     'format': format_selector,
-    'postprocessors': [{
-        # Embed metadata in video using ffmpeg.
-        # ℹ️ See yt_dlp.postprocessor.FFmpegMetadataPP for the arguments it accepts
-        'key': 'FFmpegMetadata',
-        'add_chapters': True,
-        'add_metadata': True,
-    }],
-    'logger': MyLogger(),
-    'progress_hooks': [my_hook],
-    # Add custom headers
-    'http_headers': {'Referer': 'https://www.google.com'}
 }
 
-
-# ℹ️ See the public functions in yt_dlp.YoutubeDL for for other available functions.
-# Eg: "ydl.download", "ydl.download_with_info_file"
 with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-    ydl.add_post_processor(MyCustomPP())
-    info = ydl.extract_info('https://www.youtube.com/watch?v=BaW_jenozKc')
-
-    # ℹ️ ydl.sanitize_info makes the info json-serializable
-    print(json.dumps(ydl.sanitize_info(info)))
+    ydl.download(URLS)
 ```
 
-**Tip**: If you are porting your code from youtube-dl to yt-dlp, one important point to look out for is that we do not guarantee the return value of `YoutubeDL.extract_info` to be json serializable, or even be a dictionary. It will be dictionary-like, but if you want to ensure it is a serializable dictionary, pass it through `YoutubeDL.sanitize_info` as shown in the example above
-
-
 <!-- MANPAGE: MOVE "NEW FEATURES" SECTION HERE -->
 
 # DEPRECATED OPTIONS
@@ -1960,8 +2062,7 @@ These options may no longer work as intended
 These options were deprecated since 2014 and have now been entirely removed
 
     -A, --auto-number                -o "%(autonumber)s-%(id)s.%(ext)s"
-    -t, --title                      -o "%(title)s-%(id)s.%(ext)s"
-    -l, --literal                    -o accepts literal names
+    -t, -l, --title, --literal       -o "%(title)s-%(id)s.%(ext)s"
 
 # CONTRIBUTING
 See [CONTRIBUTING.md](CONTRIBUTING.md#contributing-to-yt-dlp) for instructions on [Opening an Issue](CONTRIBUTING.md#opening-an-issue) and [Contributing code to the project](CONTRIBUTING.md#developer-instructions)

devscripts/bash-completion.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import os
-from os.path import dirname as dirn
 import sys
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import yt_dlp
 
 BASH_COMPLETION_FILE = "completions/bash/yt-dlp"
@@ -26,5 +24,5 @@ def build_completion(opt_parser):
         f.write(filled_template)
 
 
-parser = yt_dlp.parseOpts()[0]
+parser = yt_dlp.parseOpts(ignore_config_files=True)[0]
 build_completion(parser)

devscripts/buildserver.py

@@ -1,435 +0,0 @@
-# UNUSED
-
-#!/usr/bin/python3
-
-import argparse
-import ctypes
-import functools
-import shutil
-import subprocess
-import sys
-import tempfile
-import threading
-import traceback
-import os.path
-
-sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
-from yt_dlp.compat import (
-    compat_input,
-    compat_http_server,
-    compat_str,
-    compat_urlparse,
-)
-
-# These are not used outside of buildserver.py thus not in compat.py
-
-try:
-    import winreg as compat_winreg
-except ImportError:  # Python 2
-    import _winreg as compat_winreg
-
-try:
-    import socketserver as compat_socketserver
-except ImportError:  # Python 2
-    import SocketServer as compat_socketserver
-
-
-class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
-    allow_reuse_address = True
-
-
-advapi32 = ctypes.windll.advapi32
-
-SC_MANAGER_ALL_ACCESS = 0xf003f
-SC_MANAGER_CREATE_SERVICE = 0x02
-SERVICE_WIN32_OWN_PROCESS = 0x10
-SERVICE_AUTO_START = 0x2
-SERVICE_ERROR_NORMAL = 0x1
-DELETE = 0x00010000
-SERVICE_STATUS_START_PENDING = 0x00000002
-SERVICE_STATUS_RUNNING = 0x00000004
-SERVICE_ACCEPT_STOP = 0x1
-
-SVCNAME = 'youtubedl_builder'
-
-LPTSTR = ctypes.c_wchar_p
-START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
-
-
-class SERVICE_TABLE_ENTRY(ctypes.Structure):
-    _fields_ = [
-        ('lpServiceName', LPTSTR),
-        ('lpServiceProc', START_CALLBACK)
-    ]
-
-
-HandlerEx = ctypes.WINFUNCTYPE(
-    ctypes.c_int,     # return
-    ctypes.c_int,     # dwControl
-    ctypes.c_int,     # dwEventType
-    ctypes.c_void_p,  # lpEventData,
-    ctypes.c_void_p,  # lpContext,
-)
-
-
-def _ctypes_array(c_type, py_array):
-    ar = (c_type * len(py_array))()
-    ar[:] = py_array
-    return ar
-
-
-def win_OpenSCManager():
-    res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
-    if not res:
-        raise Exception('Opening service manager failed - '
-                        'are you running this as administrator?')
-    return res
-
-
-def win_install_service(service_name, cmdline):
-    manager = win_OpenSCManager()
-    try:
-        h = advapi32.CreateServiceW(
-            manager, service_name, None,
-            SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
-            SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
-            cmdline, None, None, None, None, None)
-        if not h:
-            raise OSError('Service creation failed: %s' % ctypes.FormatError())
-
-        advapi32.CloseServiceHandle(h)
-    finally:
-        advapi32.CloseServiceHandle(manager)
-
-
-def win_uninstall_service(service_name):
-    manager = win_OpenSCManager()
-    try:
-        h = advapi32.OpenServiceW(manager, service_name, DELETE)
-        if not h:
-            raise OSError('Could not find service %s: %s' % (
-                service_name, ctypes.FormatError()))
-
-        try:
-            if not advapi32.DeleteService(h):
-                raise OSError('Deletion failed: %s' % ctypes.FormatError())
-        finally:
-            advapi32.CloseServiceHandle(h)
-    finally:
-        advapi32.CloseServiceHandle(manager)
-
-
-def win_service_report_event(service_name, msg, is_error=True):
-    with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
-        f.write(msg + '\n')
-
-    event_log = advapi32.RegisterEventSourceW(None, service_name)
-    if not event_log:
-        raise OSError('Could not report event: %s' % ctypes.FormatError())
-
-    try:
-        type_id = 0x0001 if is_error else 0x0004
-        event_id = 0xc0000000 if is_error else 0x40000000
-        lines = _ctypes_array(LPTSTR, [msg])
-
-        if not advapi32.ReportEventW(
-                event_log, type_id, 0, event_id, None, len(lines), 0,
-                lines, None):
-            raise OSError('Event reporting failed: %s' % ctypes.FormatError())
-    finally:
-        advapi32.DeregisterEventSource(event_log)
-
-
-def win_service_handler(stop_event, *args):
-    try:
-        raise ValueError('Handler called with args ' + repr(args))
-        TODO
-    except Exception as e:
-        tb = traceback.format_exc()
-        msg = str(e) + '\n' + tb
-        win_service_report_event(service_name, msg, is_error=True)
-        raise
-
-
-def win_service_set_status(handle, status_code):
-    svcStatus = SERVICE_STATUS()
-    svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
-    svcStatus.dwCurrentState = status_code
-    svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
-
-    svcStatus.dwServiceSpecificExitCode = 0
-
-    if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
-        raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
-
-
-def win_service_main(service_name, real_main, argc, argv_raw):
-    try:
-        # args = [argv_raw[i].value for i in range(argc)]
-        stop_event = threading.Event()
-        handler = HandlerEx(functools.partial(stop_event, win_service_handler))
-        h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
-        if not h:
-            raise OSError('Handler registration failed: %s' %
-                          ctypes.FormatError())
-
-        TODO
-    except Exception as e:
-        tb = traceback.format_exc()
-        msg = str(e) + '\n' + tb
-        win_service_report_event(service_name, msg, is_error=True)
-        raise
-
-
-def win_service_start(service_name, real_main):
-    try:
-        cb = START_CALLBACK(
-            functools.partial(win_service_main, service_name, real_main))
-        dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
-            SERVICE_TABLE_ENTRY(
-                service_name,
-                cb
-            ),
-            SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
-        ])
-
-        if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
-            raise OSError('ctypes start failed: %s' % ctypes.FormatError())
-    except Exception as e:
-        tb = traceback.format_exc()
-        msg = str(e) + '\n' + tb
-        win_service_report_event(service_name, msg, is_error=True)
-        raise
-
-
-def main(args=None):
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-i', '--install',
-                        action='store_const', dest='action', const='install',
-                        help='Launch at Windows startup')
-    parser.add_argument('-u', '--uninstall',
-                        action='store_const', dest='action', const='uninstall',
-                        help='Remove Windows service')
-    parser.add_argument('-s', '--service',
-                        action='store_const', dest='action', const='service',
-                        help='Run as a Windows service')
-    parser.add_argument('-b', '--bind', metavar='<host:port>',
-                        action='store', default='0.0.0.0:8142',
-                        help='Bind to host:port (default %default)')
-    options = parser.parse_args(args=args)
-
-    if options.action == 'install':
-        fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
-        cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
-        win_install_service(SVCNAME, cmdline)
-        return
-
-    if options.action == 'uninstall':
-        win_uninstall_service(SVCNAME)
-        return
-
-    if options.action == 'service':
-        win_service_start(SVCNAME, main)
-        return
-
-    host, port_str = options.bind.split(':')
-    port = int(port_str)
-
-    print('Listening on %s:%d' % (host, port))
-    srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
-    thr = threading.Thread(target=srv.serve_forever)
-    thr.start()
-    compat_input('Press ENTER to shut down')
-    srv.shutdown()
-    thr.join()
-
-
-def rmtree(path):
-    for name in os.listdir(path):
-        fname = os.path.join(path, name)
-        if os.path.isdir(fname):
-            rmtree(fname)
-        else:
-            os.chmod(fname, 0o666)
-            os.remove(fname)
-    os.rmdir(path)
-
-
-class BuildError(Exception):
-    def __init__(self, output, code=500):
-        self.output = output
-        self.code = code
-
-    def __str__(self):
-        return self.output
-
-
-class HTTPError(BuildError):
-    pass
-
-
-class PythonBuilder(object):
-    def __init__(self, **kwargs):
-        python_version = kwargs.pop('python', '3.4')
-        python_path = None
-        for node in ('Wow6432Node\\', ''):
-            try:
-                key = compat_winreg.OpenKey(
-                    compat_winreg.HKEY_LOCAL_MACHINE,
-                    r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
-                try:
-                    python_path, _ = compat_winreg.QueryValueEx(key, '')
-                finally:
-                    compat_winreg.CloseKey(key)
-                break
-            except Exception:
-                pass
-
-        if not python_path:
-            raise BuildError('No such Python version: %s' % python_version)
-
-        self.pythonPath = python_path
-
-        super(PythonBuilder, self).__init__(**kwargs)
-
-
-class GITInfoBuilder(object):
-    def __init__(self, **kwargs):
-        try:
-            self.user, self.repoName = kwargs['path'][:2]
-            self.rev = kwargs.pop('rev')
-        except ValueError:
-            raise BuildError('Invalid path')
-        except KeyError as e:
-            raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
-
-        path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
-        if not os.path.exists(path):
-            os.makedirs(path)
-        self.basePath = tempfile.mkdtemp(dir=path)
-        self.buildPath = os.path.join(self.basePath, 'build')
-
-        super(GITInfoBuilder, self).__init__(**kwargs)
-
-
-class GITBuilder(GITInfoBuilder):
-    def build(self):
-        try:
-            subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
-            subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
-        except subprocess.CalledProcessError as e:
-            raise BuildError(e.output)
-
-        super(GITBuilder, self).build()
-
-
-class YoutubeDLBuilder(object):
-    authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org']
-
-    def __init__(self, **kwargs):
-        if self.repoName != 'yt-dlp':
-            raise BuildError('Invalid repository "%s"' % self.repoName)
-        if self.user not in self.authorizedUsers:
-            raise HTTPError('Unauthorized user "%s"' % self.user, 401)
-
-        super(YoutubeDLBuilder, self).__init__(**kwargs)
-
-    def build(self):
-        try:
-            proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
-            proc.wait()
-            #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
-            #                        cwd=self.buildPath)
-        except subprocess.CalledProcessError as e:
-            raise BuildError(e.output)
-
-        super(YoutubeDLBuilder, self).build()
-
-
-class DownloadBuilder(object):
-    def __init__(self, **kwargs):
-        self.handler = kwargs.pop('handler')
-        self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
-        self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
-        if not self.srcPath.startswith(self.buildPath):
-            raise HTTPError(self.srcPath, 401)
-
-        super(DownloadBuilder, self).__init__(**kwargs)
-
-    def build(self):
-        if not os.path.exists(self.srcPath):
-            raise HTTPError('No such file', 404)
-        if os.path.isdir(self.srcPath):
-            raise HTTPError('Is a directory: %s' % self.srcPath, 401)
-
-        self.handler.send_response(200)
-        self.handler.send_header('Content-Type', 'application/octet-stream')
-        self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
-        self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
-        self.handler.end_headers()
-
-        with open(self.srcPath, 'rb') as src:
-            shutil.copyfileobj(src, self.handler.wfile)
-
-        super(DownloadBuilder, self).build()
-
-
-class CleanupTempDir(object):
-    def build(self):
-        try:
-            rmtree(self.basePath)
-        except Exception as e:
-            print('WARNING deleting "%s": %s' % (self.basePath, e))
-
-        super(CleanupTempDir, self).build()
-
-
-class Null(object):
-    def __init__(self, **kwargs):
-        pass
-
-    def start(self):
-        pass
-
-    def close(self):
-        pass
-
-    def build(self):
-        pass
-
-
-class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
-    pass
-
-
-class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
-    actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.
-
-    def do_GET(self):
-        path = compat_urlparse.urlparse(self.path)
-        paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
-        action, _, path = path.path.strip('/').partition('/')
-        if path:
-            path = path.split('/')
-            if action in self.actionDict:
-                try:
-                    builder = self.actionDict[action](path=path, handler=self, **paramDict)
-                    builder.start()
-                    try:
-                        builder.build()
-                    finally:
-                        builder.close()
-                except BuildError as e:
-                    self.send_response(e.code)
-                    msg = compat_str(e).encode('UTF-8')
-                    self.send_header('Content-Type', 'text/plain; charset=UTF-8')
-                    self.send_header('Content-Length', len(msg))
-                    self.end_headers()
-                    self.wfile.write(msg)
-            else:
-                self.send_response(500, 'Unknown build method "%s"' % action)
-        else:
-            self.send_response(500, 'Malformed URL')
-
-if __name__ == '__main__':
-    main()

devscripts/check-porn.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 """
 This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
 if we are not 'age_limit' tagging some porn site
@@ -12,11 +10,12 @@ pass the list filename as the only argument
 # Allow direct execution
 import os
 import sys
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import gettestcases
-from yt_dlp.utils import compat_urllib_parse_urlparse
-from yt_dlp.utils import compat_urllib_request
+
+from yt_dlp.utils import compat_urllib_parse_urlparse, compat_urllib_request
 
 if len(sys.argv) > 1:
     METHOD = 'LIST'
@@ -29,7 +28,7 @@ for test in gettestcases():
         try:
             webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
         except Exception:
-            print('\nFail: {0}'.format(test['name']))
+            print('\nFail: {}'.format(test['name']))
             continue
 
         webpage = webpage.decode('utf8', 'replace')
@@ -39,7 +38,7 @@ for test in gettestcases():
     elif METHOD == 'LIST':
         domain = compat_urllib_parse_urlparse(test['url']).netloc
         if not domain:
-            print('\nFail: {0}'.format(test['name']))
+            print('\nFail: {}'.format(test['name']))
             continue
         domain = '.'.join(domain.split('.')[-2:])
 
@@ -47,11 +46,11 @@ for test in gettestcases():
 
     if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
                    or test['info_dict']['age_limit'] != 18):
-        print('\nPotential missing age_limit check: {0}'.format(test['name']))
+        print('\nPotential missing age_limit check: {}'.format(test['name']))
 
     elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
                          and test['info_dict']['age_limit'] == 18):
-        print('\nPotential false negative: {0}'.format(test['name']))
+        print('\nPotential false negative: {}'.format(test['name']))
 
     else:
         sys.stdout.write('.')

devscripts/create-github-release.py

@@ -1,112 +0,0 @@
-# Unused
-
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import io
-import json
-import mimetypes
-import netrc
-import optparse
-import os
-import re
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from yt_dlp.compat import (
-    compat_basestring,
-    compat_getpass,
-    compat_print,
-    compat_urllib_request,
-)
-from yt_dlp.utils import (
-    make_HTTPS_handler,
-    sanitized_Request,
-)
-
-
-class GitHubReleaser(object):
-    _API_URL = 'https://api.github.com/repos/ytdl-org/youtube-dl/releases'
-    _UPLOADS_URL = 'https://uploads.github.com/repos/ytdl-org/youtube-dl/releases/%s/assets?name=%s'
-    _NETRC_MACHINE = 'github.com'
-
-    def __init__(self, debuglevel=0):
-        self._init_github_account()
-        https_handler = make_HTTPS_handler({}, debuglevel=debuglevel)
-        self._opener = compat_urllib_request.build_opener(https_handler)
-
-    def _init_github_account(self):
-        try:
-            info = netrc.netrc().authenticators(self._NETRC_MACHINE)
-            if info is not None:
-                self._token = info[2]
-                compat_print('Using GitHub credentials found in .netrc...')
-                return
-            else:
-                compat_print('No GitHub credentials found in .netrc')
-        except (IOError, netrc.NetrcParseError):
-            compat_print('Unable to parse .netrc')
-        self._token = compat_getpass(
-            'Type your GitHub PAT (personal access token) and press [Return]: ')
-
-    def _call(self, req):
-        if isinstance(req, compat_basestring):
-            req = sanitized_Request(req)
-        req.add_header('Authorization', 'token %s' % self._token)
-        response = self._opener.open(req).read().decode('utf-8')
-        return json.loads(response)
-
-    def list_releases(self):
-        return self._call(self._API_URL)
-
-    def create_release(self, tag_name, name=None, body='', draft=False, prerelease=False):
-        data = {
-            'tag_name': tag_name,
-            'target_commitish': 'master',
-            'name': name,
-            'body': body,
-            'draft': draft,
-            'prerelease': prerelease,
-        }
-        req = sanitized_Request(self._API_URL, json.dumps(data).encode('utf-8'))
-        return self._call(req)
-
-    def create_asset(self, release_id, asset):
-        asset_name = os.path.basename(asset)
-        url = self._UPLOADS_URL % (release_id, asset_name)
-        # Our files are small enough to be loaded directly into memory.
-        data = open(asset, 'rb').read()
-        req = sanitized_Request(url, data)
-        mime_type, _ = mimetypes.guess_type(asset_name)
-        req.add_header('Content-Type', mime_type or 'application/octet-stream')
-        return self._call(req)
-
-
-def main():
-    parser = optparse.OptionParser(usage='%prog CHANGELOG VERSION BUILDPATH')
-    options, args = parser.parse_args()
-    if len(args) != 3:
-        parser.error('Expected a version and a build directory')
-
-    changelog_file, version, build_path = args
-
-    with io.open(changelog_file, encoding='utf-8') as inf:
-        changelog = inf.read()
-
-    mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog)
-    body = mobj.group(1) if mobj else ''
-
-    releaser = GitHubReleaser()
-
-    new_release = releaser.create_release(
-        version, name='yt-dlp %s' % version, body=body)
-    release_id = new_release['id']
-
-    for asset in os.listdir(build_path):
-        compat_print('Uploading %s...' % asset)
-        releaser.create_asset(release_id, os.path.join(build_path, asset))
-
-
-if __name__ == '__main__':
-    main()

devscripts/fish-completion.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import optparse
 import os
-from os.path import dirname as dirn
 import sys
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import yt_dlp
 from yt_dlp.utils import shell_quote
 
@@ -46,5 +44,5 @@ def build_completion(opt_parser):
         f.write(filled_template)
 
 
-parser = yt_dlp.parseOpts()[0]
+parser = yt_dlp.parseOpts(ignore_config_files=True)[0]
 build_completion(parser)

devscripts/generate_aes_testdata.py

@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import codecs
-import subprocess
-
 import os
+import subprocess
 import sys
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from yt_dlp.utils import intlist_to_bytes
 from yt_dlp.aes import aes_encrypt, key_expansion
+from yt_dlp.utils import intlist_to_bytes
 
 secret_msg = b'Secret message goes here'
 

devscripts/gh-pages.unused/add-version.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import json
-import sys
-import hashlib
-import os.path
-
-
-if len(sys.argv) <= 1:
-    print('Specify the version number as parameter')
-    sys.exit()
-version = sys.argv[1]
-
-with open('update/LATEST_VERSION', 'w') as f:
-    f.write(version)
-
-versions_info = json.load(open('update/versions.json'))
-if 'signature' in versions_info:
-    del versions_info['signature']
-
-new_version = {}
-
-filenames = {
-    'bin': 'yt-dlp',
-    'exe': 'yt-dlp.exe',
-    'tar': 'yt-dlp-%s.tar.gz' % version}
-build_dir = os.path.join('..', '..', 'build', version)
-for key, filename in filenames.items():
-    url = 'https://yt-dl.org/downloads/%s/%s' % (version, filename)
-    fn = os.path.join(build_dir, filename)
-    with open(fn, 'rb') as f:
-        data = f.read()
-    if not data:
-        raise ValueError('File %s is empty!' % fn)
-    sha256sum = hashlib.sha256(data).hexdigest()
-    new_version[key] = (url, sha256sum)
-
-versions_info['versions'][version] = new_version
-versions_info['latest'] = version
-
-with open('update/versions.json', 'w') as jsonf:
-    json.dump(versions_info, jsonf, indent=4, sort_keys=True)

devscripts/gh-pages.unused/generate-download.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import json
-
-versions_info = json.load(open('update/versions.json'))
-version = versions_info['latest']
-version_dict = versions_info['versions'][version]
-
-# Read template page
-with open('download.html.in', 'r', encoding='utf-8') as tmplf:
-    template = tmplf.read()
-
-template = template.replace('@PROGRAM_VERSION@', version)
-template = template.replace('@PROGRAM_URL@', version_dict['bin'][0])
-template = template.replace('@PROGRAM_SHA256SUM@', version_dict['bin'][1])
-template = template.replace('@EXE_URL@', version_dict['exe'][0])
-template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1])
-template = template.replace('@TAR_URL@', version_dict['tar'][0])
-template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1])
-with open('download.html', 'w', encoding='utf-8') as dlf:
-    dlf.write(template)

devscripts/gh-pages.unused/sign-versions.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import unicode_literals, with_statement
-
-import rsa
-import json
-from binascii import hexlify
-
-try:
-    input = raw_input
-except NameError:
-    pass
-
-versions_info = json.load(open('update/versions.json'))
-if 'signature' in versions_info:
-    del versions_info['signature']
-
-print('Enter the PKCS1 private key, followed by a blank line:')
-privkey = b''
-while True:
-    try:
-        line = input()
-    except EOFError:
-        break
-    if line == '':
-        break
-    privkey += line.encode('ascii') + b'\n'
-privkey = rsa.PrivateKey.load_pkcs1(privkey)
-
-signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
-print('signature: ' + signature)
-
-versions_info['signature'] = signature
-with open('update/versions.json', 'w') as versionsf:
-    json.dump(versions_info, versionsf, indent=4, sort_keys=True)

devscripts/gh-pages.unused/update-copyright.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import with_statement, unicode_literals
-
-import datetime
-import glob
-import io  # For Python 2 compatibility
-import os
-import re
-
-year = str(datetime.datetime.now().year)
-for fn in glob.glob('*.html*'):
-    with io.open(fn, encoding='utf-8') as f:
-        content = f.read()
-    newc = re.sub(r'(?P<copyright>Copyright © 2011-)(?P<year>[0-9]{4})', 'Copyright © 2011-' + year, content)
-    if content != newc:
-        tmpFn = fn + '.part'
-        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
-            outf.write(newc)
-        os.rename(tmpFn, fn)

devscripts/gh-pages.unused/update-feed.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import datetime
-import io
-import json
-import textwrap
-
-
-atom_template = textwrap.dedent("""\
-    <?xml version="1.0" encoding="utf-8"?>
-    <feed xmlns="http://www.w3.org/2005/Atom">
-        <link rel="self" href="http://ytdl-org.github.io/youtube-dl/update/releases.atom" />
-        <title>yt-dlp releases</title>
-        <id>https://yt-dl.org/feed/yt-dlp-updates-feed</id>
-        <updated>@TIMESTAMP@</updated>
-        @ENTRIES@
-    </feed>""")
-
-entry_template = textwrap.dedent("""
-    <entry>
-        <id>https://yt-dl.org/feed/yt-dlp-updates-feed/yt-dlp-@VERSION@</id>
-        <title>New version @VERSION@</title>
-        <link href="http://ytdl-org.github.io/yt-dlp" />
-        <content type="xhtml">
-            <div xmlns="http://www.w3.org/1999/xhtml">
-                Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a>
-            </div>
-        </content>
-        <author>
-            <name>The yt-dlp maintainers</name>
-        </author>
-        <updated>@TIMESTAMP@</updated>
-    </entry>
-    """)
-
-now = datetime.datetime.now()
-now_iso = now.isoformat() + 'Z'
-
-atom_template = atom_template.replace('@TIMESTAMP@', now_iso)
-
-versions_info = json.load(open('update/versions.json'))
-versions = list(versions_info['versions'].keys())
-versions.sort()
-
-entries = []
-for v in versions:
-    fields = v.split('.')
-    year, month, day = map(int, fields[:3])
-    faked = 0
-    patchlevel = 0
-    while True:
-        try:
-            datetime.date(year, month, day)
-        except ValueError:
-            day -= 1
-            faked += 1
-            assert day > 0
-            continue
-        break
-    if len(fields) >= 4:
-        try:
-            patchlevel = int(fields[3])
-        except ValueError:
-            patchlevel = 1
-    timestamp = '%04d-%02d-%02dT00:%02d:%02dZ' % (year, month, day, faked, patchlevel)
-
-    entry = entry_template.replace('@TIMESTAMP@', timestamp)
-    entry = entry.replace('@VERSION@', v)
-    entries.append(entry)
-
-entries_str = textwrap.indent(''.join(entries), '\t')
-atom_template = atom_template.replace('@ENTRIES@', entries_str)
-
-with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
-    atom_file.write(atom_template)

devscripts/gh-pages.unused/update-sites.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import sys
-import os
-import textwrap
-
-# We must be able to import yt_dlp
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
-
-import yt_dlp
-
-
-def main():
-    with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
-        template = tmplf.read()
-
-    ie_htmls = []
-    for ie in yt_dlp.list_extractors(age_limit=None):
-        ie_html = '<b>{}</b>'.format(ie.IE_NAME)
-        ie_desc = getattr(ie, 'IE_DESC', None)
-        if ie_desc is False:
-            continue
-        elif ie_desc is not None:
-            ie_html += ': {}'.format(ie.IE_DESC)
-        if not ie.working():
-            ie_html += ' (Currently broken)'
-        ie_htmls.append('<li>{}</li>'.format(ie_html))
-
-    template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))
-
-    with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
-        sitesf.write(template)
-
-
-if __name__ == '__main__':
-    main()

devscripts/lazy_load_template.py

@@ -1,31 +1,33 @@
-# coding: utf-8
+import importlib
+import random
 import re
 
-from ..utils import bug_reports_message, write_string
+from ..utils import (
+    age_restricted,
+    bug_reports_message,
+    classproperty,
+    write_string,
+)
 
 
 class LazyLoadMetaClass(type):
     def __getattr__(cls, name):
-        if '_real_class' not in cls.__dict__:
+        # "_TESTS" bloat the lazy_extractors
+        if '_real_class' not in cls.__dict__ and name != 'get_testcases':
             write_string(
-                f'WARNING: Falling back to normal extractor since lazy extractor '
-                f'{cls.__name__} does not have attribute {name}{bug_reports_message()}')
-        return getattr(cls._get_real_class(), name)
+                'WARNING: Falling back to normal extractor since lazy extractor '
+                f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n')
+        return getattr(cls.real_class, name)
 
 
 class LazyLoadExtractor(metaclass=LazyLoadMetaClass):
-    _module = None
-    _WORKING = True
-
-    @classmethod
-    def _get_real_class(cls):
+    @classproperty
+    def real_class(cls):
         if '_real_class' not in cls.__dict__:
-            mod = __import__(cls._module, fromlist=(cls.__name__,))
-            cls._real_class = getattr(mod, cls.__name__)
+            cls._real_class = getattr(importlib.import_module(cls._module), cls.__name__)
         return cls._real_class
 
     def __new__(cls, *args, **kwargs):
-        real_cls = cls._get_real_class()
-        instance = real_cls.__new__(real_cls)
+        instance = cls.real_class.__new__(cls.real_class)
         instance.__init__(*args, **kwargs)
         return instance

devscripts/make_contributing.py

@@ -1,7 +1,4 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import io
 import optparse
 import re
 
@@ -16,7 +13,7 @@ def main():
 
     infile, outfile = args
 
-    with io.open(infile, encoding='utf-8') as inf:
+    with open(infile, encoding='utf-8') as inf:
         readme = inf.read()
 
     bug_text = re.search(
@@ -26,7 +23,7 @@ def main():
 
     out = bug_text + dev_text
 
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
+    with open(outfile, 'w', encoding='utf-8') as outf:
         outf.write(out)
 
 

devscripts/make_issue_template.py

@@ -1,10 +1,19 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import io
 import optparse
 
 
+def read(fname):
+    with open(fname, encoding='utf-8') as f:
+        return f.read()
+
+
+# Get the version from yt_dlp/version.py without importing the package
+def read_version(fname):
+    exec(compile(read(fname), fname, 'exec'))
+    return locals()['__version__']
+
+
 def main():
     parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
     options, args = parser.parse_args()
@@ -12,18 +21,10 @@ def main():
         parser.error('Expected an input and an output filename')
 
     infile, outfile = args
+    with open(outfile, 'w', encoding='utf-8') as outf:
+        outf.write(
+            read(infile) % {'version': read_version('yt_dlp/version.py')})
 
-    with io.open(infile, encoding='utf-8') as inf:
-        issue_template_tmpl = inf.read()
-
-    # Get the version from yt_dlp/version.py without importing the package
-    exec(compile(open('yt_dlp/version.py').read(),
-                 'yt_dlp/version.py', 'exec'))
-
-    out = issue_template_tmpl % {'version': locals()['__version__']}
-
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(out)
 
 if __name__ == '__main__':
     main()

devscripts/make_lazy_extractors.py

@@ -1,105 +1,125 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals, print_function
-
-from inspect import getsource
-import io
+import optparse
 import os
-from os.path import dirname as dirn
 import sys
+from inspect import getsource
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py'
-if os.path.exists(lazy_extractors_filename):
-    os.remove(lazy_extractors_filename)
 
-# Block plugins from loading
-plugins_dirname = 'ytdlp_plugins'
-plugins_blocked_dirname = 'ytdlp_plugins_blocked'
-if os.path.exists(plugins_dirname):
-    os.rename(plugins_dirname, plugins_blocked_dirname)
-
-from yt_dlp.extractor import _ALL_CLASSES
-from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor
-
-if os.path.exists(plugins_blocked_dirname):
-    os.rename(plugins_blocked_dirname, plugins_dirname)
-
-with open('devscripts/lazy_load_template.py', 'rt') as f:
-    module_template = f.read()
-
-CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id']
-module_contents = [
-    module_template,
-    *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES],
-    '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n']
-
-ie_template = '''
+NO_ATTR = object()
+STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit']
+CLASS_METHODS = [
+    'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable'
+]
+IE_TEMPLATE = '''
 class {name}({bases}):
-    _module = '{module}'
+    _module = {module!r}
 '''
+with open('devscripts/lazy_load_template.py', encoding='utf-8') as f:
+    MODULE_TEMPLATE = f.read()
 
 
-def get_base_name(base):
-    if base is InfoExtractor:
-        return 'LazyLoadExtractor'
-    elif base is SearchInfoExtractor:
-        return 'LazyLoadSearchExtractor'
-    else:
-        return base.__name__
+def main():
+    parser = optparse.OptionParser(usage='%prog [OUTFILE.py]')
+    args = parser.parse_args()[1] or ['yt_dlp/extractor/lazy_extractors.py']
+    if len(args) != 1:
+        parser.error('Expected only an output filename')
+
+    lazy_extractors_filename = args[0]
+    if os.path.exists(lazy_extractors_filename):
+        os.remove(lazy_extractors_filename)
+
+    _ALL_CLASSES = get_all_ies()  # Must be before import
+
+    from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor
+
+    DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR})
+    module_src = '\n'.join((
+        MODULE_TEMPLATE,
+        '    _module = None',
+        *extra_ie_code(DummyInfoExtractor),
+        '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n',
+        *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor),
+    ))
+
+    with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
+        f.write(f'{module_src}\n')
 
 
-def build_lazy_ie(ie, name):
-    s = ie_template.format(
-        name=name,
-        bases=', '.join(map(get_base_name, ie.__bases__)),
-        module=ie.__module__)
+def get_all_ies():
+    PLUGINS_DIRNAME = 'ytdlp_plugins'
+    BLOCKED_DIRNAME = f'{PLUGINS_DIRNAME}_blocked'
+    if os.path.exists(PLUGINS_DIRNAME):
+        os.rename(PLUGINS_DIRNAME, BLOCKED_DIRNAME)
+    try:
+        from yt_dlp.extractor import _ALL_CLASSES
+    finally:
+        if os.path.exists(BLOCKED_DIRNAME):
+            os.rename(BLOCKED_DIRNAME, PLUGINS_DIRNAME)
+    return _ALL_CLASSES
+
+
+def extra_ie_code(ie, base=None):
+    for var in STATIC_CLASS_PROPERTIES:
+        val = getattr(ie, var)
+        if val != (getattr(base, var) if base else NO_ATTR):
+            yield f'    {var} = {val!r}'
+    yield ''
+
+    for name in CLASS_METHODS:
+        f = getattr(ie, name)
+        if not base or f.__func__ != getattr(base, name).__func__:
+            yield getsource(f)
+
+
+def build_ies(ies, bases, attr_base):
+    names = []
+    for ie in sort_ies(ies, bases):
+        yield build_lazy_ie(ie, ie.__name__, attr_base)
+        if ie in ies:
+            names.append(ie.__name__)
+
+    yield f'\n_ALL_CLASSES = [{", ".join(names)}]'
+
+
+def sort_ies(ies, ignored_bases):
+    """find the correct sorting and add the required base classes so that subclasses can be correctly created"""
+    classes, returned_classes = ies[:-1], set()
+    assert ies[-1].__name__ == 'GenericIE', 'Last IE must be GenericIE'
+    while classes:
+        for c in classes[:]:
+            bases = set(c.__bases__) - {object, *ignored_bases}
+            restart = False
+            for b in bases:
+                if b not in classes and b not in returned_classes:
+                    assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE'
+                    classes.insert(0, b)
+                    restart = True
+            if restart:
+                break
+            if bases <= returned_classes:
+                yield c
+                returned_classes.add(c)
+                classes.remove(c)
+                break
+    yield ies[-1]
+
+
+def build_lazy_ie(ie, name, attr_base):
+    bases = ', '.join({
+        'InfoExtractor': 'LazyLoadExtractor',
+        'SearchInfoExtractor': 'LazyLoadSearchExtractor',
+    }.get(base.__name__, base.__name__) for base in ie.__bases__)
+
+    s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases)
     valid_url = getattr(ie, '_VALID_URL', None)
     if not valid_url and hasattr(ie, '_make_valid_url'):
         valid_url = ie._make_valid_url()
     if valid_url:
         s += f'    _VALID_URL = {valid_url!r}\n'
-    if not ie._WORKING:
-        s += '    _WORKING = False\n'
-    if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
-        s += f'\n{getsource(ie.suitable)}'
-    return s
+    return s + '\n'.join(extra_ie_code(ie, attr_base))
 
 
-# find the correct sorting and add the required base classes so that subclasses
-# can be correctly created
-classes = _ALL_CLASSES[:-1]
-ordered_cls = []
-while classes:
-    for c in classes[:]:
-        bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor))
-        stop = False
-        for b in bases:
-            if b not in classes and b not in ordered_cls:
-                if b.__name__ == 'GenericIE':
-                    exit()
-                classes.insert(0, b)
-                stop = True
-        if stop:
-            break
-        if all(b in ordered_cls for b in bases):
-            ordered_cls.append(c)
-            classes.remove(c)
-            break
-ordered_cls.append(_ALL_CLASSES[-1])
-
-names = []
-for ie in ordered_cls:
-    name = ie.__name__
-    src = build_lazy_ie(ie, name)
-    module_contents.append(src)
-    if ie in _ALL_CLASSES:
-        names.append(name)
-
-module_contents.append(
-    '\n_ALL_CLASSES = [{0}]'.format(', '.join(names)))
-
-module_src = '\n'.join(module_contents) + '\n'
-
-with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
-    f.write(module_src)
+if __name__ == '__main__':
+    main()

devscripts/make_readme.py

@@ -2,30 +2,29 @@
 
 # yt-dlp --help | make_readme.py
 # This must be run in a console of correct width
-
-from __future__ import unicode_literals
-
-import io
-import sys
 import re
+import sys
 
 README_FILE = 'README.md'
+
+OPTIONS_START = 'General Options:'
+OPTIONS_END = 'CONFIGURATION'
+EPILOG_START = 'See full documentation'
+
+
 helptext = sys.stdin.read()
-
 if isinstance(helptext, bytes):
-    helptext = helptext.decode('utf-8')
+    helptext = helptext.decode()
 
-with io.open(README_FILE, encoding='utf-8') as f:
-    oldreadme = f.read()
+start, end = helptext.index(f'\n  {OPTIONS_START}'), helptext.index(f'\n{EPILOG_START}')
+options = re.sub(r'(?m)^  (\w.+)$', r'## \1', helptext[start + 1: end + 1])
 
-header = oldreadme[:oldreadme.index('## General Options:')]
-footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
+with open(README_FILE, encoding='utf-8') as f:
+    readme = f.read()
 
-options = helptext[helptext.index('  General Options:'):]
-options = re.sub(r'(?m)^  (\w.+)$', r'## \1', options)
-options = options + '\n'
+header = readme[:readme.index(f'## {OPTIONS_START}')]
+footer = readme[readme.index(f'# {OPTIONS_END}'):]
 
-with io.open(README_FILE, 'w', encoding='utf-8') as f:
-    f.write(header)
-    f.write(options)
-    f.write(footer)
+with open(README_FILE, 'w', encoding='utf-8') as f:
+    for part in (header, options, footer):
+        f.write(part)

devscripts/make_supportedsites.py

@@ -1,47 +1,23 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import io
 import optparse
 import os
 import sys
 
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Import yt_dlp
-ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
-sys.path.insert(0, ROOT_DIR)
-import yt_dlp
+from yt_dlp.extractor import list_extractor_classes
 
 
 def main():
     parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    options, args = parser.parse_args()
+    _, args = parser.parse_args()
     if len(args) != 1:
         parser.error('Expected an output filename')
 
-    outfile, = args
+    out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False)
 
-    def gen_ies_md(ies):
-        for ie in ies:
-            ie_md = '**{0}**'.format(ie.IE_NAME)
-            if ie.IE_DESC is False:
-                continue
-            if ie.IE_DESC is not None:
-                ie_md += ': {0}'.format(ie.IE_DESC)
-            search_key = getattr(ie, 'SEARCH_KEY', None)
-            if search_key is not None:
-                ie_md += f'; "{ie.SEARCH_KEY}:" prefix'
-            if not ie.working():
-                ie_md += ' (Currently broken)'
-            yield ie_md
-
-    ies = sorted(yt_dlp.gen_extractors(), key=lambda i: i.IE_NAME.lower())
-    out = '# Supported sites\n' + ''.join(
-        ' - ' + md + '\n'
-        for md in gen_ies_md(ies))
-
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(out)
+    with open(args[0], 'w', encoding='utf-8') as outf:
+        outf.write(f'# Supported sites\n{out}\n')
 
 
 if __name__ == '__main__':

devscripts/posix-locale.sh

@@ -1,6 +0,0 @@
-
-# source this file in your shell to get a POSIX locale (which will break many programs, but that's kind of the point)
-
-export LC_ALL=POSIX
-export LANG=POSIX
-export LANGUAGE=POSIX

devscripts/prepare_manpage.py

@@ -1,7 +1,4 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import io
 import optparse
 import os.path
 import re
@@ -32,14 +29,14 @@ def main():
 
     outfile, = args
 
-    with io.open(README_FILE, encoding='utf-8') as f:
+    with open(README_FILE, encoding='utf-8') as f:
         readme = f.read()
 
     readme = filter_excluded_sections(readme)
     readme = move_sections(readme)
     readme = filter_options(readme)
 
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
+    with open(outfile, 'w', encoding='utf-8') as outf:
         outf.write(PREFIX + readme)
 
 

devscripts/release.sh

@@ -1,143 +0,0 @@
-# Unused
-
-#!/bin/bash
-
-# IMPORTANT: the following assumptions are made
-# * the GH repo is on the origin remote
-# * the gh-pages branch is named so locally
-# * the git config user.signingkey is properly set
-
-# You will need
-# pip install coverage nose rsa wheel
-
-# TODO
-# release notes
-# make hash on local files
-
-set -e
-
-skip_tests=true
-gpg_sign_commits=""
-buildserver='localhost:8142'
-
-while true
-do
-case "$1" in
-    --run-tests)
-        skip_tests=false
-        shift
-    ;;
-    --gpg-sign-commits|-S)
-        gpg_sign_commits="-S"
-        shift
-    ;;
-    --buildserver)
-        buildserver="$2"
-        shift 2
-    ;;
-    --*)
-        echo "ERROR: unknown option $1"
-        exit 1
-    ;;
-    *)
-        break
-    ;;
-esac
-done
-
-if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
-version="$1"
-major_version=$(echo "$version" | sed -n 's#^\([0-9]*\.[0-9]*\.[0-9]*\).*#\1#p')
-if test "$major_version" '!=' "$(date '+%Y.%m.%d')"; then
-    echo "$version does not start with today's date!"
-    exit 1
-fi
-
-if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
-if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
-useless_files=$(find yt_dlp -type f -not -name '*.py')
-if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in yt_dlp: $useless_files"; exit 1; fi
-if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
-if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi
-if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi
-if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi
-
-read -p "Is Changelog up to date? (y/n) " -n 1
-if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
-
-/bin/echo -e "\n### First of all, testing..."
-make clean
-if $skip_tests ; then
-    echo 'SKIPPING TESTS'
-else
-    nosetests --verbose --with-coverage --cover-package=yt_dlp --cover-html test --stop || exit 1
-fi
-
-/bin/echo -e "\n### Changing version in version.py..."
-sed -i "s/__version__ = '.*'/__version__ = '$version'/" yt_dlp/version.py
-
-/bin/echo -e "\n### Changing version in Changelog..."
-sed -i "s/<unreleased>/$version/" Changelog.md
-
-/bin/echo -e "\n### Committing documentation, templates and yt_dlp/version.py..."
-make README.md CONTRIBUTING.md issuetemplates supportedsites
-git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md yt_dlp/version.py Changelog.md
-git commit $gpg_sign_commits -m "release $version"
-
-/bin/echo -e "\n### Now tagging, signing and pushing..."
-git tag -s -m "Release $version" "$version"
-git show "$version"
-read -p "Is it good, can I push? (y/n) " -n 1
-if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
-echo
-MASTER=$(git rev-parse --abbrev-ref HEAD)
-git push origin $MASTER:master
-git push origin "$version"
-
-/bin/echo -e "\n### OK, now it is time to build the binaries..."
-REV=$(git rev-parse HEAD)
-make yt-dlp yt-dlp.tar.gz
-read -p "VM running? (y/n) " -n 1
-wget "http://$buildserver/build/ytdl-org/youtube-dl/yt-dlp.exe?rev=$REV" -O yt-dlp.exe
-mkdir -p "build/$version"
-mv yt-dlp yt-dlp.exe "build/$version"
-mv yt-dlp.tar.gz "build/$version/yt-dlp-$version.tar.gz"
-RELEASE_FILES="yt-dlp yt-dlp.exe yt-dlp-$version.tar.gz"
-(cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS)
-(cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS)
-(cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
-(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
-
-/bin/echo -e "\n### Signing and uploading the new binaries to GitHub..."
-for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done
-
-ROOT=$(pwd)
-python devscripts/create-github-release.py Changelog.md $version "$ROOT/build/$version"
-
-ssh ytdl@yt-dl.org "sh html/update_latest.sh $version"
-
-/bin/echo -e "\n### Now switching to gh-pages..."
-git clone --branch gh-pages --single-branch . build/gh-pages
-(
-    set -e
-    ORIGIN_URL=$(git config --get remote.origin.url)
-    cd build/gh-pages
-    "$ROOT/devscripts/gh-pages/add-version.py" $version
-    "$ROOT/devscripts/gh-pages/update-feed.py"
-    "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
-    "$ROOT/devscripts/gh-pages/generate-download.py"
-    "$ROOT/devscripts/gh-pages/update-copyright.py"
-    "$ROOT/devscripts/gh-pages/update-sites.py"
-    git add *.html *.html.in update
-    git commit $gpg_sign_commits -m "release $version"
-    git push "$ROOT" gh-pages
-    git push "$ORIGIN_URL" gh-pages
-)
-rm -rf build
-
-make pypi-files
-echo "Uploading to PyPi ..."
-python setup.py sdist bdist_wheel upload
-make clean
-
-/bin/echo -e "\n### DONE!"

devscripts/run_tests.bat

@@ -13,4 +13,5 @@ if ["%~1"]==[""] (
     exit /b 1
 )
 
+set PYTHONWARNINGS=error
 pytest %test_set%

devscripts/run_tests.sh

@@ -11,4 +11,4 @@ else
     exit 1
 fi
 
-python3 -m pytest "$test_set"
+python3 -bb -Werror -m pytest "$test_set"

devscripts/show-downloads-statistics.py

@@ -1,49 +0,0 @@
-# Unused
-
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-import itertools
-import json
-import os
-import re
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from yt_dlp.compat import (
-    compat_print,
-    compat_urllib_request,
-)
-from yt_dlp.utils import format_bytes
-
-
-def format_size(bytes):
-    return '%s (%d bytes)' % (format_bytes(bytes), bytes)
-
-
-total_bytes = 0
-
-for page in itertools.count(1):
-    releases = json.loads(compat_urllib_request.urlopen(
-        'https://api.github.com/repos/ytdl-org/youtube-dl/releases?page=%s' % page
-    ).read().decode('utf-8'))
-
-    if not releases:
-        break
-
-    for release in releases:
-        compat_print(release['name'])
-        for asset in release['assets']:
-            asset_name = asset['name']
-            total_bytes += asset['download_count'] * asset['size']
-            if all(not re.match(p, asset_name) for p in (
-                    r'^yt-dlp$',
-                    r'^yt-dlp-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$',
-                    r'^yt-dlp\.exe$')):
-                continue
-            compat_print(
-                ' %s size: %s downloads: %d'
-                % (asset_name, format_size(asset['size']), asset['download_count']))
-
-compat_print('total downloads traffic: %s' % format_size(total_bytes))

devscripts/update-formulae.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import json
 import os
 import re
@@ -10,7 +8,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from yt_dlp.compat import compat_urllib_request
 
-
 # usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
 # version can be either 0-aligned (yt-dlp version) or normalized (PyPl version)
 
@@ -20,14 +17,14 @@ normalized_version = '.'.join(str(int(x)) for x in version.split('.'))
 
 pypi_release = json.loads(compat_urllib_request.urlopen(
     'https://pypi.org/pypi/yt-dlp/%s/json' % normalized_version
-).read().decode('utf-8'))
+).read().decode())
 
 tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('.tar.gz'))
 
 sha256sum = tarball_file['digests']['sha256']
 url = tarball_file['url']
 
-with open(filename, 'r') as r:
+with open(filename) as r:
     formulae_text = r.read()
 
 formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text)

devscripts/update-version.py

@@ -1,10 +1,9 @@
 #!/usr/bin/env python3
-from datetime import datetime
-import sys
 import subprocess
+import sys
+from datetime import datetime
 
-
-with open('yt_dlp/version.py', 'rt') as f:
+with open('yt_dlp/version.py') as f:
     exec(compile(f.read(), 'yt_dlp/version.py', 'exec'))
 old_version = locals()['__version__']
 

devscripts/wine-py2exe.sh

@@ -1,58 +0,0 @@
-# UNUSED
-
-#!/bin/bash
-
-# Run with as parameter a setup.py that works in the current directory
-# e.g. no os.chdir()
-# It will run twice, the first time will crash
-
-set -e
-
-SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
-
-if [ ! -d wine-py2exe ]; then
-
-    sudo apt-get install wine1.3 axel bsdiff
-
-    mkdir wine-py2exe
-    cd wine-py2exe
-    export WINEPREFIX=`pwd`
-
-    axel -a "http://www.python.org/ftp/python/2.7/python-2.7.msi"
-    axel -a "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe"
-    #axel -a "http://winetricks.org/winetricks"
-
-    # http://appdb.winehq.org/objectManager.php?sClass=version&iId=21957
-    echo "Follow python setup on screen"
-    wine msiexec /i python-2.7.msi
-    
-    echo "Follow py2exe setup on screen"
-    wine py2exe-0.6.9.win32-py2.7.exe
-    
-    #echo "Follow Microsoft Visual C++ 2008 Redistributable Package setup on screen"
-    #bash winetricks vcrun2008
-
-    rm py2exe-0.6.9.win32-py2.7.exe
-    rm python-2.7.msi
-    #rm winetricks
-    
-    # http://bugs.winehq.org/show_bug.cgi?id=3591
-    
-    mv drive_c/Python27/Lib/site-packages/py2exe/run.exe drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup
-    bspatch drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run.exe "$SCRIPT_DIR/SizeOfImage.patch"
-    mv drive_c/Python27/Lib/site-packages/py2exe/run_w.exe drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup
-    bspatch drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run_w.exe "$SCRIPT_DIR/SizeOfImage_w.patch"
-
-    cd -
-    
-else
-
-    export WINEPREFIX="$( cd wine-py2exe && pwd )"
-
-fi
-
-wine "C:\\Python27\\python.exe" "$1" py2exe > "py2exe.log" 2>&1 || true
-echo '# Copying python27.dll' >> "py2exe.log"
-cp "$WINEPREFIX/drive_c/windows/system32/python27.dll" build/bdist.win32/winexe/bundle-2.7/
-wine "C:\\Python27\\python.exe" "$1" py2exe >> "py2exe.log" 2>&1
-

devscripts/zsh-completion.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import os
-from os.path import dirname as dirn
 import sys
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import yt_dlp
 
 ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp"
@@ -45,5 +43,5 @@ def build_completion(opt_parser):
         f.write(template)
 
 
-parser = yt_dlp.parseOpts()[0]
+parser = yt_dlp.parseOpts(ignore_config_files=True)[0]
 build_completion(parser)

pyinst.py

@@ -1,32 +1,41 @@
 #!/usr/bin/env python3
-# coding: utf-8
 import os
 import platform
 import sys
-from PyInstaller.utils.hooks import collect_submodules
 
+from PyInstaller.__main__ import run as run_pyinstaller
 
 OS_NAME = platform.system()
 if OS_NAME == 'Windows':
     from PyInstaller.utils.win32.versioninfo import (
-        VarStruct, VarFileInfo, StringStruct, StringTable,
-        StringFileInfo, FixedFileInfo, VSVersionInfo, SetVersion,
+        FixedFileInfo,
+        SetVersion,
+        StringFileInfo,
+        StringStruct,
+        StringTable,
+        VarFileInfo,
+        VarStruct,
+        VSVersionInfo,
     )
 elif OS_NAME == 'Darwin':
     pass
 else:
-    raise Exception('{OS_NAME} is not supported')
+    raise Exception(f'{OS_NAME} is not supported')
 
 ARCH = platform.architecture()[0][:2]
 
 
 def main():
     opts = parse_options()
-    version = read_version()
+    version = read_version('yt_dlp/version.py')
 
-    suffix = '_macos' if OS_NAME == 'Darwin' else '_x86' if ARCH == '32' else ''
-    final_file = 'dist/%syt-dlp%s%s' % (
-        'yt-dlp/' if '--onedir' in opts else '', suffix, '.exe' if OS_NAME == 'Windows' else '')
+    onedir = '--onedir' in opts or '-D' in opts
+    if not onedir and '-F' not in opts and '--onefile' not in opts:
+        opts.append('--onefile')
+
+    name = 'yt-dlp%s' % ('_macos' if OS_NAME == 'Darwin' else '_x86' if ARCH == '32' else '')
+    final_file = ''.join((
+        'dist/', f'{name}/' if onedir else '', name, '.exe' if OS_NAME == 'Windows' else ''))
 
     print(f'Building yt-dlp v{version} {ARCH}bit for {OS_NAME} with options {opts}')
     print('Remember to update the version using  "devscripts/update-version.py"')
@@ -36,20 +45,20 @@ def main():
     print(f'Destination: {final_file}\n')
 
     opts = [
-        f'--name=yt-dlp{suffix}',
+        f'--name={name}',
         '--icon=devscripts/logo.ico',
         '--upx-exclude=vcruntime140.dll',
         '--noconfirm',
+        # NB: Modules that are only imported dynamically must be added here.
+        # --collect-submodules may not work correctly if user has a yt-dlp installed via PIP
+        '--hidden-import=yt_dlp.compat._legacy',
         *dependency_options(),
         *opts,
         'yt_dlp/__main__.py',
     ]
+
     print(f'Running PyInstaller with {opts}')
-
-    import PyInstaller.__main__
-
-    PyInstaller.__main__.run(opts)
-
+    run_pyinstaller(opts)
     set_version_info(final_file, version)
 
 
@@ -60,12 +69,14 @@ def parse_options():
         if ARCH != opts[0]:
             raise Exception(f'{opts[0]}bit executable cannot be built on a {ARCH}bit system')
         opts = opts[1:]
-    return opts or ['--onefile']
+    return opts
 
 
-def read_version():
-    exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec'))
-    return locals()['__version__']
+# Get the version from yt_dlp/version.py without importing the package
+def read_version(fname):
+    with open(fname, encoding='utf-8') as f:
+        exec(compile(f.read(), fname, 'exec'))
+        return locals()['__version__']
 
 
 def version_to_list(version):
@@ -74,10 +85,12 @@ def version_to_list(version):
 
 
 def dependency_options():
-    dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi'] + collect_submodules('websockets')
-    excluded_modules = ['test', 'ytdlp_plugins', 'youtube-dl', 'youtube-dlc']
+    # Due to the current implementation, these are auto-detected, but explicitly add them just in case
+    dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi', 'websockets']
+    excluded_modules = ['test', 'ytdlp_plugins', 'youtube_dl', 'youtube_dlc']
 
     yield from (f'--hidden-import={module}' for module in dependencies)
+    yield '--collect-submodules=websockets'
     yield from (f'--exclude-module={module}' for module in excluded_modules)
 
 

setup.cfg

@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = yt_dlp/extractor/__init__.py,devscripts/buildserver.py,devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git,venv,devscripts/create-github-release.py,devscripts/release.sh,devscripts/show-downloads-statistics.py
-ignore = E402,E501,E731,E741,W503
+exclude = devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git,venv
+ignore = E402,E501,E731,E741,W503

setup.py

@@ -1,29 +1,38 @@
 #!/usr/bin/env python3
-# coding: utf-8
 import os.path
-import warnings
 import sys
+import warnings
 
 try:
-    from setuptools import setup, Command, find_packages
+    from setuptools import Command, find_packages, setup
     setuptools_available = True
 except ImportError:
-    from distutils.core import setup, Command
+    from distutils.core import Command, setup
     setuptools_available = False
 from distutils.spawn import spawn
 
-# Get the version from yt_dlp/version.py without importing the package
-exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec'))
 
+def read(fname):
+    with open(fname, encoding='utf-8') as f:
+        return f.read()
+
+
+# Get the version from yt_dlp/version.py without importing the package
+def read_version(fname):
+    exec(compile(read(fname), fname, 'exec'))
+    return locals()['__version__']
+
+
+VERSION = read_version('yt_dlp/version.py')
 
 DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
 LONG_DESCRIPTION = '\n\n'.join((
     'Official repository: <https://github.com/yt-dlp/yt-dlp>',
     '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
-    open('README.md', encoding='utf-8').read()))
+    read('README.md')))
 
-REQUIREMENTS = open('requirements.txt', encoding='utf-8').read().splitlines()
+REQUIREMENTS = read('requirements.txt').splitlines()
 
 
 if sys.argv[1:2] == ['py2exe']:
@@ -35,11 +44,11 @@ if sys.argv[1:2] == ['py2exe']:
         'console': [{
             'script': './yt_dlp/__main__.py',
             'dest_base': 'yt-dlp',
-            'version': __version__,
+            'version': VERSION,
             'description': DESCRIPTION,
             'comments': LONG_DESCRIPTION.split('\n')[0],
             'product_name': 'yt-dlp',
-            'product_version': __version__,
+            'product_version': VERSION,
         }],
         'options': {
             'py2exe': {
@@ -49,6 +58,8 @@ if sys.argv[1:2] == ['py2exe']:
                 'dist_dir': './dist',
                 'excludes': ['Crypto', 'Cryptodome'],  # py2exe cannot import Crypto
                 'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
+                # Modules that are only imported dynamically must be added here
+                'includes': ['yt_dlp.compat._legacy'],
             }
         },
         'zipfile': None
@@ -106,7 +117,7 @@ else:
 
 setup(
     name='yt-dlp',
-    version=__version__,
+    version=VERSION,
     maintainer='pukkandan',
     maintainer_email='pukkandan.ytdlp@gmail.com',
     description=DESCRIPTION,
@@ -116,7 +127,7 @@ setup(
     packages=packages,
     install_requires=REQUIREMENTS,
     project_urls={
-        'Documentation': 'https://yt-dlp.readthedocs.io',
+        'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
         'Source': 'https://github.com/yt-dlp/yt-dlp',
         'Tracker': 'https://github.com/yt-dlp/yt-dlp/issues',
         'Funding': 'https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators',

supportedsites.md

@@ -24,12 +24,12 @@
  - **abcnews:video**
  - **abcotvs**: ABC Owned Television Stations
  - **abcotvs:clips**
- - **AbemaTV**
+ - **AbemaTV**: [<abbr title="netrc machine"><em>abematv</em></abbr>]
  - **AbemaTVTitle**
  - **AcademicEarth:Course**
  - **acast**
  - **acast:channel**
- - **ADN**: Anime Digital Network
+ - **ADN**: [<abbr title="netrc machine"><em>animedigitalnetwork</em></abbr>] Anime Digital Network
  - **AdobeConnect**
  - **adobetv**
  - **adobetv:channel**
@@ -40,8 +40,8 @@
  - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault
  - **aenetworks:collection**
  - **aenetworks:show**
- - **afreecatv**: afreecatv.com
- - **afreecatv:live**: afreecatv.com
+ - **afreecatv**: [<abbr title="netrc machine"><em>afreecatv</em></abbr>] afreecatv.com
+ - **afreecatv:live**: [<abbr title="netrc machine"><em>afreecatv</em></abbr>] afreecatv.com
  - **afreecatv:user**
  - **AirMozilla**
  - **AliExpressLive**
@@ -50,8 +50,8 @@
  - **AlphaPorno**
  - **Alsace20TV**
  - **Alsace20TVEmbed**
- - **Alura**
- - **AluraCourse**
+ - **Alura**: [<abbr title="netrc machine"><em>alura</em></abbr>]
+ - **AluraCourse**: [<abbr title="netrc machine"><em>aluracourse</em></abbr>]
  - **Amara**
  - **AmazonStore**
  - **AMCNetworks**
@@ -60,9 +60,9 @@
  - **AmHistoryChannel**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimalPlanet**
- - **AnimeLab**
- - **AnimeLabShows**
- - **AnimeOnDemand**
+ - **AnimeLab**: [<abbr title="netrc machine"><em>animelab</em></abbr>]
+ - **AnimeLabShows**: [<abbr title="netrc machine"><em>animelab</em></abbr>]
+ - **AnimeOnDemand**: [<abbr title="netrc machine"><em>animeondemand</em></abbr>]
  - **ant1newsgr:article**: ant1news.gr articles
  - **ant1newsgr:embed**: ant1news.gr embedded videos
  - **ant1newsgr:watch**: ant1news.gr videos
@@ -88,7 +88,7 @@
  - **ArteTVPlaylist**
  - **AsianCrush**
  - **AsianCrushPlaylist**
- - **AtresPlayer**
+ - **AtresPlayer**: [<abbr title="netrc machine"><em>atresplayer</em></abbr>]
  - **ATTTechChannel**
  - **ATVAt**
  - **AudiMedia**
@@ -114,13 +114,13 @@
  - **Bandcamp:weekly**
  - **bangumi.bilibili.com**: BiliBili番剧
  - **BannedVideo**
- - **bbc**: BBC
- - **bbc.co.uk**: BBC iPlayer
+ - **bbc**: [<abbr title="netrc machine"><em>bbc</em></abbr>] BBC
+ - **bbc.co.uk**: [<abbr title="netrc machine"><em>bbc</em></abbr>] BBC iPlayer
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:iplayer:episodes**
  - **bbc.co.uk:iplayer:group**
  - **bbc.co.uk:playlist**
- - **BBVTV**
+ - **BBVTV**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
  - **Beatport**
  - **Beeg**
  - **BehindKink**
@@ -142,8 +142,9 @@
  - **BilibiliChannel**
  - **BiliBiliPlayer**
  - **BiliBiliSearch**: Bilibili video search; "bilisearch:" prefix
- - **BiliIntl**
- - **BiliIntlSeries**
+ - **BiliIntl**: [<abbr title="netrc machine"><em>biliintl</em></abbr>]
+ - **BiliIntlSeries**: [<abbr title="netrc machine"><em>biliintl</em></abbr>]
+ - **BiliLive**
  - **BioBioChileTV**
  - **Biography**
  - **BIQLE**
@@ -252,10 +253,10 @@
  - **CrooksAndLiars**
  - **CrowdBunker**
  - **CrowdBunkerChannel**
- - **crunchyroll**
- - **crunchyroll:beta**
- - **crunchyroll:playlist**
- - **crunchyroll:playlist:beta**
+ - **crunchyroll**: [<abbr title="netrc machine"><em>crunchyroll</em></abbr>]
+ - **crunchyroll:beta**: [<abbr title="netrc machine"><em>crunchyroll</em></abbr>]
+ - **crunchyroll:playlist**: [<abbr title="netrc machine"><em>crunchyroll</em></abbr>]
+ - **crunchyroll:playlist:beta**: [<abbr title="netrc machine"><em>crunchyroll</em></abbr>]
  - **CSpan**: C-SPAN
  - **CSpanCongress**
  - **CtsNews**: 華視新聞
@@ -263,18 +264,18 @@
  - **CTVNews**
  - **cu.ntv.co.jp**: Nippon Television Network
  - **CultureUnplugged**
- - **curiositystream**
- - **curiositystream:collections**
- - **curiositystream:series**
+ - **curiositystream**: [<abbr title="netrc machine"><em>curiositystream</em></abbr>]
+ - **curiositystream:collections**: [<abbr title="netrc machine"><em>curiositystream</em></abbr>]
+ - **curiositystream:series**: [<abbr title="netrc machine"><em>curiositystream</em></abbr>]
  - **CWTV**
- - **Cybrary**
- - **CybraryCourse**
+ - **Cybrary**: [<abbr title="netrc machine"><em>cybrary</em></abbr>]
+ - **CybraryCourse**: [<abbr title="netrc machine"><em>cybrary</em></abbr>]
  - **Daftsex**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
- - **dailymotion**
- - **dailymotion:playlist**
- - **dailymotion:user**
+ - **dailymotion**: [<abbr title="netrc machine"><em>dailymotion</em></abbr>]
+ - **dailymotion:playlist**: [<abbr title="netrc machine"><em>dailymotion</em></abbr>]
+ - **dailymotion:user**: [<abbr title="netrc machine"><em>dailymotion</em></abbr>]
  - **damtomo:record**
  - **damtomo:video**
  - **daum.net**
@@ -291,7 +292,7 @@
  - **DestinationAmerica**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Digg**
- - **DigitalConcertHall**: DigitalConcertHall extractor
+ - **DigitalConcertHall**: [<abbr title="netrc machine"><em>digitalconcerthall</em></abbr>] DigitalConcertHall extractor
  - **DigitallySpeaking**
  - **Digiteka**
  - **Discovery**
@@ -315,7 +316,7 @@
  - **DRBonanza**
  - **Drooble**
  - **Dropbox**
- - **Dropout**
+ - **Dropout**: [<abbr title="netrc machine"><em>dropout</em></abbr>]
  - **DropoutSeason**
  - **DrTuber**
  - **drtv**
@@ -334,7 +335,7 @@
  - **egghead:lesson**: egghead.io lesson
  - **ehftv**
  - **eHow**
- - **EinsUndEinsTV**
+ - **EinsUndEinsTV**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
  - **Einthusan**
  - **eitb.tv**
  - **EllenTube**
@@ -348,7 +349,7 @@
  - **Epicon**
  - **EpiconSeries**
  - **Eporner**
- - **EroProfile**
+ - **EroProfile**: [<abbr title="netrc machine"><em>eroprofile</em></abbr>]
  - **EroProfile:album**
  - **ertflix**: ERTFLIX videos
  - **ertflix:codename**: ERTFLIX videos by codename
@@ -361,20 +362,21 @@
  - **Europa**
  - **EuropeanTour**
  - **EUScreen**
- - **EWETV**
+ - **EWETV**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
  - **ExpoTV**
  - **Expressen**
  - **ExtremeTube**
  - **EyedoTV**
- - **facebook**
+ - **facebook**: [<abbr title="netrc machine"><em>facebook</em></abbr>]
  - **FacebookPluginsVideo**
- - **fancode:live**
- - **fancode:vod**
+ - **fancode:live**: [<abbr title="netrc machine"><em>fancode</em></abbr>]
+ - **fancode:vod**: [<abbr title="netrc machine"><em>fancode</em></abbr>]
  - **faz.net**
- - **fc2**
+ - **fc2**: [<abbr title="netrc machine"><em>fc2</em></abbr>]
  - **fc2:embed**
  - **fc2:live**
  - **Fczenit**
+ - **Fifa**
  - **Filmmodu**
  - **filmon**
  - **filmon:channel**
@@ -401,19 +403,19 @@
  - **FranceTVSite**
  - **Freesound**
  - **freespeech.org**
- - **FrontendMasters**
- - **FrontendMastersCourse**
- - **FrontendMastersLesson**
+ - **FrontendMasters**: [<abbr title="netrc machine"><em>frontendmasters</em></abbr>]
+ - **FrontendMastersCourse**: [<abbr title="netrc machine"><em>frontendmasters</em></abbr>]
+ - **FrontendMastersLesson**: [<abbr title="netrc machine"><em>frontendmasters</em></abbr>]
  - **FujiTVFODPlus7**
- - **Funimation**
- - **funimation:page**
- - **funimation:show**
+ - **Funimation**: [<abbr title="netrc machine"><em>funimation</em></abbr>]
+ - **funimation:page**: [<abbr title="netrc machine"><em>funimation</em></abbr>]
+ - **funimation:show**: [<abbr title="netrc machine"><em>funimation</em></abbr>]
  - **Funk**
  - **Fusion**
  - **Fux**
  - **Gab**
  - **GabTV**
- - **Gaia**
+ - **Gaia**: [<abbr title="netrc machine"><em>gaia</em></abbr>]
  - **GameInformer**
  - **GameJolt**
  - **GameJoltCommunity**
@@ -425,20 +427,19 @@
  - **GameStar**
  - **Gaskrank**
  - **Gazeta**
- - **GDCVault**
+ - **GDCVault**: [<abbr title="netrc machine"><em>gdcvault</em></abbr>]
  - **GediDigital**
- - **gem.cbc.ca**
+ - **gem.cbc.ca**: [<abbr title="netrc machine"><em>cbcgem</em></abbr>]
  - **gem.cbc.ca:live**
  - **gem.cbc.ca:playlist**
- - **generic**: Generic downloader that works on some sites
  - **Gettr**
  - **GettrStreaming**
  - **Gfycat**
  - **GiantBomb**
  - **Giga**
- - **GlattvisionTV**
+ - **GlattvisionTV**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
  - **Glide**: Glide mobile video messages (glide.me)
- - **Globo**
+ - **Globo**: [<abbr title="netrc machine"><em>globo</em></abbr>]
  - **GloboArticle**
  - **glomex**: Glomex videos
  - **glomex:embed**: Glomex embedded videos
@@ -447,6 +448,7 @@
  - **GodTube**
  - **Gofile**
  - **Golem**
+ - **goodgame:stream**
  - **google:podcasts**
  - **google:podcasts:feed**
  - **GoogleDrive**
@@ -455,6 +457,8 @@
  - **GoToStage**
  - **GPUTechConf**
  - **Gronkh**
+ - **gronkh:feed**
+ - **gronkh:vods**
  - **Groupon**
  - **hbo**
  - **HearThisAt**
@@ -466,7 +470,7 @@
  - **hgtv.com:show**
  - **HGTVDe**
  - **HGTVUsa**
- - **HiDive**
+ - **HiDive**: [<abbr title="netrc machine"><em>hidive</em></abbr>]
  - **HistoricFilms**
  - **history:player**
  - **history:topic**: History.com Topic
@@ -481,8 +485,8 @@
  - **Howcast**
  - **HowStuffWorks**
  - **hrfernsehen**
- - **HRTi**
- - **HRTiPlaylist**
+ - **HRTi**: [<abbr title="netrc machine"><em>hrti</em></abbr>]
+ - **HRTiPlaylist**: [<abbr title="netrc machine"><em>hrti</em></abbr>]
  - **HSEProduct**
  - **HSEShow**
  - **Huajiao**: 花椒直播
@@ -492,6 +496,7 @@
  - **HungamaSong**
  - **huya:live**: huya.com
  - **Hypem**
+ - **Icareus**
  - **ign.com**
  - **IGNArticle**
  - **IGNVideo**
@@ -506,19 +511,19 @@
  - **Inc**
  - **IndavideoEmbed**
  - **InfoQ**
- - **Instagram**
- - **instagram:story**
- - **instagram:tag**: Instagram hashtag search URLs
- - **instagram:user**: Instagram user profile
+ - **Instagram**: [<abbr title="netrc machine"><em>instagram</em></abbr>]
+ - **instagram:story**: [<abbr title="netrc machine"><em>instagram</em></abbr>]
+ - **instagram:tag**: [<abbr title="netrc machine"><em>instagram</em></abbr>] Instagram hashtag search URLs
+ - **instagram:user**: [<abbr title="netrc machine"><em>instagram</em></abbr>] Instagram user profile
  - **InstagramIOS**: IOS instagram:// URL
  - **Internazionale**
  - **InternetVideoArchive**
  - **InvestigationDiscovery**
- - **IPrima**
+ - **IPrima**: [<abbr title="netrc machine"><em>iprima</em></abbr>]
  - **IPrimaCNN**
  - **iq.com**: International version of iQiyi
  - **iq.com:album**
- - **iqiyi**: 爱奇艺
+ - **iqiyi**: [<abbr title="netrc machine"><em>iqiyi</em></abbr>] 爱奇艺
  - **ITProTV**
  - **ITProTVCourse**
  - **ITTF**
@@ -528,6 +533,8 @@
  - **ivi:compilation**: ivi.ru compilations
  - **ivideon**: Ivideon TV
  - **Iwara**
+ - **iwara:playlist**
+ - **iwara:user**
  - **Izlesene**
  - **Jable**
  - **JablePlaylist**
@@ -576,9 +583,9 @@
  - **LcpPlay**
  - **Le**: 乐视网
  - **Lecture2Go**
- - **Lecturio**
- - **LecturioCourse**
- - **LecturioDeCourse**
+ - **Lecturio**: [<abbr title="netrc machine"><em>lecturio</em></abbr>]
+ - **LecturioCourse**: [<abbr title="netrc machine"><em>lecturio</em></abbr>]
+ - **LecturioDeCourse**: [<abbr title="netrc machine"><em>lecturio</em></abbr>]
  - **LEGO**
  - **Lemonde**
  - **Lenta**
@@ -587,15 +594,17 @@
  - **Libsyn**
  - **life**: Life.ru
  - **life:embed**
+ - **likee**
+ - **likee:user**
  - **limelight**
  - **limelight:channel**
  - **limelight:channel_list**
  - **LineLive**
  - **LineLiveChannel**
- - **LinkedIn**
- - **linkedin:learning**
- - **linkedin:learning:course**
- - **LinuxAcademy**
+ - **LinkedIn**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
+ - **linkedin:learning**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
+ - **linkedin:learning:course**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
+ - **LinuxAcademy**: [<abbr title="netrc machine"><em>linuxacademy</em></abbr>]
  - **LiTV**
  - **LiveJournal**
  - **livestream**
@@ -605,9 +614,10 @@
  - **loc**: Library of Congress
  - **LocalNews8**
  - **LoveHomePorn**
- - **lrt.lt**
- - **lynda**: lynda.com videos
- - **lynda:course**: lynda.com online courses
+ - **LRTStream**
+ - **LRTVOD**
+ - **lynda**: [<abbr title="netrc machine"><em>lynda</em></abbr>] lynda.com videos
+ - **lynda:course**: [<abbr title="netrc machine"><em>lynda</em></abbr>] lynda.com online courses
  - **m6**
  - **MagentaMusik360**
  - **mailru**: Видео@Mail.Ru
@@ -626,6 +636,7 @@
  - **Markiza**
  - **MarkizaPage**
  - **massengeschmack.tv**
+ - **Masters**
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **MedalTV**
@@ -674,7 +685,7 @@
  - **MLBVideo**
  - **MLSSoccer**
  - **Mnet**
- - **MNetTV**
+ - **MNetTV**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
  - **MofosexEmbed**
@@ -746,8 +757,9 @@
  - **ndr:embed**
  - **ndr:embed:base**
  - **NDTV**
- - **Nebula**
- - **nebula:collection**
+ - **Nebula**: [<abbr title="netrc machine"><em>watchnebula</em></abbr>]
+ - **nebula:channel**: [<abbr title="netrc machine"><em>watchnebula</em></abbr>]
+ - **nebula:subscriptions**: [<abbr title="netrc machine"><em>watchnebula</em></abbr>]
  - **NerdCubedFeed**
  - **netease:album**: 网易云音乐 - 专辑
  - **netease:djradio**: 网易云音乐 - 电台
@@ -756,7 +768,7 @@
  - **netease:program**: 网易云音乐 - 电台节目
  - **netease:singer**: 网易云音乐 - 歌手
  - **netease:song**: 网易云音乐
- - **NetPlus**
+ - **NetPlus**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
  - **Netzkino**
  - **Newgrounds**
  - **Newgrounds:playlist**
@@ -770,8 +782,8 @@
  - **NexxEmbed**
  - **NFB**
  - **NFHSNetwork**
- - **nfl.com** (Currently broken)
- - **nfl.com:article** (Currently broken)
+ - **nfl.com**: (**Currently broken**)
+ - **nfl.com:article**: (**Currently broken**)
  - **NhkForSchoolBangumi**
  - **NhkForSchoolProgramList**
  - **NhkForSchoolSubject**: Portal page for each school subjects, like Japanese (kokugo, 国語) or math (sansuu/suugaku or 算数・数学)
@@ -783,7 +795,7 @@
  - **nickelodeon:br**
  - **nickelodeonru**
  - **nicknight**
- - **niconico**: ニコニコ動画
+ - **niconico**: [<abbr title="netrc machine"><em>niconico</em></abbr>] ニコニコ動画
  - **niconico:history**: NicoNico user history. Requires cookies.
  - **niconico:playlist**
  - **niconico:series**
@@ -796,7 +808,7 @@
  - **Nitter**
  - **njoy**: N-JOY
  - **njoy:embed**
- - **NJPWWorld**: 新日本プロレスワールド
+ - **NJPWWorld**: [<abbr title="netrc machine"><em>njpwworld</em></abbr>] 新日本プロレスワールド
  - **NobelPrize**
  - **NonkTube**
  - **NoodleMagazine**
@@ -869,9 +881,9 @@
  - **orf:tvthek**: ORF TVthek
  - **orf:vorarlberg**: Radio Vorarlberg
  - **orf:wien**: Radio Wien
- - **OsnatelTV**
+ - **OsnatelTV**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
  - **OutsideTV**
- - **PacktPub**
+ - **PacktPub**: [<abbr title="netrc machine"><em>packtpub</em></abbr>]
  - **PacktPubCourse**
  - **PalcoMP3:artist**
  - **PalcoMP3:song**
@@ -893,7 +905,7 @@
  - **peer.tv**
  - **PeerTube**
  - **PeerTube:Playlist**
- - **peloton**
+ - **peloton**: [<abbr title="netrc machine"><em>peloton</em></abbr>]
  - **peloton:live**: Peloton Live
  - **People**
  - **PerformGroup**
@@ -902,7 +914,7 @@
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **phoenix.de**
  - **Photobucket**
- - **Piapro**
+ - **Piapro**: [<abbr title="netrc machine"><em>piapro</em></abbr>]
  - **Picarto**
  - **PicartoVod**
  - **Piksel**
@@ -913,26 +925,27 @@
  - **pixiv:sketch:user**
  - **Pladform**
  - **PlanetMarathi**
- - **Platzi**
- - **PlatziCourse**
+ - **Platzi**: [<abbr title="netrc machine"><em>platzi</em></abbr>]
+ - **PlatziCourse**: [<abbr title="netrc machine"><em>platzi</em></abbr>]
  - **play.fm**
  - **player.sky.it**
- - **PlayPlusTV**
+ - **PlayPlusTV**: [<abbr title="netrc machine"><em>playplustv</em></abbr>]
  - **PlayStuff**
  - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **PlayVids**
  - **Playwire**
- - **pluralsight**
+ - **pluralsight**: [<abbr title="netrc machine"><em>pluralsight</em></abbr>]
  - **pluralsight:course**
  - **PlutoTV**
+ - **Podchaser**
  - **podomatic**
  - **Pokemon**
  - **PokemonSoundLibrary**
  - **PokemonWatch**
- - **PokerGo**
- - **PokerGoCollection**
+ - **PokerGo**: [<abbr title="netrc machine"><em>pokergo</em></abbr>]
+ - **PokerGoCollection**: [<abbr title="netrc machine"><em>pokergo</em></abbr>]
  - **PolsatGo**
  - **PolskieRadio**
  - **polskieradio:kierowcow**
@@ -947,11 +960,11 @@
  - **Pornez**
  - **PornFlip**
  - **PornHd**
- - **PornHub**: PornHub and Thumbzilla
- - **PornHubPagedVideoList**
- - **PornHubPlaylist**
- - **PornHubUser**
- - **PornHubUserVideosUpload**
+ - **PornHub**: [<abbr title="netrc machine"><em>pornhub</em></abbr>] PornHub and Thumbzilla
+ - **PornHubPagedVideoList**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
+ - **PornHubPlaylist**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
+ - **PornHubUser**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
+ - **PornHubUserVideosUpload**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
@@ -973,7 +986,7 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
- - **QuantumTV**
+ - **QuantumTV**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
  - **Qub**
  - **R7**
  - **R7Article**
@@ -1026,11 +1039,12 @@
  - **RICE**
  - **RMCDecouverte**
  - **RockstarGames**
- - **Rokfin**
- - **rokfin:channel**
- - **rokfin:stack**
- - **RoosterTeeth**
- - **RoosterTeethSeries**
+ - **Rokfin**: [<abbr title="netrc machine"><em>rokfin</em></abbr>]
+ - **rokfin:channel**: Rokfin Channels
+ - **rokfin:search**: Rokfin Search; "rkfnsearch:" prefix
+ - **rokfin:stack**: Rokfin Stacks
+ - **RoosterTeeth**: [<abbr title="netrc machine"><em>roosterteeth</em></abbr>]
+ - **RoosterTeethSeries**: [<abbr title="netrc machine"><em>roosterteeth</em></abbr>]
  - **RottenTomatoes**
  - **Rozhlas**
  - **RTBF**
@@ -1069,12 +1083,12 @@
  - **Ruutu**
  - **Ruv**
  - **ruv.is:spila**
- - **safari**: safaribooksonline.com online video
- - **safari:api**
- - **safari:course**: safaribooksonline.com online courses
+ - **safari**: [<abbr title="netrc machine"><em>safari</em></abbr>] safaribooksonline.com online video
+ - **safari:api**: [<abbr title="netrc machine"><em>safari</em></abbr>]
+ - **safari:course**: [<abbr title="netrc machine"><em>safari</em></abbr>] safaribooksonline.com online courses
  - **Saitosan**
- - **SAKTV**
- - **SaltTV**
+ - **SAKTV**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
+ - **SaltTV**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
  - **SampleFocus**
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
@@ -1086,8 +1100,8 @@
  - **ScreencastOMatic**
  - **ScrippsNetworks**
  - **scrippsnetworks:watch**
- - **SCTE**
- - **SCTECourse**
+ - **SCTE**: [<abbr title="netrc machine"><em>scte</em></abbr>]
+ - **SCTECourse**: [<abbr title="netrc machine"><em>scte</em></abbr>]
  - **Seeker**
  - **SenateGov**
  - **SenateISVP**
@@ -1096,7 +1110,7 @@
  - **Sexu**
  - **SeznamZpravy**
  - **SeznamZpravyArticle**
- - **Shahid**
+ - **Shahid**: [<abbr title="netrc machine"><em>shahid</em></abbr>]
  - **ShahidShow**
  - **Shared**: shared.sx
  - **ShemarooMe**
@@ -1121,15 +1135,15 @@
  - **Slutload**
  - **Snotr**
  - **Sohu**
- - **SonyLIV**
+ - **SonyLIV**: [<abbr title="netrc machine"><em>sonyliv</em></abbr>]
  - **SonyLIVSeries**
- - **soundcloud**
- - **soundcloud:playlist**
- - **soundcloud:related**
- - **soundcloud:search**: Soundcloud search; "scsearch:" prefix
- - **soundcloud:set**
- - **soundcloud:trackstation**
- - **soundcloud:user**
+ - **soundcloud**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
+ - **soundcloud:playlist**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
+ - **soundcloud:related**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
+ - **soundcloud:search**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>] Soundcloud search; "scsearch:" prefix
+ - **soundcloud:set**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
+ - **soundcloud:trackstation**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
+ - **soundcloud:user**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
  - **SoundcloudEmbed**
  - **soundgasm**
  - **soundgasm:profile**
@@ -1147,8 +1161,8 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
- - **spotify**
- - **spotify:show**
+ - **spotify**: Spotify episodes
+ - **spotify:show**: Spotify shows
  - **Spreaker**
  - **SpreakerPage**
  - **SpreakerShow**
@@ -1190,13 +1204,13 @@
  - **Tass**
  - **TBS**
  - **TDSLifeway**
- - **Teachable**
- - **TeachableCourse**
+ - **Teachable**: [<abbr title="netrc machine"><em>teachable</em></abbr>]
+ - **TeachableCourse**: [<abbr title="netrc machine"><em>teachable</em></abbr>]
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
  - **TeachingChannel**
  - **Teamcoco**
- - **TeamTreeHouse**
+ - **TeamTreeHouse**: [<abbr title="netrc machine"><em>teamtreehouse</em></abbr>]
  - **TechTalks**
  - **techtv.mit.edu**
  - **TedEmbed**
@@ -1218,8 +1232,8 @@
  - **TeleQuebecVideo**
  - **TeleTask**
  - **Telewebion**
- - **TennisTV**
- - **TenPlay**
+ - **TennisTV**: [<abbr title="netrc machine"><em>tennistv</em></abbr>]
+ - **TenPlay**: [<abbr title="netrc machine"><em>10play</em></abbr>]
  - **TF1**
  - **TFO**
  - **TheIntercept**
@@ -1250,10 +1264,10 @@
  - **Tokentube**
  - **Tokentube:channel**
  - **ToonGoggles**
- - **tou.tv**
+ - **tou.tv**: [<abbr title="netrc machine"><em>toutv</em></abbr>]
  - **Toypics**: Toypics video
  - **ToypicsUser**: Toypics user profile
- - **TrailerAddict** (Currently broken)
+ - **TrailerAddict**: (**Currently broken**)
  - **TravelChannel**
  - **Trilulilu**
  - **Trovo**
@@ -1264,9 +1278,9 @@
  - **TruNews**
  - **TruTV**
  - **Tube8**
- - **TubiTv**
+ - **TubiTv**: [<abbr title="netrc machine"><em>tubitv</em></abbr>]
  - **TubiTvShow**
- - **Tumblr**
+ - **Tumblr**: [<abbr title="netrc machine"><em>tumblr</em></abbr>]
  - **tunein:clip**
  - **tunein:program**
  - **tunein:station**
@@ -1314,23 +1328,23 @@
  - **TwitCasting**
  - **TwitCastingLive**
  - **TwitCastingUser**
- - **twitch:clips**
- - **twitch:stream**
- - **twitch:vod**
- - **TwitchCollection**
- - **TwitchVideos**
- - **TwitchVideosClips**
- - **TwitchVideosCollections**
+ - **twitch:clips**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **twitch:stream**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **twitch:vod**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **TwitchCollection**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **TwitchVideos**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **TwitchVideosClips**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **TwitchVideosCollections**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
  - **twitter**
  - **twitter:amplify**
  - **twitter:broadcast**
  - **twitter:card**
  - **twitter:shortener**
- - **udemy**
- - **udemy:course**
+ - **udemy**: [<abbr title="netrc machine"><em>udemy</em></abbr>]
+ - **udemy:course**: [<abbr title="netrc machine"><em>udemy</em></abbr>]
  - **UDNEmbed**: 聯合影音
- - **UFCArabia**
- - **UFCTV**
+ - **UFCArabia**: [<abbr title="netrc machine"><em>ufcarabia</em></abbr>]
+ - **UFCTV**: [<abbr title="netrc machine"><em>ufctv</em></abbr>]
  - **ukcolumn**
  - **UKTVPlay**
  - **umg:de**: Universal Music Deutschland
@@ -1358,7 +1372,7 @@
  - **VevoPlaylist**
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
- - **vhx:embed**
+ - **vhx:embed**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
  - **Viafree**
  - **vice**
  - **vice:article**
@@ -1370,50 +1384,49 @@
  - **video.google:search**: Google Video search; "gvsearch:" prefix
  - **video.sky.it**
  - **video.sky.it:live**
- - **VideocampusSachsen**
- - **VideocampusSachsenEmbed**
  - **VideoDetective**
  - **videofy.me**
  - **videomore**
  - **videomore:season**
  - **videomore:video**
  - **VideoPress**
- - **Vidio**
- - **VidioLive**
- - **VidioPremier**
+ - **Vidio**: [<abbr title="netrc machine"><em>vidio</em></abbr>]
+ - **VidioLive**: [<abbr title="netrc machine"><em>vidio</em></abbr>]
+ - **VidioPremier**: [<abbr title="netrc machine"><em>vidio</em></abbr>]
  - **VidLii**
- - **vier**: vier.be and vijf.be
+ - **vier**: [<abbr title="netrc machine"><em>vier</em></abbr>] vier.be and vijf.be
  - **vier:videos**
  - **viewlift**
  - **viewlift:embed**
  - **Viidea**
- - **viki**
- - **viki:channel**
- - **vimeo**
- - **vimeo:album**
- - **vimeo:channel**
- - **vimeo:group**
- - **vimeo:likes**: Vimeo user likes
- - **vimeo:ondemand**
- - **vimeo:review**: Review pages on vimeo
- - **vimeo:user**
- - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
+ - **viki**: [<abbr title="netrc machine"><em>viki</em></abbr>]
+ - **viki:channel**: [<abbr title="netrc machine"><em>viki</em></abbr>]
+ - **vimeo**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vimeo:album**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vimeo:channel**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vimeo:group**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vimeo:likes**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Vimeo user likes
+ - **vimeo:ondemand**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vimeo:review**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Review pages on vimeo
+ - **vimeo:user**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vimeo:watchlater**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication)
  - **Vimm:recording**
  - **Vimm:stream**
+ - **Vimp**
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
  - **Viqeo**
  - **Viu**
- - **viu:ott**
+ - **viu:ott**: [<abbr title="netrc machine"><em>viu</em></abbr>]
  - **viu:playlist**
  - **Vivo**: vivo.sx
- - **vk**: VK
- - **vk:uservideos**: VK - User's Videos
- - **vk:wallpost**
- - **vlive**
- - **vlive:channel**
- - **vlive:post**
+ - **vk**: [<abbr title="netrc machine"><em>vk</em></abbr>] VK
+ - **vk:uservideos**: [<abbr title="netrc machine"><em>vk</em></abbr>] VK - User's Videos
+ - **vk:wallpost**: [<abbr title="netrc machine"><em>vk</em></abbr>]
+ - **vlive**: [<abbr title="netrc machine"><em>vlive</em></abbr>]
+ - **vlive:channel**: [<abbr title="netrc machine"><em>vlive</em></abbr>]
+ - **vlive:post**: [<abbr title="netrc machine"><em>vlive</em></abbr>]
  - **vm.tiktok**
  - **Vodlocker**
  - **VODPl**
@@ -1428,12 +1441,12 @@
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
  - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
- - **VrtNU**: VrtNU.be
- - **vrv**
+ - **VrtNU**: [<abbr title="netrc machine"><em>vrtnu</em></abbr>] VrtNU.be
+ - **vrv**: [<abbr title="netrc machine"><em>vrv</em></abbr>]
  - **vrv:series**
  - **VShare**
  - **VTM**
- - **VTXTV**
+ - **VTXTV**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
  - **VuClip**
  - **Vupload**
  - **VVVVID**
@@ -1442,7 +1455,7 @@
  - **Vzaar**
  - **Wakanim**
  - **Walla**
- - **WalyTV**
+ - **WalyTV**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
  - **wasdtv:clip**
  - **wasdtv:record**
  - **wasdtv:stream**
@@ -1452,7 +1465,7 @@
  - **WatchBox**
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
- - **wdr:mobile** (Currently broken)
+ - **wdr:mobile**: (**Currently broken**)
  - **WDRElefant**
  - **WDRPage**
  - **web.archive:youtube**: web.archive.org saved youtube videos, "ytarchive:" prefix
@@ -1525,11 +1538,13 @@
  - **youtube:favorites**: YouTube liked videos; ":ytfav" keyword (requires cookies)
  - **youtube:history**: Youtube watch history; ":ythis" keyword (requires cookies)
  - **youtube:music:search_url**: YouTube music search URLs with selectable sections (Eg: #songs)
+ - **youtube:notif**: YouTube notifications; ":ytnotif" keyword (requires cookies)
  - **youtube:playlist**: YouTube playlists
  - **youtube:recommended**: YouTube recommended videos; ":ytrec" keyword
  - **youtube:search**: YouTube search; "ytsearch:" prefix
  - **youtube:search:date**: YouTube search, newest videos first; "ytsearchdate:" prefix
  - **youtube:search_url**: YouTube search URLs with sorting and filter support
+ - **youtube:stories**: YouTube channel stories; "ytstories:" prefix
  - **youtube:subscriptions**: YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)
  - **youtube:tab**: YouTube Tabs
  - **youtube:user**: YouTube user videos; "ytuser:" prefix
@@ -1537,18 +1552,23 @@
  - **YoutubeLivestreamEmbed**: YouTube livestream embeds
  - **YoutubeYtBe**: youtu.be
  - **Zapiks**
- - **Zattoo**
- - **ZattooLive**
- - **ZattooMovies**
- - **ZattooRecordings**
+ - **Zattoo**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
+ - **ZattooLive**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
+ - **ZattooMovies**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
+ - **ZattooRecordings**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
  - **ZDF**
  - **ZDFChannel**
- - **Zee5**
+ - **Zee5**: [<abbr title="netrc machine"><em>zee5</em></abbr>]
  - **zee5:series**
  - **ZenYandex**
  - **ZenYandexChannel**
  - **Zhihu**
  - **zingmp3**: zingmp3.vn
  - **zingmp3:album**
+ - **zingmp3:chart-home**
+ - **zingmp3:chart-music-video**
+ - **zingmp3:user**
+ - **zingmp3:week-chart**
  - **zoom**
  - **Zype**
+ - **generic**: Generic downloader that works on some sites

test/helper.py

@@ -1,26 +1,16 @@
-from __future__ import unicode_literals
-
 import errno
-import io
 import hashlib
 import json
 import os.path
 import re
-import types
 import ssl
 import sys
+import types
 
 import yt_dlp.extractor
 from yt_dlp import YoutubeDL
-from yt_dlp.compat import (
-    compat_os_name,
-    compat_str,
-)
-from yt_dlp.utils import (
-    preferredencoding,
-    write_string,
-)
-
+from yt_dlp.compat import compat_os_name, compat_str
+from yt_dlp.utils import preferredencoding, write_string
 
 if 'pytest' in sys.modules:
     import pytest
@@ -35,10 +25,10 @@ def get_params(override=None):
                                    'parameters.json')
     LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                          'local_parameters.json')
-    with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    with open(PARAMETERS_FILE, encoding='utf-8') as pf:
         parameters = json.load(pf)
     if os.path.exists(LOCAL_PARAMETERS_FILE):
-        with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
+        with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
             parameters.update(json.load(pf))
     if override:
         parameters.update(override)
@@ -63,8 +53,8 @@ def report_warning(message):
         _msg_header = '\033[0;33mWARNING:\033[0m'
     else:
         _msg_header = 'WARNING:'
-    output = '%s %s\n' % (_msg_header, message)
-    if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3:
+    output = f'{_msg_header} {message}\n'
+    if 'b' in getattr(sys.stderr, 'mode', ''):
         output = output.encode(preferredencoding())
     sys.stderr.write(output)
 
@@ -74,7 +64,7 @@ class FakeYDL(YoutubeDL):
         # Different instances of the downloader can't share the same dictionary
         # some test set the "sublang" parameter, which would break the md5 checks.
         params = get_params(override=override)
-        super(FakeYDL, self).__init__(params, auto_init=False)
+        super().__init__(params, auto_init=False)
         self.result = []
 
     def to_screen(self, s, skip_eol=None):
@@ -99,11 +89,10 @@ class FakeYDL(YoutubeDL):
 
 def gettestcases(include_onlymatching=False):
     for ie in yt_dlp.extractor.gen_extractors():
-        for tc in ie.get_testcases(include_onlymatching):
-            yield tc
+        yield from ie.get_testcases(include_onlymatching)
 
 
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+md5 = lambda s: hashlib.md5(s.encode()).hexdigest()
 
 
 def expect_value(self, got, expected, field):
@@ -113,33 +102,30 @@ def expect_value(self, got, expected, field):
 
         self.assertTrue(
             isinstance(got, compat_str),
-            'Expected a %s object, but got %s for field %s' % (
-                compat_str.__name__, type(got).__name__, field))
+            f'Expected a {compat_str.__name__} object, but got {type(got).__name__} for field {field}')
         self.assertTrue(
             match_rex.match(got),
-            'field %s (value: %r) should match %r' % (field, got, match_str))
+            f'field {field} (value: {got!r}) should match {match_str!r}')
     elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
         start_str = expected[len('startswith:'):]
         self.assertTrue(
             isinstance(got, compat_str),
-            'Expected a %s object, but got %s for field %s' % (
-                compat_str.__name__, type(got).__name__, field))
+            f'Expected a {compat_str.__name__} object, but got {type(got).__name__} for field {field}')
         self.assertTrue(
             got.startswith(start_str),
-            'field %s (value: %r) should start with %r' % (field, got, start_str))
+            f'field {field} (value: {got!r}) should start with {start_str!r}')
     elif isinstance(expected, compat_str) and expected.startswith('contains:'):
         contains_str = expected[len('contains:'):]
         self.assertTrue(
             isinstance(got, compat_str),
-            'Expected a %s object, but got %s for field %s' % (
-                compat_str.__name__, type(got).__name__, field))
+            f'Expected a {compat_str.__name__} object, but got {type(got).__name__} for field {field}')
         self.assertTrue(
             contains_str in got,
-            'field %s (value: %r) should contain %r' % (field, got, contains_str))
+            f'field {field} (value: {got!r}) should contain {contains_str!r}')
     elif isinstance(expected, type):
         self.assertTrue(
             isinstance(got, expected),
-            'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+            f'Expected type {expected!r} for field {field}, but got value {got!r} of type {type(got)!r}')
     elif isinstance(expected, dict) and isinstance(got, dict):
         expect_dict(self, got, expected)
     elif isinstance(expected, list) and isinstance(got, list):
@@ -159,13 +145,12 @@ def expect_value(self, got, expected, field):
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
             self.assertTrue(
                 isinstance(got, compat_str),
-                'Expected field %s to be a unicode object, but got value %r of type %r' % (field, got, type(got)))
+                f'Expected field {field} to be a unicode object, but got value {got!r} of type {type(got)!r}')
             got = 'md5:' + md5(got)
         elif isinstance(expected, compat_str) and re.match(r'^(?:min|max)?count:\d+', expected):
             self.assertTrue(
                 isinstance(got, (list, dict)),
-                'Expected field %s to be a list or a dict, but it is of type %s' % (
-                    field, type(got).__name__))
+                f'Expected field {field} to be a list or a dict, but it is of type {type(got).__name__}')
             op, _, expected_num = expected.partition(':')
             expected_num = int(expected_num)
             if op == 'mincount':
@@ -185,7 +170,7 @@ def expect_value(self, got, expected, field):
             return
         self.assertEqual(
             expected, got,
-            'Invalid value for field %s, expected %r, got %r' % (field, expected, got))
+            f'Invalid value for field {field}, expected {expected!r}, got {got!r}')
 
 
 def expect_dict(self, got_dict, expected_dict):
@@ -260,13 +245,13 @@ def expect_info_dict(self, got_dict, expected_dict):
         info_dict_str = ''
         if len(missing_keys) != len(expected_dict):
             info_dict_str += ''.join(
-                '    %s: %s,\n' % (_repr(k), _repr(v))
+                f'    {_repr(k)}: {_repr(v)},\n'
                 for k, v in test_info_dict.items() if k not in missing_keys)
 
             if info_dict_str:
                 info_dict_str += '\n'
         info_dict_str += ''.join(
-            '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
+            f'    {_repr(k)}: {_repr(test_info_dict[k])},\n'
             for k in missing_keys)
         write_string(
             '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
@@ -295,21 +280,21 @@ def assertRegexpMatches(self, text, regexp, msg=None):
 def assertGreaterEqual(self, got, expected, msg=None):
     if not (got >= expected):
         if msg is None:
-            msg = '%r not greater than or equal to %r' % (got, expected)
+            msg = f'{got!r} not greater than or equal to {expected!r}'
         self.assertTrue(got >= expected, msg)
 
 
 def assertLessEqual(self, got, expected, msg=None):
     if not (got <= expected):
         if msg is None:
-            msg = '%r not less than or equal to %r' % (got, expected)
+            msg = f'{got!r} not less than or equal to {expected!r}'
         self.assertTrue(got <= expected, msg)
 
 
 def assertEqual(self, got, expected, msg=None):
     if not (got == expected):
         if msg is None:
-            msg = '%r not equal to %r' % (got, expected)
+            msg = f'{got!r} not equal to {expected!r}'
         self.assertTrue(got == expected, msg)
 
 

test/swftests.unused/.gitignore

@@ -1 +0,0 @@
-*.swf

test/swftests.unused/ArrayAccess.as

@@ -1,19 +0,0 @@
-// input: [["a", "b", "c", "d"]]
-// output: ["c", "b", "a", "d"]
-
-package {
-public class ArrayAccess {
-    public static function main(ar:Array):Array {
-    	var aa:ArrayAccess = new ArrayAccess();
-    	return aa.f(ar, 2);
-    }
-
-    private function f(ar:Array, num:Number):Array{
-        var x:String = ar[0];
-        var y:String = ar[num % ar.length];
-        ar[0] = y;
-        ar[num] = x;
-        return ar;
-    }
-}
-}

test/swftests.unused/ClassCall.as

@@ -1,17 +0,0 @@
-// input: []
-// output: 121
-
-package {
-public class ClassCall {
-    public static function main():int{
-    	var f:OtherClass = new OtherClass();
-        return f.func(100,20);
-    }
-}
-}
-
-class OtherClass {
-	public function func(x: int, y: int):int {
-		return x+y+1;
-	}
-}

test/swftests.unused/ClassConstruction.as

@@ -1,15 +0,0 @@
-// input: []
-// output: 0
-
-package {
-public class ClassConstruction {
-    public static function main():int{
-    	var f:Foo = new Foo();
-        return 0;
-    }
-}
-}
-
-class Foo {
-
-}

test/swftests.unused/ConstArrayAccess.as

@@ -1,18 +0,0 @@
-// input: []
-// output: 4
-
-package {
-public class ConstArrayAccess {
-	private static const x:int = 2;
-	private static const ar:Array = ["42", "3411"];
-
-    public static function main():int{
-        var c:ConstArrayAccess = new ConstArrayAccess();
-        return c.f();
-    }
-
-    public function f(): int {
-    	return ar[1].length;
-    }
-}
-}

test/swftests.unused/ConstantInt.as

@@ -1,12 +0,0 @@
-// input: []
-// output: 2
-
-package {
-public class ConstantInt {
-	private static const x:int = 2;
-
-    public static function main():int{
-        return x;
-    }
-}
-}

test/swftests.unused/DictCall.as

@@ -1,10 +0,0 @@
-// input: [{"x": 1, "y": 2}]
-// output: 3
-
-package {
-public class DictCall {
-    public static function main(d:Object):int{
-        return d.x + d.y;
-    }
-}
-}

test/swftests.unused/EqualsOperator.as

@@ -1,10 +0,0 @@
-// input: []
-// output: false
-
-package {
-public class EqualsOperator {
-    public static function main():Boolean{
-        return 1 == 2;
-    }
-}
-}

test/swftests.unused/LocalVars.as

@@ -1,13 +0,0 @@
-// input: [1, 2]
-// output: 3
-
-package {
-public class LocalVars {
-    public static function main(a:int, b:int):int{
-        var c:int = a + b + b;
-        var d:int = c - b;
-        var e:int = d;
-        return e;
-    }
-}
-}

test/swftests.unused/MemberAssignment.as

@@ -1,22 +0,0 @@
-// input: [1]
-// output: 2
-
-package {
-public class MemberAssignment {
-    public var v:int;
-
-    public function g():int {
-        return this.v;
-    }
-
-    public function f(a:int):int{
-        this.v = a;
-        return this.v + this.g();
-    }
-
-    public static function main(a:int): int {
-        var v:MemberAssignment = new MemberAssignment();
-        return v.f(a);
-    }
-}
-}

test/swftests.unused/NeOperator.as

@@ -1,24 +0,0 @@
-// input: []
-// output: 123
-
-package {
-public class NeOperator {
-    public static function main(): int {
-        var res:int = 0;
-        if (1 != 2) {
-            res += 3;
-        } else {
-            res += 4;
-        }
-        if (2 != 2) {
-            res += 10;
-        } else {
-            res += 20;
-        }
-        if (9 == 9) {
-            res += 100;
-        }
-        return res;
-    }
-}
-}

test/swftests.unused/PrivateCall.as

@@ -1,21 +0,0 @@
-// input: []
-// output: 9
-
-package {
-public class PrivateCall {
-    public static function main():int{
-    	var f:OtherClass = new OtherClass();
-        return f.func();
-    }
-}
-}
-
-class OtherClass {
-	private function pf():int {
-		return 9;
-	}
-
-	public function func():int {
-		return this.pf();
-	}
-}

test/swftests.unused/PrivateVoidCall.as

@@ -1,22 +0,0 @@
-// input: []
-// output: 9
-
-package {
-public class PrivateVoidCall {
-    public static function main():int{
-        var f:OtherClass = new OtherClass();
-        f.func();
-        return 9;
-    }
-}
-}
-
-class OtherClass {
-    private function pf():void {
-        ;
-    }
-
-    public function func():void {
-        this.pf();
-    }
-}

test/swftests.unused/StaticAssignment.as

@@ -1,13 +0,0 @@
-// input: [1]
-// output: 1
-
-package {
-public class StaticAssignment {
-	public static var v:int;
-
-    public static function main(a:int):int{
-        v = a;
-        return v;
-    }
-}
-}

test/swftests.unused/StaticRetrieval.as

@@ -1,16 +0,0 @@
-// input: []
-// output: 1
-
-package {
-public class StaticRetrieval {
-	public static var v:int;
-
-    public static function main():int{
-        if (v) {
-        	return 0;
-        } else {
-        	return 1;
-        }
-    }
-}
-}

test/swftests.unused/StringBasics.as

@@ -1,11 +0,0 @@
-// input: []
-// output: 3
-
-package {
-public class StringBasics {
-    public static function main():int{
-        var s:String = "abc";
-        return s.length;
-    }
-}
-}

test/swftests.unused/StringCharCodeAt.as

@@ -1,11 +0,0 @@
-// input: []
-// output: 9897
-
-package {
-public class StringCharCodeAt {
-    public static function main():int{
-        var s:String = "abc";
-        return s.charCodeAt(1) * 100 + s.charCodeAt();
-    }
-}
-}

test/swftests.unused/StringConversion.as

@@ -1,11 +0,0 @@
-// input: []
-// output: 2
-
-package {
-public class StringConversion {
-    public static function main():int{
-        var s:String = String(99);
-        return s.length;
-    }
-}
-}

test/test_InfoExtractor.py

@@ -1,21 +1,23 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
-import io
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
-from yt_dlp.compat import compat_etree_fromstring, compat_http_server
-from yt_dlp.extractor.common import InfoExtractor
-from yt_dlp.extractor import YoutubeIE, get_info_extractor
-from yt_dlp.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
 import threading
+from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
 
+from yt_dlp.compat import compat_etree_fromstring, compat_http_server
+from yt_dlp.extractor import YoutubeIE, get_info_extractor
+from yt_dlp.extractor.common import InfoExtractor
+from yt_dlp.utils import (
+    ExtractorError,
+    RegexNotFoundError,
+    encode_data_uri,
+    strip_jsonp,
+)
 
 TEAPOT_RESPONSE_STATUS = 418
 TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
@@ -1011,8 +1013,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for m3u8_file, m3u8_url, expected_formats, expected_subs in _TEST_CASES:
-            with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/m3u8/%s.m3u8' % m3u8_file, encoding='utf-8') as f:
                 formats, subs = self.ie._parse_m3u8_formats_and_subtitles(
                     f.read(), m3u8_url, ext='mp4')
                 self.ie._sort_formats(formats)
@@ -1357,10 +1358,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for mpd_file, mpd_url, mpd_base_url, expected_formats, expected_subtitles in _TEST_CASES:
-            with io.open('./test/testdata/mpd/%s.mpd' % mpd_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/mpd/%s.mpd' % mpd_file, encoding='utf-8') as f:
                 formats, subtitles = self.ie._parse_mpd_formats_and_subtitles(
-                    compat_etree_fromstring(f.read().encode('utf-8')),
+                    compat_etree_fromstring(f.read().encode()),
                     mpd_base_url=mpd_base_url, mpd_url=mpd_url)
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
@@ -1549,10 +1549,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for ism_file, ism_url, expected_formats, expected_subtitles in _TEST_CASES:
-            with io.open('./test/testdata/ism/%s.Manifest' % ism_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/ism/%s.Manifest' % ism_file, encoding='utf-8') as f:
                 formats, subtitles = self.ie._parse_ism_formats_and_subtitles(
-                    compat_etree_fromstring(f.read().encode('utf-8')), ism_url=ism_url)
+                    compat_etree_fromstring(f.read().encode()), ism_url=ism_url)
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
                 expect_value(self, subtitles, expected_subtitles, None)
@@ -1576,10 +1575,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for f4m_file, f4m_url, expected_formats in _TEST_CASES:
-            with io.open('./test/testdata/f4m/%s.f4m' % f4m_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/f4m/%s.f4m' % f4m_file, encoding='utf-8') as f:
                 formats = self.ie._parse_f4m_formats(
-                    compat_etree_fromstring(f.read().encode('utf-8')),
+                    compat_etree_fromstring(f.read().encode()),
                     f4m_url, None)
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
@@ -1624,10 +1622,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for xspf_file, xspf_url, expected_entries in _TEST_CASES:
-            with io.open('./test/testdata/xspf/%s.xspf' % xspf_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/xspf/%s.xspf' % xspf_file, encoding='utf-8') as f:
                 entries = self.ie._parse_xspf(
-                    compat_etree_fromstring(f.read().encode('utf-8')),
+                    compat_etree_fromstring(f.read().encode()),
                     xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url)
                 expect_value(self, entries, expected_entries, None)
                 for i in range(len(entries)):

test/test_YoutubeDL.py

@@ -1,31 +1,38 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import copy
 import json
-
 from test.helper import FakeYDL, assertRegexpMatches
+
 from yt_dlp import YoutubeDL
-from yt_dlp.compat import compat_os_name, compat_setenv, compat_str, compat_urllib_error
+from yt_dlp.compat import (
+    compat_os_name,
+    compat_setenv,
+    compat_str,
+    compat_urllib_error,
+)
 from yt_dlp.extractor import YoutubeIE
 from yt_dlp.extractor.common import InfoExtractor
 from yt_dlp.postprocessor.common import PostProcessor
-from yt_dlp.utils import ExtractorError, int_or_none, match_filter_func, LazyList
+from yt_dlp.utils import (
+    ExtractorError,
+    LazyList,
+    int_or_none,
+    match_filter_func,
+)
 
 TEST_URL = 'http://localhost/sample.mp4'
 
 
 class YDL(FakeYDL):
     def __init__(self, *args, **kwargs):
-        super(YDL, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.downloaded_info_dicts = []
         self.msgs = []
 
@@ -551,11 +558,11 @@ class TestYoutubeDL(unittest.TestCase):
         def s_formats(lang, autocaption=False):
             return [{
                 'ext': ext,
-                'url': 'http://localhost/video.%s.%s' % (lang, ext),
+                'url': f'http://localhost/video.{lang}.{ext}',
                 '_auto': autocaption,
             } for ext in ['vtt', 'srt', 'ass']]
-        subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
-        auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
+        subtitles = {l: s_formats(l) for l in ['en', 'fr', 'es']}
+        auto_captions = {l: s_formats(l, True) for l in ['it', 'pt', 'es']}
         info_dict = {
             'id': 'test',
             'title': 'Test',
@@ -580,7 +587,7 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_info({'writesubtitles': True})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['en']))
+        self.assertEqual(set(subs.keys()), {'en'})
         self.assertTrue(subs['en'].get('data') is None)
         self.assertEqual(subs['en']['ext'], 'ass')
 
@@ -591,39 +598,39 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['es', 'fr']))
+        self.assertEqual(set(subs.keys()), {'es', 'fr'})
 
         result = get_info({'writesubtitles': True, 'subtitleslangs': ['all', '-en']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['es', 'fr']))
+        self.assertEqual(set(subs.keys()), {'es', 'fr'})
 
         result = get_info({'writesubtitles': True, 'subtitleslangs': ['en', 'fr', '-en']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['fr']))
+        self.assertEqual(set(subs.keys()), {'fr'})
 
         result = get_info({'writesubtitles': True, 'subtitleslangs': ['-en', 'en']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['en']))
+        self.assertEqual(set(subs.keys()), {'en'})
 
         result = get_info({'writesubtitles': True, 'subtitleslangs': ['e.+']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['es', 'en']))
+        self.assertEqual(set(subs.keys()), {'es', 'en'})
 
         result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
+        self.assertEqual(set(subs.keys()), {'es', 'pt'})
         self.assertFalse(subs['es']['_auto'])
         self.assertTrue(subs['pt']['_auto'])
 
         result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
         subs = result['requested_subtitles']
         self.assertTrue(subs)
-        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
+        self.assertEqual(set(subs.keys()), {'es', 'pt'})
         self.assertTrue(subs['es']['_auto'])
         self.assertTrue(subs['pt']['_auto'])
 
@@ -654,7 +661,7 @@ class TestYoutubeDL(unittest.TestCase):
         'duration': 100000,
         'playlist_index': 1,
         'playlist_autonumber': 2,
-        '_last_playlist_index': 100,
+        '__last_playlist_index': 100,
         'n_entries': 10,
         'formats': [{'id': 'id 1'}, {'id': 'id 2'}, {'id': 'id 3'}]
     }
@@ -1082,7 +1089,7 @@ class TestYoutubeDL(unittest.TestCase):
 
         class _YDL(YDL):
             def __init__(self, *args, **kwargs):
-                super(_YDL, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
 
             def trouble(self, s, tb=None):
                 pass

test/test_YoutubeDLCookieJar.py

@@ -1,13 +1,10 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
 import os
 import re
 import sys
 import tempfile
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from yt_dlp.utils import YoutubeDLCookieJar
@@ -20,7 +17,7 @@ class TestYoutubeDLCookieJar(unittest.TestCase):
         tf = tempfile.NamedTemporaryFile(delete=False)
         try:
             cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True)
-            temp = tf.read().decode('utf-8')
+            temp = tf.read().decode()
             self.assertTrue(re.search(
                 r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp))
             self.assertTrue(re.search(

test/test_aes.py

@@ -1,30 +1,30 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import base64
+
 from yt_dlp.aes import (
-    aes_decrypt,
-    aes_encrypt,
-    aes_ecb_encrypt,
-    aes_ecb_decrypt,
+    BLOCK_SIZE_BYTES,
     aes_cbc_decrypt,
     aes_cbc_decrypt_bytes,
     aes_cbc_encrypt,
     aes_ctr_decrypt,
     aes_ctr_encrypt,
+    aes_decrypt,
+    aes_decrypt_text,
+    aes_ecb_decrypt,
+    aes_ecb_encrypt,
+    aes_encrypt,
     aes_gcm_decrypt_and_verify,
     aes_gcm_decrypt_and_verify_bytes,
-    aes_decrypt_text,
-    BLOCK_SIZE_BYTES,
 )
-from yt_dlp.compat import compat_pycrypto_AES
+from yt_dlp.dependencies import Cryptodome_AES
 from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes
-import base64
 
 # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'
 
@@ -45,7 +45,7 @@ class TestAES(unittest.TestCase):
         data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd'
         decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
-        if compat_pycrypto_AES:
+        if Cryptodome_AES:
             decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
@@ -75,25 +75,25 @@ class TestAES(unittest.TestCase):
         decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
             bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12]))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
-        if compat_pycrypto_AES:
+        if Cryptodome_AES:
             decrypted = aes_gcm_decrypt_and_verify_bytes(
                 data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12]))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_decrypt_text(self):
-        password = intlist_to_bytes(self.key).decode('utf-8')
+        password = intlist_to_bytes(self.key).decode()
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8])
             + b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
-        ).decode('utf-8')
+        ).decode()
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
-        password = intlist_to_bytes(self.key).decode('utf-8')
+        password = intlist_to_bytes(self.key).decode()
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8])
             + b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
-        ).decode('utf-8')
+        ).decode()
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 

test/test_age_restriction.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import try_rm, is_download_test
+from test.helper import is_download_test, try_rm
 
 from yt_dlp import YoutubeDL
 

test/test_all_urls.py

@@ -1,22 +1,16 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
+import collections
 import os
 import sys
 import unittest
-import collections
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 from test.helper import gettestcases
 
-from yt_dlp.extractor import (
-    FacebookIE,
-    gen_extractors,
-    YoutubeIE,
-)
+from yt_dlp.extractor import FacebookIE, YoutubeIE, gen_extractors
 
 
 class TestAllURLsMatching(unittest.TestCase):
@@ -81,11 +75,11 @@ class TestAllURLsMatching(unittest.TestCase):
             url = tc['url']
             for ie in ies:
                 if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'):
-                    self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url))
+                    self.assertTrue(ie.suitable(url), f'{type(ie).__name__} should match URL {url!r}')
                 else:
                     self.assertFalse(
                         ie.suitable(url),
-                        '%s should not match URL %r . That URL belongs to %s.' % (type(ie).__name__, url, tc['name']))
+                        f'{type(ie).__name__} should not match URL {url!r} . That URL belongs to {tc["name"]}.')
 
     def test_keywords(self):
         self.assertMatch(':ytsubs', ['youtube:subscriptions'])
@@ -120,7 +114,7 @@ class TestAllURLsMatching(unittest.TestCase):
         for (ie_name, ie_list) in name_accu.items():
             self.assertEqual(
                 len(ie_list), 1,
-                'Multiple extractors with the same IE_NAME "%s" (%s)' % (ie_name, ', '.join(ie_list)))
+                f'Multiple extractors with the same IE_NAME "{ie_name}" ({", ".join(ie_list)})')
 
 
 if __name__ == '__main__':

test/test_cache.py

@@ -1,18 +1,15 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
-import shutil
-
 # Allow direct execution
 import os
+import shutil
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 from test.helper import FakeYDL
+
 from yt_dlp.cache import Cache
 
 

test/test_compat.py

@@ -1,26 +1,20 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+from yt_dlp import compat
 from yt_dlp.compat import (
-    compat_getenv,
-    compat_setenv,
-    compat_etree_Element,
     compat_etree_fromstring,
     compat_expanduser,
-    compat_shlex_split,
+    compat_getenv,
+    compat_setenv,
     compat_str,
     compat_struct_unpack,
-    compat_urllib_parse_quote,
-    compat_urllib_parse_quote_plus,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
@@ -28,6 +22,12 @@ from yt_dlp.compat import (
 
 
 class TestCompat(unittest.TestCase):
+    def test_compat_passthrough(self):
+        with self.assertWarns(DeprecationWarning):
+            compat.compat_basestring
+
+        compat.asyncio.events  # Must not raise error
+
     def test_compat_getenv(self):
         test_str = 'тест'
         compat_setenv('yt_dlp_COMPAT_GETENV', test_str)
@@ -42,39 +42,12 @@ class TestCompat(unittest.TestCase):
 
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')
-        test_str = r'C:\Documents and Settings\тест\Application Data'
-        compat_setenv('HOME', test_str)
-        self.assertEqual(compat_expanduser('~'), test_str)
-        compat_setenv('HOME', old_home or '')
-
-    def test_all_present(self):
-        import yt_dlp.compat
-        all_names = yt_dlp.compat.__all__
-        present_names = set(filter(
-            lambda c: '_' in c and not c.startswith('_'),
-            dir(yt_dlp.compat))) - set(['unicode_literals'])
-        self.assertEqual(all_names, sorted(present_names))
-
-    def test_compat_urllib_parse_quote(self):
-        self.assertEqual(compat_urllib_parse_quote('abc def'), 'abc%20def')
-        self.assertEqual(compat_urllib_parse_quote('/user/abc+def'), '/user/abc%2Bdef')
-        self.assertEqual(compat_urllib_parse_quote('/user/abc+def', safe='+'), '%2Fuser%2Fabc+def')
-        self.assertEqual(compat_urllib_parse_quote(''), '')
-        self.assertEqual(compat_urllib_parse_quote('%'), '%25')
-        self.assertEqual(compat_urllib_parse_quote('%', safe='%'), '%')
-        self.assertEqual(compat_urllib_parse_quote('津波'), '%E6%B4%A5%E6%B3%A2')
-        self.assertEqual(
-            compat_urllib_parse_quote('''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
-%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''', safe='<>=":%/ \r\n'),
-            '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
-%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a''')
-        self.assertEqual(
-            compat_urllib_parse_quote('''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%25Things%''', safe='% '),
-            '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''')
-
-    def test_compat_urllib_parse_quote_plus(self):
-        self.assertEqual(compat_urllib_parse_quote_plus('abc def'), 'abc+def')
-        self.assertEqual(compat_urllib_parse_quote_plus('/abc def'), '%2Fabc+def')
+        test_str = R'C:\Documents and Settings\тест\Application Data'
+        try:
+            compat_setenv('HOME', test_str)
+            self.assertEqual(compat_expanduser('~'), test_str)
+        finally:
+            compat_setenv('HOME', old_home or '')
 
     def test_compat_urllib_parse_unquote(self):
         self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
@@ -109,17 +82,6 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_urlencode([(b'abc', 'def')]), 'abc=def')
         self.assertEqual(compat_urllib_parse_urlencode([(b'abc', b'def')]), 'abc=def')
 
-    def test_compat_shlex_split(self):
-        self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
-        self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag'])
-        self.assertEqual(compat_shlex_split('-val 中文'), ['-val', '中文'])
-
-    def test_compat_etree_Element(self):
-        try:
-            compat_etree_Element.items
-        except AttributeError:
-            self.fail('compat_etree_Element is not a type')
-
     def test_compat_etree_fromstring(self):
         xml = '''
             <root foo="bar" spam="中文">
@@ -128,7 +90,7 @@ class TestCompat(unittest.TestCase):
                 <foo><bar>spam</bar></foo>
             </root>
         '''
-        doc = compat_etree_fromstring(xml.encode('utf-8'))
+        doc = compat_etree_fromstring(xml.encode())
         self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
         self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
         self.assertTrue(isinstance(doc.find('normal').text, compat_str))

test/test_cookies.py

@@ -6,10 +6,10 @@ from yt_dlp.cookies import (
     LinuxChromeCookieDecryptor,
     MacChromeCookieDecryptor,
     WindowsChromeCookieDecryptor,
-    parse_safari_cookies,
-    pbkdf2_sha1,
     _get_linux_desktop_environment,
     _LinuxDesktopEnvironment,
+    parse_safari_cookies,
+    pbkdf2_sha1,
 )
 
 

test/test_download.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
+import hashlib
+import json
 import os
+import socket
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
@@ -19,25 +20,19 @@ from test.helper import (
     try_rm,
 )
 
-
-import hashlib
-import io
-import json
-import socket
-
 import yt_dlp.YoutubeDL
 from yt_dlp.compat import (
     compat_http_client,
-    compat_urllib_error,
     compat_HTTPError,
+    compat_urllib_error,
 )
+from yt_dlp.extractor import get_info_extractor
 from yt_dlp.utils import (
     DownloadError,
     ExtractorError,
-    format_bytes,
     UnavailableVideoError,
+    format_bytes,
 )
-from yt_dlp.extractor import get_info_extractor
 
 RETRIES = 3
 
@@ -46,7 +41,7 @@ class YoutubeDL(yt_dlp.YoutubeDL):
     def __init__(self, *args, **kwargs):
         self.to_stderr = self.to_screen
         self.processed_info_dicts = []
-        super(YoutubeDL, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def report_warning(self, message):
         # Don't accept warnings during tests
@@ -54,7 +49,7 @@ class YoutubeDL(yt_dlp.YoutubeDL):
 
     def process_info(self, info_dict):
         self.processed_info_dicts.append(info_dict.copy())
-        return super(YoutubeDL, self).process_info(info_dict)
+        return super().process_info(info_dict)
 
 
 def _file_md5(fn):
@@ -80,7 +75,7 @@ class TestDownload(unittest.TestCase):
 
         def strclass(cls):
             """From 2.7's unittest; 2.6 had _strclass so we can't import it."""
-            return '%s.%s' % (cls.__module__, cls.__name__)
+            return f'{cls.__module__}.{cls.__name__}'
 
         add_ie = getattr(self, self._testMethodName).add_ie
         return '%s (%s)%s:' % (self._testMethodName,
@@ -179,7 +174,7 @@ def generator(test_case, tname):
                         report_warning('%s failed due to network errors, skipping...' % tname)
                         return
 
-                    print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
+                    print(f'Retrying: {try_num} failed tries\n\n##########\n\n')
 
                     try_num += 1
                 else:
@@ -245,7 +240,7 @@ def generator(test_case, tname):
                 self.assertTrue(
                     os.path.exists(info_json_fn),
                     'Missing info file %s' % info_json_fn)
-                with io.open(info_json_fn, encoding='utf-8') as infof:
+                with open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
                 expect_info_dict(self, info_dict, tc.get('info_dict', {}))
         finally:

test/test_downloader_http.py

@@ -1,20 +1,19 @@
 #!/usr/bin/env python3
-# coding: utf-8
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import re
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import threading
 from test.helper import http_server_port, try_rm
+
 from yt_dlp import YoutubeDL
 from yt_dlp.compat import compat_http_server
 from yt_dlp.downloader.http import HttpFD
 from yt_dlp.utils import encodeFilename
-import threading
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -66,7 +65,7 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             assert False
 
 
-class FakeLogger(object):
+class FakeLogger:
     def debug(self, msg):
         pass
 

test/test_execution.py

@@ -1,13 +1,10 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
-import unittest
-
-import sys
+import contextlib
 import os
 import subprocess
+import sys
+import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from yt_dlp.utils import encodeArgument
@@ -26,15 +23,14 @@ class TestExecution(unittest.TestCase):
         subprocess.check_call([sys.executable, '-c', 'import yt_dlp'], cwd=rootDir)
 
     def test_module_exec(self):
-        if sys.version_info >= (2, 7):  # Python 2.6 doesn't support package execution
-            subprocess.check_call([sys.executable, '-m', 'yt_dlp', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+        subprocess.check_call([sys.executable, '-m', 'yt_dlp', '--ignore-config', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
-        subprocess.check_call([sys.executable, 'yt_dlp/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+        subprocess.check_call([sys.executable, 'yt_dlp/__main__.py', '--ignore-config', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_cmdline_umlauts(self):
         p = subprocess.Popen(
-            [sys.executable, 'yt_dlp/__main__.py', encodeArgument('ä'), '--version'],
+            [sys.executable, 'yt_dlp/__main__.py', '--ignore-config', encodeArgument('ä'), '--version'],
             cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
         _, stderr = p.communicate()
         self.assertFalse(stderr)
@@ -44,10 +40,8 @@ class TestExecution(unittest.TestCase):
             subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL)
             subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL)
         finally:
-            try:
+            with contextlib.suppress(OSError):
                 os.remove('yt_dlp/extractor/lazy_extractors.py')
-            except (IOError, OSError):
-                pass
 
 
 if __name__ == '__main__':

test/test_http.py

@@ -1,18 +1,17 @@
 #!/usr/bin/env python3
-# coding: utf-8
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import http_server_port
-from yt_dlp import YoutubeDL
-from yt_dlp.compat import compat_http_server, compat_urllib_request
 import ssl
 import threading
+from test.helper import http_server_port
+
+from yt_dlp import YoutubeDL
+from yt_dlp.compat import compat_http_server, compat_urllib_request
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -32,17 +31,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             self.send_header('Content-Type', 'video/mp4')
             self.end_headers()
             self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
-        elif self.path == '/302':
-            if sys.version_info[0] == 3:
-                # XXX: Python 3 http server does not allow non-ASCII header values
-                self.send_response(404)
-                self.end_headers()
-                return
-
-            new_url = 'http://127.0.0.1:%d/中文.html' % http_server_port(self.server)
-            self.send_response(302)
-            self.send_header(b'Location', new_url.encode('utf-8'))
-            self.end_headers()
         elif self.path == '/%E4%B8%AD%E6%96%87.html':
             self.send_response(200)
             self.send_header('Content-Type', 'text/html; charset=utf-8')
@@ -52,7 +40,7 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             assert False
 
 
-class FakeLogger(object):
+class FakeLogger:
     def debug(self, msg):
         pass
 
@@ -72,40 +60,75 @@ class TestHTTP(unittest.TestCase):
         self.server_thread.daemon = True
         self.server_thread.start()
 
-    def test_unicode_path_redirection(self):
-        # XXX: Python 3 http server does not allow non-ASCII header values
-        if sys.version_info[0] == 3:
-            return
-
-        ydl = YoutubeDL({'logger': FakeLogger()})
-        r = ydl.extract_info('http://127.0.0.1:%d/302' % self.port)
-        self.assertEqual(r['entries'][0]['url'], 'http://127.0.0.1:%d/vid.mp4' % self.port)
-
 
 class TestHTTPS(unittest.TestCase):
     def setUp(self):
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
         self.httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
-        self.httpd.socket = ssl.wrap_socket(
-            self.httpd.socket, certfile=certfn, server_side=True)
+        sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        sslctx.load_cert_chain(certfn, None)
+        self.httpd.socket = sslctx.wrap_socket(self.httpd.socket, server_side=True)
         self.port = http_server_port(self.httpd)
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)
         self.server_thread.daemon = True
         self.server_thread.start()
 
     def test_nocheckcertificate(self):
-        if sys.version_info >= (2, 7, 9):  # No certificate checking anyways
-            ydl = YoutubeDL({'logger': FakeLogger()})
-            self.assertRaises(
-                Exception,
-                ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port)
+        ydl = YoutubeDL({'logger': FakeLogger()})
+        self.assertRaises(
+            Exception,
+            ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port)
 
         ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
         r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
         self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
 
 
+class TestClientCert(unittest.TestCase):
+    def setUp(self):
+        certfn = os.path.join(TEST_DIR, 'testcert.pem')
+        self.certdir = os.path.join(TEST_DIR, 'testdata', 'certificate')
+        cacertfn = os.path.join(self.certdir, 'ca.crt')
+        self.httpd = compat_http_server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler)
+        sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        sslctx.verify_mode = ssl.CERT_REQUIRED
+        sslctx.load_verify_locations(cafile=cacertfn)
+        sslctx.load_cert_chain(certfn, None)
+        self.httpd.socket = sslctx.wrap_socket(self.httpd.socket, server_side=True)
+        self.port = http_server_port(self.httpd)
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def _run_test(self, **params):
+        ydl = YoutubeDL({
+            'logger': FakeLogger(),
+            # Disable client-side validation of unacceptable self-signed testcert.pem
+            # The test is of a check on the server side, so unaffected
+            'nocheckcertificate': True,
+            **params,
+        })
+        r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
+        self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+
+    def test_certificate_combined_nopass(self):
+        self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithkey.crt'))
+
+    def test_certificate_nocombined_nopass(self):
+        self._run_test(client_certificate=os.path.join(self.certdir, 'client.crt'),
+                       client_certificate_key=os.path.join(self.certdir, 'client.key'))
+
+    def test_certificate_combined_pass(self):
+        self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithencryptedkey.crt'),
+                       client_certificate_password='foobar')
+
+    def test_certificate_nocombined_pass(self):
+        self._run_test(client_certificate=os.path.join(self.certdir, 'client.crt'),
+                       client_certificate_key=os.path.join(self.certdir, 'clientencrypted.key'),
+                       client_certificate_password='foobar')
+
+
 def _build_proxy_handler(name):
     class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
         proxy_name = name
@@ -117,7 +140,7 @@ def _build_proxy_handler(name):
             self.send_response(200)
             self.send_header('Content-Type', 'text/plain; charset=utf-8')
             self.end_headers()
-            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
+            self.wfile.write(f'{self.proxy_name}: {self.path}'.encode())
     return HTTPTestRequestHandler
 
 
@@ -138,26 +161,26 @@ class TestProxy(unittest.TestCase):
         self.geo_proxy_thread.start()
 
     def test_proxy(self):
-        geo_proxy = '127.0.0.1:{0}'.format(self.geo_port)
+        geo_proxy = f'127.0.0.1:{self.geo_port}'
         ydl = YoutubeDL({
-            'proxy': '127.0.0.1:{0}'.format(self.port),
+            'proxy': f'127.0.0.1:{self.port}',
             'geo_verification_proxy': geo_proxy,
         })
         url = 'http://foo.com/bar'
-        response = ydl.urlopen(url).read().decode('utf-8')
-        self.assertEqual(response, 'normal: {0}'.format(url))
+        response = ydl.urlopen(url).read().decode()
+        self.assertEqual(response, f'normal: {url}')
 
         req = compat_urllib_request.Request(url)
         req.add_header('Ytdl-request-proxy', geo_proxy)
-        response = ydl.urlopen(req).read().decode('utf-8')
-        self.assertEqual(response, 'geo: {0}'.format(url))
+        response = ydl.urlopen(req).read().decode()
+        self.assertEqual(response, f'geo: {url}')
 
     def test_proxy_with_idn(self):
         ydl = YoutubeDL({
-            'proxy': '127.0.0.1:{0}'.format(self.port),
+            'proxy': f'127.0.0.1:{self.port}',
         })
         url = 'http://中文.tw/'
-        response = ydl.urlopen(url).read().decode('utf-8')
+        response = ydl.urlopen(url).read().decode()
         # b'xn--fiq228c' is '中文'.encode('idna')
         self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
 

test/test_iqiyi_sdk_interpreter.py

@@ -1,18 +1,17 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import FakeYDL, is_download_test
+
 from yt_dlp.extractor import IqiyiIE
 
 
-class WarningLogger(object):
+class WarningLogger:
     def __init__(self):
         self.messages = []
 

test/test_jsinterp.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from yt_dlp.jsinterp import JSInterpreter

test/test_netrc.py

@@ -1,9 +1,7 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 

test/test_overwrites.py

@@ -1,18 +1,15 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 import os
-from os.path import join
 import subprocess
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import is_download_test, try_rm
 
-
 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-download_file = join(root_dir, 'test.webm')
+download_file = os.path.join(root_dir, 'test.webm')
 
 
 @is_download_test
@@ -46,7 +43,7 @@ class TestOverwrites(unittest.TestCase):
         self.assertTrue(os.path.getsize(download_file) > 1)
 
     def tearDown(self):
-        try_rm(join(root_dir, 'test.webm'))
+        try_rm(os.path.join(root_dir, 'test.webm'))
 
 
 if __name__ == '__main__':

test/test_post_hooks.py

@@ -1,20 +1,19 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import get_params, try_rm, is_download_test
+from test.helper import get_params, is_download_test, try_rm
+
 import yt_dlp.YoutubeDL
 from yt_dlp.utils import DownloadError
 
 
 class YoutubeDL(yt_dlp.YoutubeDL):
     def __init__(self, *args, **kwargs):
-        super(YoutubeDL, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.to_stderr = self.to_screen
 
 

test/test_postprocessors.py

@@ -1,7 +1,4 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
@@ -16,7 +13,7 @@ from yt_dlp.postprocessor import (
     FFmpegThumbnailsConvertorPP,
     MetadataFromFieldPP,
     MetadataParserPP,
-    ModifyChaptersPP
+    ModifyChaptersPP,
 )
 
 

test/test_socks.py

@@ -1,25 +1,16 @@
 #!/usr/bin/env python3
-# coding: utf-8
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import random
 import subprocess
+from test.helper import FakeYDL, get_params, is_download_test
 
-from test.helper import (
-    FakeYDL,
-    get_params,
-    is_download_test,
-)
-from yt_dlp.compat import (
-    compat_str,
-    compat_urllib_request,
-)
+from yt_dlp.compat import compat_str, compat_urllib_request
 
 
 @is_download_test
@@ -41,7 +32,7 @@ class TestMultipleSocks(unittest.TestCase):
             'proxy': params['primary_proxy']
         })
         self.assertEqual(
-            ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8'),
+            ydl.urlopen('http://yt-dl.org/ip').read().decode(),
             params['primary_server_ip'])
 
     def test_proxy_https(self):
@@ -52,7 +43,7 @@ class TestMultipleSocks(unittest.TestCase):
             'proxy': params['primary_proxy']
         })
         self.assertEqual(
-            ydl.urlopen('https://yt-dl.org/ip').read().decode('utf-8'),
+            ydl.urlopen('https://yt-dl.org/ip').read().decode(),
             params['primary_server_ip'])
 
     def test_secondary_proxy_http(self):
@@ -63,7 +54,7 @@ class TestMultipleSocks(unittest.TestCase):
         req = compat_urllib_request.Request('http://yt-dl.org/ip')
         req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
         self.assertEqual(
-            ydl.urlopen(req).read().decode('utf-8'),
+            ydl.urlopen(req).read().decode(),
             params['secondary_server_ip'])
 
     def test_secondary_proxy_https(self):
@@ -74,7 +65,7 @@ class TestMultipleSocks(unittest.TestCase):
         req = compat_urllib_request.Request('https://yt-dl.org/ip')
         req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
         self.assertEqual(
-            ydl.urlopen(req).read().decode('utf-8'),
+            ydl.urlopen(req).read().decode(),
             params['secondary_server_ip'])
 
 
@@ -105,7 +96,7 @@ class TestSocks(unittest.TestCase):
         ydl = FakeYDL({
             'proxy': '%s://127.0.0.1:%d' % (protocol, self.port),
         })
-        return ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8')
+        return ydl.urlopen('http://yt-dl.org/ip').read().decode()
 
     def test_socks4(self):
         self.assertTrue(isinstance(self._get_ip('socks4'), compat_str))

test/test_subtitles.py

@@ -1,33 +1,31 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL, md5, is_download_test
-
+from test.helper import FakeYDL, is_download_test, md5
 
 from yt_dlp.extractor import (
-    YoutubeIE,
+    NPOIE,
+    NRKTVIE,
+    PBSIE,
+    CeskaTelevizeIE,
+    ComedyCentralIE,
     DailymotionIE,
+    DemocracynowIE,
+    LyndaIE,
+    RaiPlayIE,
+    RTVEALaCartaIE,
     TedTalkIE,
+    ThePlatformFeedIE,
+    ThePlatformIE,
+    VikiIE,
     VimeoIE,
     WallaIE,
-    CeskaTelevizeIE,
-    LyndaIE,
-    NPOIE,
-    PBSIE,
-    ComedyCentralIE,
-    NRKTVIE,
-    RaiPlayIE,
-    VikiIE,
-    ThePlatformIE,
-    ThePlatformFeedIE,
-    RTVEALaCartaIE,
-    DemocracynowIE,
+    YoutubeIE,
 )
 
 
@@ -53,8 +51,8 @@ class BaseTestSubtitles(unittest.TestCase):
         for sub_info in subtitles.values():
             if sub_info.get('data') is None:
                 uf = self.DL.urlopen(sub_info['url'])
-                sub_info['data'] = uf.read().decode('utf-8')
-        return dict((l, sub_info['data']) for l, sub_info in subtitles.items())
+                sub_info['data'] = uf.read().decode()
+        return {l: sub_info['data'] for l, sub_info in subtitles.items()}
 
 
 @is_download_test
@@ -163,7 +161,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
+        self.assertEqual(set(subtitles.keys()), {'de', 'en', 'es', 'fr'})
         self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
         self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
 
@@ -186,7 +184,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['heb']))
+        self.assertEqual(set(subtitles.keys()), {'heb'})
         self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
 
     def test_nosubtitles(self):
@@ -208,7 +206,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['cs']))
+        self.assertEqual(set(subtitles.keys()), {'cs'})
         self.assertTrue(len(subtitles['cs']) > 20000)
 
     def test_nosubtitles(self):
@@ -229,7 +227,7 @@ class TestLyndaSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
 
 
@@ -242,7 +240,7 @@ class TestNPOSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['nl']))
+        self.assertEqual(set(subtitles.keys()), {'nl'})
         self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
 
 
@@ -252,13 +250,13 @@ class TestMTVSubtitles(BaseTestSubtitles):
     IE = ComedyCentralIE
 
     def getInfoDict(self):
-        return super(TestMTVSubtitles, self).getInfoDict()['entries'][0]
+        return super().getInfoDict()['entries'][0]
 
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), '78206b8d8a0cfa9da64dc026eea48961')
 
 
@@ -271,7 +269,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['no']))
+        self.assertEqual(set(subtitles.keys()), {'no'})
         self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
 
 
@@ -284,7 +282,7 @@ class TestRaiPlaySubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['it']))
+        self.assertEqual(set(subtitles.keys()), {'it'})
         self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
 
     def test_subtitles_array_key(self):
@@ -292,7 +290,7 @@ class TestRaiPlaySubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['it']))
+        self.assertEqual(set(subtitles.keys()), {'it'})
         self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd')
 
 
@@ -305,7 +303,7 @@ class TestVikiSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a')
 
 
@@ -320,7 +318,7 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
 
 
@@ -333,7 +331,7 @@ class TestThePlatformFeedSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), '48649a22e82b2da21c9a67a395eedade')
 
 
@@ -348,7 +346,7 @@ class TestRtveSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['es']))
+        self.assertEqual(set(subtitles.keys()), {'es'})
         self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')
 
 
@@ -361,7 +359,7 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
 
     def test_subtitles_in_page(self):
@@ -369,7 +367,7 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
         self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
 
 
@@ -382,7 +380,7 @@ class TestPBSSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(set(subtitles.keys()), {'en'})
 
     def test_subtitles_dfxp_format(self):
         self.DL.params['writesubtitles'] = True

test/test_swfinterp.py.disabled

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-import errno
-import io
-import json
-import re
-import subprocess
-
-from yt_dlp.swfinterp import SWFInterpreter
-
-
-TEST_DIR = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)), 'swftests')
-
-
-class TestSWFInterpreter(unittest.TestCase):
-    pass
-
-
-def _make_testfunc(testfile):
-    m = re.match(r'^(.*)\.(as)$', testfile)
-    if not m:
-        return
-    test_id = m.group(1)
-
-    def test_func(self):
-        as_file = os.path.join(TEST_DIR, testfile)
-        swf_file = os.path.join(TEST_DIR, test_id + '.swf')
-        if ((not os.path.exists(swf_file))
-                or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
-            # Recompile
-            try:
-                subprocess.check_call([
-                    'mxmlc', '-output', swf_file,
-                    '-static-link-runtime-shared-libraries', as_file])
-            except OSError as ose:
-                if ose.errno == errno.ENOENT:
-                    print('mxmlc not found! Skipping test.')
-                    return
-                raise
-
-        with open(swf_file, 'rb') as swf_f:
-            swf_content = swf_f.read()
-        swfi = SWFInterpreter(swf_content)
-
-        with io.open(as_file, 'r', encoding='utf-8') as as_f:
-            as_content = as_f.read()
-
-        def _find_spec(key):
-            m = re.search(
-                r'(?m)^//\s*%s:\s*(.*?)\n' % re.escape(key), as_content)
-            if not m:
-                raise ValueError('Cannot find %s in %s' % (key, testfile))
-            return json.loads(m.group(1))
-
-        input_args = _find_spec('input')
-        output = _find_spec('output')
-
-        swf_class = swfi.extract_class(test_id)
-        func = swfi.extract_function(swf_class, 'main')
-        res = func(input_args)
-        self.assertEqual(res, output)
-
-    test_func.__name__ = str('test_swf_' + test_id)
-    setattr(TestSWFInterpreter, test_func.__name__, test_func)
-
-
-for testfile in os.listdir(TEST_DIR):
-    _make_testfunc(testfile)
-
-if __name__ == '__main__':
-    unittest.main()

test/test_unicode_literals.py.disabled

@@ -1,63 +0,0 @@
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import io
-import re
-
-rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-IGNORED_FILES = [
-    'setup.py',  # http://bugs.python.org/issue13943
-    'conf.py',
-    'buildserver.py',
-]
-
-IGNORED_DIRS = [
-    '.git',
-    '.tox',
-]
-
-from test.helper import assertRegexpMatches
-
-
-class TestUnicodeLiterals(unittest.TestCase):
-    def test_all_files(self):
-        for dirpath, dirnames, filenames in os.walk(rootDir):
-            for ignore_dir in IGNORED_DIRS:
-                if ignore_dir in dirnames:
-                    # If we remove the directory from dirnames os.walk won't
-                    # recurse into it
-                    dirnames.remove(ignore_dir)
-            for basename in filenames:
-                if not basename.endswith('.py'):
-                    continue
-                if basename in IGNORED_FILES:
-                    continue
-
-                fn = os.path.join(dirpath, basename)
-                with io.open(fn, encoding='utf-8') as inf:
-                    code = inf.read()
-
-                if "'" not in code and '"' not in code:
-                    continue
-                assertRegexpMatches(
-                    self,
-                    code,
-                    r'(?:(?:#.*?|\s*)\n)*from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
-                    'unicode_literals import  missing in %s' % fn)
-
-                m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code)
-                if m is not None:
-                    self.assertTrue(
-                        m is None,
-                        'u present in %s, around %s' % (
-                            fn, code[m.start() - 10:m.end() + 10]))
-
-
-if __name__ == '__main__':
-    unittest.main()

test/test_update.py.disabled

@@ -1,15 +1,14 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 import json
+
 from yt_dlp.update import rsa_verify
 
 
@@ -22,7 +21,7 @@ class TestUpdate(unittest.TestCase):
         signature = versions_info['signature']
         del versions_info['signature']
         self.assertTrue(rsa_verify(
-            json.dumps(versions_info, sort_keys=True).encode('utf-8'),
+            json.dumps(versions_info, sort_keys=True).encode(),
             signature, UPDATES_RSA_KEY))
 
 

test/test_utils.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
 # Allow direct execution
+import contextlib
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
@@ -16,75 +14,95 @@ import itertools
 import json
 import xml.etree.ElementTree
 
+from yt_dlp.compat import (
+    compat_chr,
+    compat_etree_fromstring,
+    compat_getenv,
+    compat_HTMLParseError,
+    compat_os_name,
+    compat_setenv,
+)
 from yt_dlp.utils import (
+    Config,
+    DateRange,
+    ExtractorError,
+    InAdvancePagedList,
+    LazyList,
+    OnDemandPagedList,
     age_restricted,
     args_to_str,
-    encode_base_n,
+    base_url,
     caesar,
     clean_html,
     clean_podcast_url,
-    Config,
+    cli_bool_option,
+    cli_option,
+    cli_valueless_option,
     date_from_str,
     datetime_from_str,
-    DateRange,
     detect_exe_version,
     determine_ext,
+    dfxp2srt,
     dict_get,
+    encode_base_n,
     encode_compat_str,
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    expand_path,
     extract_attributes,
-    ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
-    format_bytes,
     float_or_none,
-    get_element_by_class,
+    format_bytes,
     get_element_by_attribute,
-    get_elements_by_class,
-    get_elements_by_attribute,
-    get_element_html_by_class,
+    get_element_by_class,
     get_element_html_by_attribute,
-    get_elements_html_by_class,
-    get_elements_html_by_attribute,
-    get_elements_text_and_html_by_attribute,
+    get_element_html_by_class,
     get_element_text_and_html_by_tag,
-    InAdvancePagedList,
+    get_elements_by_attribute,
+    get_elements_by_class,
+    get_elements_html_by_attribute,
+    get_elements_html_by_class,
+    get_elements_text_and_html_by_attribute,
     int_or_none,
     intlist_to_bytes,
+    iri_to_uri,
     is_html,
     js_to_json,
     limit_length,
     locked_file,
+    lowercase_escape,
+    match_str,
     merge_dicts,
     mimetype2ext,
     month_by_name,
     multipart_encode,
     ohdave_rsa_encrypt,
-    OnDemandPagedList,
     orderedSet,
     parse_age_limit,
+    parse_bitrate,
+    parse_codecs,
+    parse_count,
+    parse_dfxp_time_expr,
     parse_duration,
     parse_filesize,
-    parse_count,
     parse_iso8601,
-    parse_resolution,
-    parse_bitrate,
     parse_qs,
+    parse_resolution,
     pkcs1pad,
+    prepend_extension,
     read_batch_urls,
+    remove_end,
+    remove_quotes,
+    remove_start,
+    render_table,
+    replace_extension,
+    rot47,
     sanitize_filename,
     sanitize_path,
     sanitize_url,
     sanitized_Request,
-    expand_path,
-    prepend_extension,
-    replace_extension,
-    remove_start,
-    remove_end,
-    remove_quotes,
-    rot47,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -96,38 +114,18 @@ from yt_dlp.utils import (
     unified_strdate,
     unified_timestamp,
     unsmuggle_url,
+    update_url_query,
     uppercase_escape,
-    lowercase_escape,
     url_basename,
     url_or_none,
-    base_url,
-    urljoin,
     urlencode_postdata,
+    urljoin,
     urshift,
-    update_url_query,
     version_tuple,
-    xpath_with_ns,
+    xpath_attr,
     xpath_element,
     xpath_text,
-    xpath_attr,
-    render_table,
-    match_str,
-    parse_dfxp_time_expr,
-    dfxp2srt,
-    cli_option,
-    cli_valueless_option,
-    cli_bool_option,
-    parse_codecs,
-    iri_to_uri,
-    LazyList,
-)
-from yt_dlp.compat import (
-    compat_chr,
-    compat_etree_fromstring,
-    compat_getenv,
-    compat_HTMLParseError,
-    compat_os_name,
-    compat_setenv,
+    xpath_with_ns,
 )
 
 
@@ -266,15 +264,22 @@ class TestUtil(unittest.TestCase):
 
     def test_expand_path(self):
         def env(var):
-            return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
+            return f'%{var}%' if sys.platform == 'win32' else f'${var}'
 
         compat_setenv('yt_dlp_EXPATH_PATH', 'expanded')
         self.assertEqual(expand_path(env('yt_dlp_EXPATH_PATH')), 'expanded')
-        self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME'))
-        self.assertEqual(expand_path('~'), compat_getenv('HOME'))
-        self.assertEqual(
-            expand_path('~/%s' % env('yt_dlp_EXPATH_PATH')),
-            '%s/expanded' % compat_getenv('HOME'))
+
+        old_home = os.environ.get('HOME')
+        test_str = R'C:\Documents and Settings\тест\Application Data'
+        try:
+            compat_setenv('HOME', test_str)
+            self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME'))
+            self.assertEqual(expand_path('~'), compat_getenv('HOME'))
+            self.assertEqual(
+                expand_path('~/%s' % env('yt_dlp_EXPATH_PATH')),
+                '%s/expanded' % compat_getenv('HOME'))
+        finally:
+            compat_setenv('HOME', old_home or '')
 
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
@@ -538,9 +543,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(str_to_int('123,456'), 123456)
         self.assertEqual(str_to_int('123.456'), 123456)
         self.assertEqual(str_to_int(523), 523)
-        # Python 3 has no long
-        if sys.version_info < (3, 0):
-            eval('self.assertEqual(str_to_int(123456L), 123456)')
         self.assertEqual(str_to_int('noninteger'), None)
         self.assertEqual(str_to_int([]), None)
 
@@ -669,8 +671,7 @@ class TestUtil(unittest.TestCase):
             def get_page(pagenum):
                 firstid = pagenum * pagesize
                 upto = min(size, pagenum * pagesize + pagesize)
-                for i in range(firstid, upto):
-                    yield i
+                yield from range(firstid, upto)
 
             pl = OnDemandPagedList(get_page, pagesize)
             got = pl.getslice(*sliceargs)
@@ -739,7 +740,7 @@ class TestUtil(unittest.TestCase):
             multipart_encode({b'field': b'value'}, boundary='AAAAAA')[0],
             b'--AAAAAA\r\nContent-Disposition: form-data; name="field"\r\n\r\nvalue\r\n--AAAAAA--\r\n')
         self.assertEqual(
-            multipart_encode({'欄位'.encode('utf-8'): '值'.encode('utf-8')}, boundary='AAAAAA')[0],
+            multipart_encode({'欄位'.encode(): '值'.encode()}, boundary='AAAAAA')[0],
             b'--AAAAAA\r\nContent-Disposition: form-data; name="\xe6\xac\x84\xe4\xbd\x8d"\r\n\r\n\xe5\x80\xbc\r\n--AAAAAA--\r\n')
         self.assertRaises(
             ValueError, multipart_encode, {b'field': b'value'}, boundary='value')
@@ -1400,7 +1401,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                     <p begin="3" dur="-1">Ignored, three</p>
                 </div>
             </body>
-            </tt>'''.encode('utf-8')
+            </tt>'''.encode()
         srt_data = '''1
 00:00:00,000 --> 00:00:01,000
 The following line contains Chinese characters and special symbols
@@ -1418,14 +1419,14 @@ Line
 '''
         self.assertEqual(dfxp2srt(dfxp_data), srt_data)
 
-        dfxp_data_no_default_namespace = '''<?xml version="1.0" encoding="UTF-8"?>
+        dfxp_data_no_default_namespace = b'''<?xml version="1.0" encoding="UTF-8"?>
             <tt xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
             <body>
                 <div xml:lang="en">
                     <p begin="0" end="1">The first line</p>
                 </div>
             </body>
-            </tt>'''.encode('utf-8')
+            </tt>'''
         srt_data = '''1
 00:00:00,000 --> 00:00:01,000
 The first line
@@ -1433,7 +1434,7 @@ The first line
 '''
         self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
 
-        dfxp_data_with_style = '''<?xml version="1.0" encoding="utf-8"?>
+        dfxp_data_with_style = b'''<?xml version="1.0" encoding="utf-8"?>
 <tt xmlns="http://www.w3.org/2006/10/ttaf1" xmlns:ttp="http://www.w3.org/2006/10/ttaf1#parameter" ttp:timeBase="media" xmlns:tts="http://www.w3.org/2006/10/ttaf1#style" xml:lang="en" xmlns:ttm="http://www.w3.org/2006/10/ttaf1#metadata">
   <head>
     <styling>
@@ -1451,7 +1452,7 @@ The first line
       <p style="s1" tts:textDecoration="underline" begin="00:00:09.56" id="p2" end="00:00:12.36"><span style="s2" tts:color="lime">inner<br /> </span>style</p>
     </div>
   </body>
-</tt>'''.encode('utf-8')
+</tt>'''
         srt_data = '''1
 00:00:02,080 --> 00:00:05,840
 <font color="white" face="sansSerif" size="16">default style<font color="red">custom style</font></font>
@@ -1758,7 +1759,7 @@ Line 1
 
         def test(ll, idx, val, cache):
             self.assertEqual(ll[idx], val)
-            self.assertEqual(getattr(ll, '_LazyList__cache'), list(cache))
+            self.assertEqual(ll._cache, list(cache))
 
         ll = LazyList(range(10))
         test(ll, 0, 0, range(1))
@@ -1821,10 +1822,8 @@ Line 1
                         else:
                             self.assertFalse(testing_write, f'{test_mode} is not blocked by {lock_mode}')
         finally:
-            try:
+            with contextlib.suppress(OSError):
                 os.remove(FILE)
-            except Exception:
-                pass
 
 
 if __name__ == '__main__':

test/test_verbose_output.py

@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
-# coding: utf-8
-
-from __future__ import unicode_literals
-
-import unittest
-
-import sys
 import os
 import subprocess
+import sys
+import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -17,7 +13,8 @@ class TestVerboseOutput(unittest.TestCase):
     def test_private_info_arg(self):
         outp = subprocess.Popen(
             [
-                sys.executable, 'yt_dlp/__main__.py', '-v',
+                sys.executable, 'yt_dlp/__main__.py',
+                '-v', '--ignore-config',
                 '--username', 'johnsmith@gmail.com',
                 '--password', 'my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -30,7 +27,8 @@ class TestVerboseOutput(unittest.TestCase):
     def test_private_info_shortarg(self):
         outp = subprocess.Popen(
             [
-                sys.executable, 'yt_dlp/__main__.py', '-v',
+                sys.executable, 'yt_dlp/__main__.py',
+                '-v', '--ignore-config',
                 '-u', 'johnsmith@gmail.com',
                 '-p', 'my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -43,7 +41,8 @@ class TestVerboseOutput(unittest.TestCase):
     def test_private_info_eq(self):
         outp = subprocess.Popen(
             [
-                sys.executable, 'yt_dlp/__main__.py', '-v',
+                sys.executable, 'yt_dlp/__main__.py',
+                '-v', '--ignore-config',
                 '--username=johnsmith@gmail.com',
                 '--password=my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -56,7 +55,8 @@ class TestVerboseOutput(unittest.TestCase):
     def test_private_info_shortarg_eq(self):
         outp = subprocess.Popen(
             [
-                sys.executable, 'yt_dlp/__main__.py', '-v',
+                sys.executable, 'yt_dlp/__main__.py',
+                '-v', '--ignore-config',
                 '-u=johnsmith@gmail.com',
                 '-p=my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

test/test_write_annotations.py.disabled

@@ -1,27 +1,21 @@
 #!/usr/bin/env python3
-# coding: utf-8
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import get_params, try_rm, is_download_test
-
-
-import io
-
 import xml.etree.ElementTree
+from test.helper import get_params, is_download_test, try_rm
 
-import yt_dlp.YoutubeDL
 import yt_dlp.extractor
+import yt_dlp.YoutubeDL
 
 
 class YoutubeDL(yt_dlp.YoutubeDL):
     def __init__(self, *args, **kwargs):
-        super(YoutubeDL, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.to_stderr = self.to_screen
 
 
@@ -52,7 +46,7 @@ class TestAnnotations(unittest.TestCase):
         ydl.download([TEST_ID])
         self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
         annoxml = None
-        with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
+        with open(ANNOTATIONS_FILE, encoding='utf-8') as annof:
             annoxml = xml.etree.ElementTree.parse(annof)
         self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
         root = annoxml.getroot()

test/test_youtube_lists.py

@@ -1,18 +1,14 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import FakeYDL, is_download_test
 
-from yt_dlp.extractor import (
-    YoutubeIE,
-    YoutubeTabIE,
-)
+from yt_dlp.extractor import YoutubeIE, YoutubeTabIE
 
 
 @is_download_test

test/test_youtube_misc.py

@@ -1,10 +1,9 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 # Allow direct execution
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 

test/test_youtube_signature.py

@@ -1,21 +1,20 @@
 #!/usr/bin/env python3
-
-from __future__ import unicode_literals
-
 # Allow direct execution
+import contextlib
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-import io
 import re
 import string
-
+import urllib.request
 from test.helper import FakeYDL, is_download_test
+
+from yt_dlp.compat import compat_str
 from yt_dlp.extractor import YoutubeIE
 from yt_dlp.jsinterp import JSInterpreter
-from yt_dlp.compat import compat_str, compat_urlretrieve
 
 _SIG_TESTS = [
     (
@@ -129,11 +128,9 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
     def tearDown(self):
-        try:
+        with contextlib.suppress(OSError):
             for f in os.listdir(self.TESTDATA_DIR):
                 os.remove(f)
-        except OSError:
-            pass
 
 
 def t_factory(name, sig_func, url_pattern):
@@ -147,8 +144,8 @@ def t_factory(name, sig_func, url_pattern):
             fn = os.path.join(self.TESTDATA_DIR, basename)
 
             if not os.path.exists(fn):
-                compat_urlretrieve(url, fn)
-            with io.open(fn, encoding='utf-8') as testf:
+                urllib.request.urlretrieve(url, fn)
+            with open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()
             self.assertEqual(sig_func(jscode, sig_input), expected_sig)
 

test/testdata/certificate/ca.crt

@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE-----
+MIIBfDCCASOgAwIBAgIUUgngoxFpuWft8gjj3uEFoqJyoJowCgYIKoZIzj0EAwIw
+FDESMBAGA1UEAwwJeXRkbHB0ZXN0MB4XDTIyMDQxNTAzMDEwMVoXDTM4MTAxNTAz
+MDEwMVowFDESMBAGA1UEAwwJeXRkbHB0ZXN0MFkwEwYHKoZIzj0CAQYIKoZIzj0D
+AQcDQgAEcTaKMtIn2/1kgid1zXFpLm87FMT5PP3/bltKVVH3DLO//0kUslCHYxFU
+KpcCfVt9aueRyUFi1TNkkkEZ9D6fbqNTMFEwHQYDVR0OBBYEFBdY2rVNLFGM6r1F
+iuamNDaiq0QoMB8GA1UdIwQYMBaAFBdY2rVNLFGM6r1FiuamNDaiq0QoMA8GA1Ud
+EwEB/wQFMAMBAf8wCgYIKoZIzj0EAwIDRwAwRAIgXJg2jio1kow2g/iP54Qq+iI2
+m4EAvZiY0Im/Ni3PHawCIC6KCl6QcHANbeq8ckOXNGusjl6OWhvEM3uPBPhqskq1
+-----END CERTIFICATE-----