[cleanup] Misc cleanup

Closes yt-dlp#1942 yt-dlp#1976 yt-dlp#2020 yt-dlp#2058 yt-dlp#1984

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -34,7 +34,7 @@ body:
       label: Example URLs
       description: |
         Provide all kinds of example URLs for which support should be added
-      value: |
+      placeholder: |
         - Single video: https://www.youtube.com/watch?v=BaW_jenozKc
         - Single video: https://youtu.be/BaW_jenozKc
         - Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc

.github/ISSUE_TEMPLATE/config.yml

@@ -2,4 +2,4 @@ blank_issues_enabled: false
 contact_links:
   - name: Get help from the community on Discord
     url: https://discord.gg/H5MNcFW63r
-    about: Join the yt-dlp Discord for community-powered support!
+    about: Join the yt-dlp Discord for community-powered support!

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -1,6 +1,6 @@
 name: Broken site support
 description: Report broken or misfunctioning site
-labels: [triage, extractor-bug]
+labels: [triage, site-bug]
 body:
   - type: checkboxes
     id: checklist

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -1,5 +1,5 @@
 name: Site feature request
-description: Request a new functionality for a site
+description: Request a new functionality for a supported site
 labels: [triage, site-enhancement]
 body:
   - type: checkboxes
@@ -47,3 +47,26 @@ body:
       placeholder: WRITE DESCRIPTION HERE
     validations:
       required: true
+  - type: textarea
+    id: log
+    attributes:
+      label: Verbose log
+      description: |
+        Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
+        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        It should look similar to this:
+      placeholder: |
+        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Portable config file: yt-dlp.conf
+        [debug] Portable config: ['-i']
+        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
+        [debug] yt-dlp version %(version)s (exe)
+        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
+        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
+        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Proxy map: {}
+        yt-dlp is up to date (%(version)s)
+        <more lines>
+      render: shell
+    validations:
+      required: true

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -1,6 +1,6 @@
 name: Bug report
 description: Report a bug unrelated to any particular site or extractor
-labels: [triage,bug]
+labels: [triage, bug]
 body:
   - type: checkboxes
     id: checklist

.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml

@@ -1,4 +1,4 @@
-name: Feature request request
+name: Feature request
 description: Request a new functionality unrelated to any particular site or extractor
 labels: [triage, enhancement]
 body:

.github/ISSUE_TEMPLATE_tmpl/6_question.yml

@@ -9,7 +9,7 @@ body:
       description: |
         Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
       options:
-        - label: I'm asking a question and not reporting a bug/feature request
+        - label: I'm asking a question and **not** reporting a bug/feature request
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
@@ -24,7 +24,8 @@ body:
       description: |
         Ask your question in an arbitrary form.
         Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information and as much context and examples as possible
+        Provide any additional information and as much context and examples as possible.
+        If your question contains "isn't working" or "can you add", this is most likely the wrong template
       placeholder: WRITE QUESTION HERE
     validations:
       required: true

.gitignore

@@ -1,27 +1,32 @@
 # Config
 *.conf
-*.spec
 cookies
 *cookies.txt
 .netrc
 
 # Downloaded
-*.3gp
 *.annotations.xml
-*.ape
 *.aria2
-*.avi
 *.description
-*.desktop
 *.dump
-*.flac
-*.flv
 *.frag
+*.frag.aria2
 *.frag.urls
 *.info.json
+*.live_chat.json
+*.part*
+*.unknown_video
+*.ytdl
+.cache/
+
+*.3gp
+*.ape
+*.avi
+*.desktop
+*.flac
+*.flv
 *.jpeg
 *.jpg
-*.live_chat.json
 *.m4a
 *.m4v
 *.mhtml
@@ -31,23 +36,18 @@ cookies
 *.mp4
 *.ogg
 *.opus
-*.part
-*.part-*
 *.png
 *.sbv
 *.srt
 *.swf
 *.swp
 *.ttml
-*.unknown_video
 *.url
 *.vtt
 *.wav
 *.webloc
 *.webm
 *.webp
-*.ytdl
-.cache/
 
 # Allow config/media files in testdata
 !test/**
@@ -86,7 +86,6 @@ README.txt
 *.1
 *.bash-completion
 *.fish
-*.exe
 *.tar.gz
 *.zsh
 *.spec

CONTRIBUTING.md

@@ -227,6 +227,13 @@ After you have ensured this site is distributing its content legally, you can fo
 
 In any case, thank you very much for your contributions!
 
+**Tip:** To test extractors that require login information, create a file `test/local_parameters.json` and add `"usenetrc": true` or your username and password in it:
+```json
+{
+    "username": "your user name",
+    "password": "your password"
+}
+```
 
 ## yt-dlp coding conventions
 

Collaborators.md

@@ -28,6 +28,7 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
 [![gh-sponsor](https://img.shields.io/badge/_-Sponsor-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
 
 * YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
+* Added support for downloading YoutubeWebArchive videos
 
 
 

Makefile

@@ -13,10 +13,10 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites com
 .PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites
 
 clean-test:
-	rm -rf *.3gp *.annotations.xml *.ape *.avi *.description *.dump *.flac *.flv *.frag *.frag.aria2 *.frag.urls \
-	*.info.json *.jpeg *.jpg *.live_chat.json *.m4a *.m4v *.mkv *.mp3 *.mp4 *.ogg *.opus *.part* *.png *.sbv *.srt \
-	*.swf *.swp *.ttml *.vtt *.wav *.webm *.webp *.mhtml *.mov *.unknown_video *.desktop *.url *.webloc *.ytdl \
-	test/testdata/player-*.js tmp/
+	rm -rf test/testdata/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
+	*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.part* *.unknown_video *.ytdl \
+	*.3gp *.ape *.avi *.desktop *.flac *.flv *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \
+	*.mp4 *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
 clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
 	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap

README.md

@@ -71,7 +71,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 # NEW FEATURES
 
-* Based on **youtube-dl 2021.06.06 [commit/379f52a](https://github.com/ytdl-org/youtube-dl/commit/379f52a4954013767219d25099cce9e0f9401961)** and **youtube-dlc 2020.11.11-3 [commit/98e248f](https://github.com/blackjack4494/yt-dlc/commit/98e248faa49e69d795abc60f7cdefcf91e2612aa)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Based on **youtube-dl 2021.12.17 [commit/5014bd6](https://github.com/ytdl-org/youtube-dl/commit/5014bd67c22b421207b2650d4dc874b95b36dda1)** and **youtube-dlc 2020.11.11-3 [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
@@ -125,7 +125,7 @@ See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/comm
 Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
 
 * The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
-* `avconv` is not supported as as an alternative to `ffmpeg`
+* `avconv` is not supported as an alternative to `ffmpeg`
 * The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
 * The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order
 * The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be preferred. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
@@ -172,7 +172,7 @@ sudo chmod a+rx /usr/local/bin/yt-dlp
 ```
 
 ```
-sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
+sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp --dir /usr/local/bin -o yt-dlp
 sudo chmod a+rx /usr/local/bin/yt-dlp
 ```
 
@@ -251,7 +251,7 @@ On windows, [Microsoft Visual C++ 2010 SP1 Redistributable Package (x86)](https:
 
 While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly recommended
 
-* [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging separate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. Licence [depends on the build](https://www.ffmpeg.org/legal.html)
+* [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging separate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. License [depends on the build](https://www.ffmpeg.org/legal.html)
 * [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
 * [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
 * [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licensed under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)

devscripts/update-version.py

@@ -27,13 +27,13 @@
 except Exception:
     GIT_HEAD = None
 
-VERSION_FILE = f'''
+VERSION_FILE = f'''\
 # Autogenerated by devscripts/update-version.py
 
 __version__ = {VERSION!r}
 
 RELEASE_GIT_HEAD = {GIT_HEAD!r}
-'''.lstrip()
+'''
 
 with open('yt_dlp/version.py', 'wt') as f:
     f.write(VERSION_FILE)

docs/Contributing.md

@@ -0,0 +1,5 @@
+---
+orphan: true
+---
+```{include} ../Contributing.md
+```

test/test_YoutubeDL.py

@@ -780,8 +780,8 @@ def expect_same_infodict(out):
         test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
         test('%(height)D', '1K')
         test('%(height)5.2D', ' 1.08K')
-        test('%(title4).10F', ('foo \'bar\' ', 'foo \'bar\'#'))
         test('%(title4)#F', 'foo_bar_test')
+        test('%(title4).10F', ('foo \'bar\' ', 'foo \'bar\'' + ('#' if compat_os_name == 'nt' else ' ')))
         if compat_os_name == 'nt':
             test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
             test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'"))

test/test_youtube_signature.py

@@ -82,6 +82,10 @@
         'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
         'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
     ),
+    (
+        'https://www.youtube.com/s/player/8040e515/player_ias.vflset/en_US/base.js',
+        'wvOFaY-yjgDuIEg5', 'HkfBFDHmgw4rsw',
+    ),
 ]
 
 

yt_dlp/YoutubeDL.py

@@ -1495,7 +1495,7 @@ def process_ie_result(self, ie_result, download=True, extra_info=None):
                 self.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls))
                 ie_result['additional_entries'] = [
                     self.extract_info(
-                        url, download, extra_info,
+                        url, download, extra_info=extra_info,
                         force_generic_extractor=self.params.get('force_generic_extractor'))
                     for url in additional_urls
                 ]
@@ -2474,10 +2474,7 @@ def is_wellformed(f):
                     info_dict['id'], automatic_captions, 'automatic captions')
             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
         if self.params.get('listformats') or interactive_format_selection:
-            if not info_dict.get('formats') and not info_dict.get('url'):
-                self.to_screen('%s has no formats' % info_dict['id'])
-            else:
-                self.list_formats(info_dict)
+            self.list_formats(info_dict)
         if list_only:
             # Without this printing, -F --print-json will not work
             self.__forced_printings(info_dict, self.prepare_filename(info_dict), incomplete=True)
@@ -3361,6 +3358,11 @@ def _list_format_headers(self, *headers):
         return headers
 
     def list_formats(self, info_dict):
+        if not info_dict.get('formats') and not info_dict.get('url'):
+            self.to_screen('%s has no formats' % info_dict['id'])
+            return
+        self.to_screen('[info] Available formats for %s:' % info_dict['id'])
+
         formats = info_dict.get('formats', [info_dict])
         new_format = self.params.get('listformats_table', True) is not False
         if new_format:
@@ -3375,7 +3377,7 @@ def list_formats(self, info_dict):
                     delim,
                     format_field(f, 'filesize', ' \t%s', func=format_bytes) + format_field(f, 'filesize_approx', '~\t%s', func=format_bytes),
                     format_field(f, 'tbr', '\t%dk'),
-                    shorten_protocol_name(f.get('protocol', '').replace('native', 'n')),
+                    shorten_protocol_name(f.get('protocol', '')),
                     delim,
                     format_field(f, 'vcodec', default='unknown').replace(
                         'none',
@@ -3411,8 +3413,6 @@ def list_formats(self, info_dict):
                 if f.get('preference') is None or f['preference'] >= -1000]
             header_line = ['format code', 'extension', 'resolution', 'note']
 
-        self.to_screen(
-            '[info] Available formats for %s:' % info_dict['id'])
         self.to_stdout(render_table(
             header_line, table,
             extra_gap=(0 if new_format else 1),

yt_dlp/__init__.py

@@ -18,6 +18,7 @@
 )
 from .compat import (
     compat_getpass,
+    compat_os_name,
     compat_shlex_quote,
     workaround_optparse_bug9161,
 )
@@ -95,7 +96,8 @@ def _real_main(argv=None):
     if opts.batchfile is not None:
         try:
             if opts.batchfile == '-':
-                write_string('Reading URLs from stdin:\n')
+                write_string('Reading URLs from stdin - EOF (%s) to end:\n' % (
+                    'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'))
                 batchfd = sys.stdin
             else:
                 batchfd = io.open(
@@ -518,7 +520,7 @@ def report_unplayable_conflict(opt_name, arg, default=False, allowed=None):
             if len(dur) == 2 and all(t is not None for t in dur):
                 remove_ranges.append(tuple(dur))
                 continue
-            parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form ?start-end')
+            parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form *start-end')
         try:
             remove_chapters_patterns.append(re.compile(regex))
         except re.error as err:

yt_dlp/downloader/common.py

@@ -397,6 +397,7 @@ def download(self, filename, info_dict, subtitle=False):
                     'status': 'finished',
                     'total_bytes': os.path.getsize(encodeFilename(filename)),
                 }, info_dict)
+                self._finish_multiline_status()
                 return True, False
 
         if subtitle is False:

yt_dlp/extractor/cbc.py

@@ -340,7 +340,7 @@ def _find_secret_formats(self, formats, video_id):
                 yield {
                     **base_format,
                     'format_id': join_nonempty('sec', height),
-                    'url': re.sub(r'(QualityLevels\()\d+(\))', fr'\<1>{bitrate}\2', base_url),
+                    'url': re.sub(r'(QualityLevels\()\d+(\))', fr'\1{bitrate}\2', base_url),
                     'width': int_or_none(video_quality.attrib.get('MaxWidth')),
                     'tbr': bitrate / 1000.0,
                     'height': height,

yt_dlp/extractor/common.py

@@ -616,7 +616,7 @@ def extract(self, url):
             kwargs = {
                 'video_id': e.video_id or self.get_temp_id(url),
                 'ie': self.IE_NAME,
-                'tb': e.traceback,
+                'tb': e.traceback or sys.exc_info()[2],
                 'expected': e.expected,
                 'cause': e.cause
             }
@@ -1574,7 +1574,7 @@ class FormatSort:
             'vcodec': {'type': 'ordered', 'regex': True,
                        'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
             'acodec': {'type': 'ordered', 'regex': True,
-                       'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
+                       'order': ['[af]lac', 'wav|aiff', 'opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
             'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
                     'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
             'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',