From d400e261cf029a3f20d364113b14de973be75404 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Fri, 3 Mar 2023 22:31:41 +0530 Subject: [devscripts] Script to generate changelog (#6220) Authored by: Grub4K --- devscripts/make_changelog.py | 491 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 491 insertions(+) create mode 100644 devscripts/make_changelog.py (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py new file mode 100644 index 000000000..b66181b53 --- /dev/null +++ b/devscripts/make_changelog.py @@ -0,0 +1,491 @@ +from __future__ import annotations + +import enum +import itertools +import json +import logging +import re +import subprocess +import sys +from collections import defaultdict +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +BASE_URL = 'https://github.com' +LOCATION_PATH = Path(__file__).parent + +logger = logging.getLogger(__name__) + + +class CommitGroup(enum.Enum): + UPSTREAM = None + PRIORITY = 'Important' + CORE = 'Core' + EXTRACTOR = 'Extractor' + DOWNLOADER = 'Downloader' + POSTPROCESSOR = 'Postprocessor' + MISC = 'Misc.' + + @classmethod + @lru_cache + def commit_lookup(cls): + return { + name: group + for group, names in { + cls.PRIORITY: {''}, + cls.UPSTREAM: {'upstream'}, + cls.CORE: { + 'aes', + 'cache', + 'compat_utils', + 'compat', + 'cookies', + 'core', + 'dependencies', + 'jsinterp', + 'outtmpl', + 'plugins', + 'update', + 'utils', + }, + cls.MISC: { + 'build', + 'cleanup', + 'devscripts', + 'docs', + 'misc', + 'test', + }, + cls.EXTRACTOR: {'extractor', 'extractors'}, + cls.DOWNLOADER: {'downloader'}, + cls.POSTPROCESSOR: {'postprocessor'}, + }.items() + for name in names + } + + @classmethod + def get(cls, value): + result = cls.commit_lookup().get(value) + if result: + logger.debug(f'Mapped {value!r} => {result.name}') + return result + + +@dataclass +class Commit: + hash: str | None + short: str + authors: list[str] + + def __str__(self): + result = f'{self.short!r}' + + if self.hash: + result += f' ({self.hash[:7]})' + + if self.authors: + authors = ', '.join(self.authors) + result += f' by {authors}' + + return result + + +@dataclass +class CommitInfo: + details: str | None + sub_details: tuple[str, ...] + message: str + issues: list[str] + commit: Commit + fixes: list[Commit] + + def key(self): + return ((self.details or '').lower(), self.sub_details, self.message) + + +class Changelog: + MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE) + + def __init__(self, groups, repo): + self._groups = groups + self._repo = repo + + def __str__(self): + return '\n'.join(self._format_groups(self._groups)).replace('\t', ' ') + + def _format_groups(self, groups): + for item in CommitGroup: + group = groups[item] + if group: + yield self.format_module(item.value, group) + + def format_module(self, name, group): + result = f'\n#### {name} changes\n' if name else '\n' + return result + '\n'.join(self._format_group(group)) + + def _format_group(self, group): + sorted_group = sorted(group, key=CommitInfo.key) + detail_groups = itertools.groupby(sorted_group, lambda item: (item.details or '').lower()) + for details, items in detail_groups: + if not details: + indent = '' + else: + yield f'- {details}' + indent = '\t' + + if details == 'cleanup': + items, cleanup_misc_items = self._filter_cleanup_misc_items(items) + + sub_detail_groups = itertools.groupby(items, lambda item: item.sub_details) + for sub_details, entries in sub_detail_groups: + if not sub_details: + for entry in entries: + yield f'{indent}- {self.format_single_change(entry)}' + continue + + prefix = f'{indent}- {", ".join(sub_details)}' + entries = list(entries) + if len(entries) == 1: + yield f'{prefix}: {self.format_single_change(entries[0])}' + continue + + yield prefix + for entry in entries: + yield f'{indent}\t- {self.format_single_change(entry)}' + + if details == 'cleanup' and cleanup_misc_items: + yield from self._format_cleanup_misc_sub_group(cleanup_misc_items) + + def _filter_cleanup_misc_items(self, items): + cleanup_misc_items = defaultdict(list) + non_misc_items = [] + for item in items: + if self.MISC_RE.search(item.message): + cleanup_misc_items[tuple(item.commit.authors)].append(item) + else: + non_misc_items.append(item) + + return non_misc_items, cleanup_misc_items + + def _format_cleanup_misc_sub_group(self, group): + prefix = '\t- Miscellaneous' + if len(group) == 1: + yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}' + return + + yield prefix + for message in self._format_cleanup_misc_items(group): + yield f'\t\t- {message}' + + def _format_cleanup_misc_items(self, group): + for authors, infos in group.items(): + message = ', '.join( + self._format_message_link(None, info.commit.hash) + for info in sorted(infos, key=lambda item: item.commit.hash or '')) + yield f'{message} by {self._format_authors(authors)}' + + def format_single_change(self, info): + message = self._format_message_link(info.message, info.commit.hash) + if info.issues: + message = f'{message} ({self._format_issues(info.issues)})' + + if info.commit.authors: + message = f'{message} by {self._format_authors(info.commit.authors)}' + + if info.fixes: + fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes) + + authors = sorted({author for fix in info.fixes for author in fix.authors}, key=str.casefold) + if authors != info.commit.authors: + fix_message = f'{fix_message} by {self._format_authors(authors)}' + + message = f'{message} (With fixes in {fix_message})' + + return message + + def _format_message_link(self, message, hash): + assert message or hash, 'Improperly defined commit message or override' + message = message if message else hash[:7] + return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message + + def _format_issues(self, issues): + return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues) + + @staticmethod + def _format_authors(authors): + return ', '.join(f'[{author}]({BASE_URL}/{author})' for author in authors) + + @property + def repo_url(self): + return f'{BASE_URL}/{self._repo}' + + +class CommitRange: + COMMAND = 'git' + COMMIT_SEPARATOR = '-----' + + AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE) + MESSAGE_RE = re.compile(r''' + (?:\[ + (?P[^\]\/:,]+) + (?:/(?P
[^\]:,]+))? + (?:[:,](?P[^\]]+))? + \]\ )? + (?:`?(?P[^:`]+)`?: )? + (?P.+?) + (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? + ''', re.VERBOSE | re.DOTALL) + EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) + FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+for)?|Revert)\s+([\da-f]{40})') + UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') + + def __init__(self, start, end, default_author=None) -> None: + self._start = start + self._end = end + self._commits, self._fixes = self._get_commits_and_fixes(default_author) + self._commits_added = [] + + @classmethod + def from_single(cls, commitish='HEAD', default_author=None): + start_commitish = cls.get_prev_tag(commitish) + end_commitish = cls.get_next_tag(commitish) + if start_commitish == end_commitish: + start_commitish = cls.get_prev_tag(f'{commitish}~') + logger.info(f'Determined range from {commitish!r}: {start_commitish}..{end_commitish}') + return cls(start_commitish, end_commitish, default_author) + + @classmethod + def get_prev_tag(cls, commitish): + command = [cls.COMMAND, 'describe', '--tags', '--abbrev=0', '--exclude=*[^0-9.]*', commitish] + return subprocess.check_output(command, text=True).strip() + + @classmethod + def get_next_tag(cls, commitish): + result = subprocess.run( + [cls.COMMAND, 'describe', '--contains', '--abbrev=0', commitish], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + if result.returncode: + return 'HEAD' + + return result.stdout.partition('~')[0].strip() + + def __iter__(self): + return iter(itertools.chain(self._commits.values(), self._commits_added)) + + def __len__(self): + return len(self._commits) + len(self._commits_added) + + def __contains__(self, commit): + if isinstance(commit, Commit): + if not commit.hash: + return False + commit = commit.hash + + return commit in self._commits + + def _is_ancestor(self, commitish): + return bool(subprocess.call( + [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start])) + + def _get_commits_and_fixes(self, default_author): + result = subprocess.check_output([ + self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}', + f'{self._start}..{self._end}'], text=True) + + commits = {} + fixes = defaultdict(list) + lines = iter(result.splitlines(False)) + for line in lines: + commit_hash = line + short = next(lines) + skip = short.startswith('Release ') or short == '[version] update' + + authors = [default_author] if default_author else [] + for line in iter(lambda: next(lines), self.COMMIT_SEPARATOR): + match = self.AUTHOR_INDICATOR_RE.match(line) + if match: + authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold) + + commit = Commit(commit_hash, short, authors) + if skip: + logger.debug(f'Skipped commit: {commit}') + continue + + fix_match = self.FIXES_RE.search(commit.short) + if fix_match: + commitish = fix_match.group(1) + fixes[commitish].append(commit) + + commits[commit.hash] = commit + + for commitish, fix_commits in fixes.items(): + if commitish in commits: + hashes = ', '.join(commit.hash[:7] for commit in fix_commits) + logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}') + for fix_commit in fix_commits: + del commits[fix_commit.hash] + else: + logger.debug(f'Commit with fixes not in changes: {commitish[:7]}') + + return commits, fixes + + def apply_overrides(self, overrides): + for override in overrides: + when = override.get('when') + if when and when not in self and when != self._start: + logger.debug(f'Ignored {when!r}, not in commits {self._start!r}') + continue + + override_hash = override.get('hash') + if override['action'] == 'add': + commit = Commit(override.get('hash'), override['short'], override.get('authors') or []) + logger.info(f'ADD {commit}') + self._commits_added.append(commit) + + elif override['action'] == 'remove': + if override_hash in self._commits: + logger.info(f'REMOVE {self._commits[override_hash]}') + del self._commits[override_hash] + + elif override['action'] == 'change': + if override_hash not in self._commits: + continue + commit = Commit(override_hash, override['short'], override['authors']) + logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}') + self._commits[commit.hash] = commit + + self._commits = {key: value for key, value in reversed(self._commits.items())} + + def groups(self): + groups = defaultdict(list) + for commit in self: + upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short) + if upstream_re: + commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}' + + match = self.MESSAGE_RE.fullmatch(commit.short) + if not match: + logger.error(f'Error parsing short commit message: {commit.short!r}') + continue + + prefix, details, sub_details, sub_details_alt, message, issues = match.groups() + group = None + if prefix: + if prefix == 'priority': + prefix, _, details = (details or '').partition('/') + logger.debug(f'Priority: {message!r}') + group = CommitGroup.PRIORITY + + if not details and prefix: + if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'): + logger.debug(f'Replaced details with {prefix!r}') + details = prefix or None + + if details == 'common': + details = None + + if details: + details = details.strip() + + else: + group = CommitGroup.CORE + + sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.lower().replace(':', ',') + sub_details = tuple(filter(None, map(str.strip, sub_details.split(',')))) + + issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else [] + + if not group: + group = CommitGroup.get(prefix.lower()) + if not group: + if self.EXTRACTOR_INDICATOR_RE.search(commit.short): + group = CommitGroup.EXTRACTOR + else: + group = CommitGroup.POSTPROCESSOR + logger.warning(f'Failed to map {commit.short!r}, selected {group.name}') + + commit_info = CommitInfo( + details, sub_details, message.strip(), + issues, commit, self._fixes[commit.hash]) + logger.debug(f'Resolved {commit.short!r} to {commit_info!r}') + groups[group].append(commit_info) + + return groups + + +def get_new_contributors(contributors_path, commits): + contributors = set() + if contributors_path.exists(): + with contributors_path.open() as file: + for line in filter(None, map(str.strip, file)): + author, _, _ = line.partition(' (') + authors = author.split('/') + contributors.update(map(str.casefold, authors)) + + new_contributors = set() + for commit in commits: + for author in commit.authors: + author_folded = author.casefold() + if author_folded not in contributors: + contributors.add(author_folded) + new_contributors.add(author) + + return sorted(new_contributors, key=str.casefold) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser( + description='Create a changelog markdown from a git commit range') + parser.add_argument( + 'commitish', default='HEAD', nargs='?', + help='The commitish to create the range from (default: %(default)s)') + parser.add_argument( + '-v', '--verbosity', action='count', default=0, + help='increase verbosity (can be used twice)') + parser.add_argument( + '-c', '--contributors', action='store_true', + help='update CONTRIBUTORS file (default: %(default)s)') + parser.add_argument( + '--contributors-path', type=Path, default=LOCATION_PATH.parent / 'CONTRIBUTORS', + help='path to the CONTRIBUTORS file') + parser.add_argument( + '--no-override', action='store_true', + help='skip override json in commit generation (default: %(default)s)') + parser.add_argument( + '--override-path', type=Path, default=LOCATION_PATH / 'changelog_override.json', + help='path to the changelog_override.json file') + parser.add_argument( + '--default-author', default='pukkandan', + help='the author to use without a author indicator (default: %(default)s)') + parser.add_argument( + '--repo', default='yt-dlp/yt-dlp', + help='the github repository to use for the operations (default: %(default)s)') + args = parser.parse_args() + + logging.basicConfig( + datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', + level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) + + commits = CommitRange.from_single(args.commitish, args.default_author) + + if not args.no_override: + if args.override_path.exists(): + with args.override_path.open() as file: + overrides = json.load(file) + commits.apply_overrides(overrides) + else: + logger.warning(f'File {args.override_path.as_posix()} does not exist') + + logger.info(f'Loaded {len(commits)} commits') + + new_contributors = get_new_contributors(args.contributors_path, commits) + if new_contributors: + if args.contributors: + with args.contributors_path.open('a') as file: + file.writelines(f'{contributor}\n' for contributor in new_contributors) + logger.info(f'New contributors: {", ".join(new_contributors)}') + + print(Changelog(commits.groups(), args.repo)) -- cgit v1.2.3 From 93449642815a6973a4b09b289982ca7e1f961b5f Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Fri, 3 Mar 2023 22:39:09 +0100 Subject: Fix d400e261cf029a3f20d364113b14de973be75404 Authored by: Grub4K --- devscripts/make_changelog.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index b66181b53..07aa3285b 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -127,7 +127,9 @@ class Changelog: def _format_group(self, group): sorted_group = sorted(group, key=CommitInfo.key) detail_groups = itertools.groupby(sorted_group, lambda item: (item.details or '').lower()) - for details, items in detail_groups: + for _, items in detail_groups: + items = list(items) + details = items[0].details if not details: indent = '' else: @@ -137,15 +139,15 @@ class Changelog: if details == 'cleanup': items, cleanup_misc_items = self._filter_cleanup_misc_items(items) - sub_detail_groups = itertools.groupby(items, lambda item: item.sub_details) + sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details))) for sub_details, entries in sub_detail_groups: if not sub_details: for entry in entries: yield f'{indent}- {self.format_single_change(entry)}' continue - prefix = f'{indent}- {", ".join(sub_details)}' entries = list(entries) + prefix = f'{indent}- {", ".join(entries[0].sub_details)}' if len(entries) == 1: yield f'{prefix}: {self.format_single_change(entries[0])}' continue @@ -232,12 +234,12 @@ class CommitRange: (?:/(?P
[^\]:,]+))? (?:[:,](?P[^\]]+))? \]\ )? - (?:`?(?P[^:`]+)`?: )? + (?:(?P`?[^:`]+`?): )? (?P.+?) (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? ''', re.VERBOSE | re.DOTALL) EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) - FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+for)?|Revert)\s+([\da-f]{40})') + FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})') UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') def __init__(self, start, end, default_author=None) -> None: @@ -391,7 +393,7 @@ class CommitRange: else: group = CommitGroup.CORE - sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.lower().replace(':', ',') + sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',') sub_details = tuple(filter(None, map(str.strip, sub_details.split(',')))) issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else [] -- cgit v1.2.3 From 7accdd9845fe7ce9d0aa5a9d16faaa489c1294eb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 4 Mar 2023 16:39:23 +0530 Subject: [devscripts] `make_changelog`: Stop at `Release ...` commit Closes #6415 --- devscripts/make_changelog.py | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 07aa3285b..722315333 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -248,30 +248,6 @@ class CommitRange: self._commits, self._fixes = self._get_commits_and_fixes(default_author) self._commits_added = [] - @classmethod - def from_single(cls, commitish='HEAD', default_author=None): - start_commitish = cls.get_prev_tag(commitish) - end_commitish = cls.get_next_tag(commitish) - if start_commitish == end_commitish: - start_commitish = cls.get_prev_tag(f'{commitish}~') - logger.info(f'Determined range from {commitish!r}: {start_commitish}..{end_commitish}') - return cls(start_commitish, end_commitish, default_author) - - @classmethod - def get_prev_tag(cls, commitish): - command = [cls.COMMAND, 'describe', '--tags', '--abbrev=0', '--exclude=*[^0-9.]*', commitish] - return subprocess.check_output(command, text=True).strip() - - @classmethod - def get_next_tag(cls, commitish): - result = subprocess.run( - [cls.COMMAND, 'describe', '--contains', '--abbrev=0', commitish], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) - if result.returncode: - return 'HEAD' - - return result.stdout.partition('~')[0].strip() - def __iter__(self): return iter(itertools.chain(self._commits.values(), self._commits_added)) @@ -293,13 +269,12 @@ class CommitRange: def _get_commits_and_fixes(self, default_author): result = subprocess.check_output([ self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}', - f'{self._start}..{self._end}'], text=True) + f'{self._start}..{self._end}' if self._start else self._end], text=True) commits = {} fixes = defaultdict(list) lines = iter(result.splitlines(False)) - for line in lines: - commit_hash = line + for i, commit_hash in enumerate(lines): short = next(lines) skip = short.startswith('Release ') or short == '[version] update' @@ -310,9 +285,12 @@ class CommitRange: authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold) commit = Commit(commit_hash, short, authors) - if skip: + if skip and (self._start or not i): logger.debug(f'Skipped commit: {commit}') continue + elif skip: + logger.debug(f'Reached Release commit, breaking: {commit}') + break fix_match = self.FIXES_RE.search(commit.short) if fix_match: @@ -471,7 +449,7 @@ if __name__ == '__main__': datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) - commits = CommitRange.from_single(args.commitish, args.default_author) + commits = CommitRange(None, args.commitish, args.default_author) if not args.no_override: if args.override_path.exists(): -- cgit v1.2.3 From 392389b7df7b818f794b231f14dc396d4875fbad Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 4 Mar 2023 22:40:08 +0530 Subject: [cleanup] Misc --- devscripts/make_changelog.py | 49 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 25 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 722315333..b159bc1b9 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -1,19 +1,26 @@ from __future__ import annotations +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import enum import itertools import json import logging import re -import subprocess -import sys from collections import defaultdict from dataclasses import dataclass from functools import lru_cache from pathlib import Path +from devscripts.utils import read_file, run_process, write_file + BASE_URL = 'https://github.com' LOCATION_PATH = Path(__file__).parent +HASH_LENGTH = 7 logger = logging.getLogger(__name__) @@ -82,7 +89,7 @@ class Commit: result = f'{self.short!r}' if self.hash: - result += f' ({self.hash[:7]})' + result += f' ({self.hash[:HASH_LENGTH]})' if self.authors: authors = ', '.join(self.authors) @@ -208,7 +215,7 @@ class Changelog: def _format_message_link(self, message, hash): assert message or hash, 'Improperly defined commit message or override' - message = message if message else hash[:7] + message = message if message else hash[:HASH_LENGTH] return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message def _format_issues(self, issues): @@ -242,9 +249,8 @@ class CommitRange: FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})') UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') - def __init__(self, start, end, default_author=None) -> None: - self._start = start - self._end = end + def __init__(self, start, end, default_author=None): + self._start, self._end = start, end self._commits, self._fixes = self._get_commits_and_fixes(default_author) self._commits_added = [] @@ -262,14 +268,10 @@ class CommitRange: return commit in self._commits - def _is_ancestor(self, commitish): - return bool(subprocess.call( - [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start])) - def _get_commits_and_fixes(self, default_author): - result = subprocess.check_output([ + result = run_process( self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}', - f'{self._start}..{self._end}' if self._start else self._end], text=True) + f'{self._start}..{self._end}' if self._start else self._end).stdout commits = {} fixes = defaultdict(list) @@ -301,12 +303,12 @@ class CommitRange: for commitish, fix_commits in fixes.items(): if commitish in commits: - hashes = ', '.join(commit.hash[:7] for commit in fix_commits) - logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}') + hashes = ', '.join(commit.hash[:HASH_LENGTH] for commit in fix_commits) + logger.info(f'Found fix(es) for {commitish[:HASH_LENGTH]}: {hashes}') for fix_commit in fix_commits: del commits[fix_commit.hash] else: - logger.debug(f'Commit with fixes not in changes: {commitish[:7]}') + logger.debug(f'Commit with fixes not in changes: {commitish[:HASH_LENGTH]}') return commits, fixes @@ -397,11 +399,10 @@ class CommitRange: def get_new_contributors(contributors_path, commits): contributors = set() if contributors_path.exists(): - with contributors_path.open() as file: - for line in filter(None, map(str.strip, file)): - author, _, _ = line.partition(' (') - authors = author.split('/') - contributors.update(map(str.casefold, authors)) + for line in read_file(contributors_path).splitlines(): + author, _, _ = line.strip().partition(' (') + authors = author.split('/') + contributors.update(map(str.casefold, authors)) new_contributors = set() for commit in commits: @@ -453,8 +454,7 @@ if __name__ == '__main__': if not args.no_override: if args.override_path.exists(): - with args.override_path.open() as file: - overrides = json.load(file) + overrides = json.loads(read_file(args.override_path)) commits.apply_overrides(overrides) else: logger.warning(f'File {args.override_path.as_posix()} does not exist') @@ -464,8 +464,7 @@ if __name__ == '__main__': new_contributors = get_new_contributors(args.contributors_path, commits) if new_contributors: if args.contributors: - with args.contributors_path.open('a') as file: - file.writelines(f'{contributor}\n' for contributor in new_contributors) + write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a') logger.info(f'New contributors: {", ".join(new_contributors)}') print(Changelog(commits.groups(), args.repo)) -- cgit v1.2.3 From 23c39a4beadee382060bb47fdaa21316ca707d38 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Mon, 3 Apr 2023 07:22:11 +0200 Subject: [devscripts] `make_changelog`: Various improvements - Make single items collapse into one line - Don't hide "Important changes" in `
` - Move upstream merge into priority - Properly support comma separated prefixes Authored by: Grub4K --- devscripts/make_changelog.py | 179 ++++++++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 78 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index b159bc1b9..1b7e251ee 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -26,7 +26,6 @@ logger = logging.getLogger(__name__) class CommitGroup(enum.Enum): - UPSTREAM = None PRIORITY = 'Important' CORE = 'Core' EXTRACTOR = 'Extractor' @@ -34,6 +33,11 @@ class CommitGroup(enum.Enum): POSTPROCESSOR = 'Postprocessor' MISC = 'Misc.' + @classmethod + @property + def ignorable_prefixes(cls): + return ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream') + @classmethod @lru_cache def commit_lookup(cls): @@ -41,7 +45,6 @@ class CommitGroup(enum.Enum): name: group for group, names in { cls.PRIORITY: {''}, - cls.UPSTREAM: {'upstream'}, cls.CORE: { 'aes', 'cache', @@ -54,6 +57,7 @@ class CommitGroup(enum.Enum): 'outtmpl', 'plugins', 'update', + 'upstream', 'utils', }, cls.MISC: { @@ -111,22 +115,36 @@ class CommitInfo: return ((self.details or '').lower(), self.sub_details, self.message) +def unique(items): + return sorted({item.strip().lower(): item for item in items if item}.values()) + + class Changelog: MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE) + ALWAYS_SHOWN = (CommitGroup.PRIORITY,) - def __init__(self, groups, repo): + def __init__(self, groups, repo, collapsible=False): self._groups = groups self._repo = repo + self._collapsible = collapsible def __str__(self): return '\n'.join(self._format_groups(self._groups)).replace('\t', ' ') def _format_groups(self, groups): + first = True for item in CommitGroup: + if self._collapsible and item not in self.ALWAYS_SHOWN and first: + first = False + yield '\n

Changelog

\n' + group = groups[item] if group: yield self.format_module(item.value, group) + if self._collapsible: + yield '\n
' + def format_module(self, name, group): result = f'\n#### {name} changes\n' if name else '\n' return result + '\n'.join(self._format_group(group)) @@ -137,62 +155,52 @@ class Changelog: for _, items in detail_groups: items = list(items) details = items[0].details - if not details: - indent = '' - else: - yield f'- {details}' - indent = '\t' if details == 'cleanup': - items, cleanup_misc_items = self._filter_cleanup_misc_items(items) + items = self._prepare_cleanup_misc_items(items) + + prefix = '-' + if details: + if len(items) == 1: + prefix = f'- **{details}**:' + else: + yield f'- **{details}**' + prefix = '\t-' sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details))) for sub_details, entries in sub_detail_groups: if not sub_details: for entry in entries: - yield f'{indent}- {self.format_single_change(entry)}' + yield f'{prefix} {self.format_single_change(entry)}' continue entries = list(entries) - prefix = f'{indent}- {", ".join(entries[0].sub_details)}' + sub_prefix = f'{prefix} {", ".join(entries[0].sub_details)}' if len(entries) == 1: - yield f'{prefix}: {self.format_single_change(entries[0])}' + yield f'{sub_prefix}: {self.format_single_change(entries[0])}' continue - yield prefix + yield sub_prefix for entry in entries: - yield f'{indent}\t- {self.format_single_change(entry)}' - - if details == 'cleanup' and cleanup_misc_items: - yield from self._format_cleanup_misc_sub_group(cleanup_misc_items) + yield f'\t{prefix} {self.format_single_change(entry)}' - def _filter_cleanup_misc_items(self, items): + def _prepare_cleanup_misc_items(self, items): cleanup_misc_items = defaultdict(list) - non_misc_items = [] + sorted_items = [] for item in items: if self.MISC_RE.search(item.message): cleanup_misc_items[tuple(item.commit.authors)].append(item) else: - non_misc_items.append(item) - - return non_misc_items, cleanup_misc_items - - def _format_cleanup_misc_sub_group(self, group): - prefix = '\t- Miscellaneous' - if len(group) == 1: - yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}' - return + sorted_items.append(item) - yield prefix - for message in self._format_cleanup_misc_items(group): - yield f'\t\t- {message}' + for commit_infos in cleanup_misc_items.values(): + sorted_items.append(CommitInfo( + 'cleanup', ('Miscellaneous',), ', '.join( + self._format_message_link(None, info.commit.hash) + for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')), + [], Commit(None, '', commit_infos[0].commit.authors), [])) - def _format_cleanup_misc_items(self, group): - for authors, infos in group.items(): - message = ', '.join( - self._format_message_link(None, info.commit.hash) - for info in sorted(infos, key=lambda item: item.commit.hash or '')) - yield f'{message} by {self._format_authors(authors)}' + return sorted_items def format_single_change(self, info): message = self._format_message_link(info.message, info.commit.hash) @@ -236,12 +244,8 @@ class CommitRange: AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE) MESSAGE_RE = re.compile(r''' - (?:\[ - (?P[^\]\/:,]+) - (?:/(?P
[^\]:,]+))? - (?:[:,](?P[^\]]+))? - \]\ )? - (?:(?P`?[^:`]+`?): )? + (?:\[(?P[^\]]+)\]\ )? + (?:(?P`?[^:`]+`?): )? (?P.+?) (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? ''', re.VERBOSE | re.DOTALL) @@ -340,60 +344,76 @@ class CommitRange: self._commits = {key: value for key, value in reversed(self._commits.items())} def groups(self): - groups = defaultdict(list) + group_dict = defaultdict(list) for commit in self: - upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short) + upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short) if upstream_re: - commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}' + commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}' match = self.MESSAGE_RE.fullmatch(commit.short) if not match: logger.error(f'Error parsing short commit message: {commit.short!r}') continue - prefix, details, sub_details, sub_details_alt, message, issues = match.groups() - group = None - if prefix: - if prefix == 'priority': - prefix, _, details = (details or '').partition('/') - logger.debug(f'Priority: {message!r}') - group = CommitGroup.PRIORITY - - if not details and prefix: - if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'): - logger.debug(f'Replaced details with {prefix!r}') - details = prefix or None - - if details == 'common': - details = None - - if details: - details = details.strip() + prefix, sub_details_alt, message, issues = match.groups() + issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else [] + if prefix: + groups, details, sub_details = zip(*map(self.details_from_prefix, prefix.split(','))) + group = next(iter(filter(None, groups)), None) + details = ', '.join(unique(details)) + sub_details = list(itertools.chain.from_iterable(sub_details)) else: group = CommitGroup.CORE + details = None + sub_details = [] - sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',') - sub_details = tuple(filter(None, map(str.strip, sub_details.split(',')))) - - issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else [] + if sub_details_alt: + sub_details.append(sub_details_alt) + sub_details = tuple(unique(sub_details)) if not group: - group = CommitGroup.get(prefix.lower()) - if not group: - if self.EXTRACTOR_INDICATOR_RE.search(commit.short): - group = CommitGroup.EXTRACTOR - else: - group = CommitGroup.POSTPROCESSOR - logger.warning(f'Failed to map {commit.short!r}, selected {group.name}') + if self.EXTRACTOR_INDICATOR_RE.search(commit.short): + group = CommitGroup.EXTRACTOR + else: + group = CommitGroup.POSTPROCESSOR + logger.warning(f'Failed to map {commit.short!r}, selected {group.name.lower()}') commit_info = CommitInfo( details, sub_details, message.strip(), issues, commit, self._fixes[commit.hash]) + logger.debug(f'Resolved {commit.short!r} to {commit_info!r}') - groups[group].append(commit_info) + group_dict[group].append(commit_info) + + return group_dict + + @staticmethod + def details_from_prefix(prefix): + if not prefix: + return CommitGroup.CORE, None, () - return groups + prefix, _, details = prefix.partition('/') + prefix = prefix.strip().lower() + details = details.strip() + + group = CommitGroup.get(prefix) + if group is CommitGroup.PRIORITY: + prefix, _, details = details.partition('/') + + if not details and prefix and prefix not in CommitGroup.ignorable_prefixes: + logger.debug(f'Replaced details with {prefix!r}') + details = prefix or None + + if details == 'common': + details = None + + if details: + details, *sub_details = details.split(':') + else: + sub_details = [] + + return group, details, sub_details def get_new_contributors(contributors_path, commits): @@ -444,6 +464,9 @@ if __name__ == '__main__': parser.add_argument( '--repo', default='yt-dlp/yt-dlp', help='the github repository to use for the operations (default: %(default)s)') + parser.add_argument( + '--collapsible', action='store_true', + help='make changelog collapsible (default: %(default)s)') args = parser.parse_args() logging.basicConfig( @@ -467,4 +490,4 @@ if __name__ == '__main__': write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a') logger.info(f'New contributors: {", ".join(new_contributors)}') - print(Changelog(commits.groups(), args.repo)) + print(Changelog(commits.groups(), args.repo, args.collapsible)) -- cgit v1.2.3 From ad54c9130e793ce433bf9da334fa80df9f3aee58 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 21 Jun 2023 09:21:20 +0530 Subject: [cleanup] Misc Closes #6288, Closes #7197, Closes #7265, Closes #7353, Closes #5773 Authored by: mikf, freezboltz, pukkandan --- devscripts/make_changelog.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 1b7e251ee..2fcdc06d7 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -44,7 +44,7 @@ class CommitGroup(enum.Enum): return { name: group for group, names in { - cls.PRIORITY: {''}, + cls.PRIORITY: {'priority'}, cls.CORE: { 'aes', 'cache', @@ -68,7 +68,7 @@ class CommitGroup(enum.Enum): 'misc', 'test', }, - cls.EXTRACTOR: {'extractor', 'extractors'}, + cls.EXTRACTOR: {'extractor'}, cls.DOWNLOADER: {'downloader'}, cls.POSTPROCESSOR: {'postprocessor'}, }.items() @@ -323,7 +323,7 @@ class CommitRange: logger.debug(f'Ignored {when!r}, not in commits {self._start!r}') continue - override_hash = override.get('hash') + override_hash = override.get('hash') or when if override['action'] == 'add': commit = Commit(override.get('hash'), override['short'], override.get('authors') or []) logger.info(f'ADD {commit}') @@ -337,7 +337,7 @@ class CommitRange: elif override['action'] == 'change': if override_hash not in self._commits: continue - commit = Commit(override_hash, override['short'], override['authors']) + commit = Commit(override_hash, override['short'], override.get('authors') or []) logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}') self._commits[commit.hash] = commit @@ -348,7 +348,7 @@ class CommitRange: for commit in self: upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short) if upstream_re: - commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}' + commit.short = f'[core/upstream] Merged with youtube-dl {upstream_re.group(1)}' match = self.MESSAGE_RE.fullmatch(commit.short) if not match: @@ -394,10 +394,10 @@ class CommitRange: return CommitGroup.CORE, None, () prefix, _, details = prefix.partition('/') - prefix = prefix.strip().lower() + prefix = prefix.strip() details = details.strip() - group = CommitGroup.get(prefix) + group = CommitGroup.get(prefix.lower()) if group is CommitGroup.PRIORITY: prefix, _, details = details.partition('/') -- cgit v1.2.3 From 812cdfa06c33a40e73a8e04b3e6f42c084666a43 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 22 Jun 2023 10:02:38 +0530 Subject: [cleanup] Misc --- devscripts/make_changelog.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 2fcdc06d7..0bcfa6ae7 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -196,7 +196,7 @@ class Changelog: for commit_infos in cleanup_misc_items.values(): sorted_items.append(CommitInfo( 'cleanup', ('Miscellaneous',), ', '.join( - self._format_message_link(None, info.commit.hash) + self._format_message_link(None, info.commit.hash).strip() for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')), [], Commit(None, '', commit_infos[0].commit.authors), [])) @@ -205,10 +205,10 @@ class Changelog: def format_single_change(self, info): message = self._format_message_link(info.message, info.commit.hash) if info.issues: - message = f'{message} ({self._format_issues(info.issues)})' + message = message.replace('\n', f' ({self._format_issues(info.issues)})\n', 1) if info.commit.authors: - message = f'{message} by {self._format_authors(info.commit.authors)}' + message = message.replace('\n', f' by {self._format_authors(info.commit.authors)}\n', 1) if info.fixes: fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes) @@ -217,14 +217,16 @@ class Changelog: if authors != info.commit.authors: fix_message = f'{fix_message} by {self._format_authors(authors)}' - message = f'{message} (With fixes in {fix_message})' + message = message.replace('\n', f' (With fixes in {fix_message})\n', 1) - return message + return message[:-1] def _format_message_link(self, message, hash): assert message or hash, 'Improperly defined commit message or override' message = message if message else hash[:HASH_LENGTH] - return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message + if not hash: + return f'{message}\n' + return f'[{message}\n'.replace('\n', f']({self.repo_url}/commit/{hash})\n', 1) def _format_issues(self, issues): return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues) -- cgit v1.2.3 From fa44802809d189fca0f4782263d48d6533384503 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 6 Jul 2023 17:34:51 +0530 Subject: [devscripts/make_changelog] Skip reverted commits --- devscripts/make_changelog.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 0bcfa6ae7..eb0e3082f 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -252,6 +252,7 @@ class CommitRange: (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? ''', re.VERBOSE | re.DOTALL) EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) + REVERT_RE = re.compile(r'(?i:Revert)\s+([\da-f]{40})') FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})') UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') @@ -279,7 +280,7 @@ class CommitRange: self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}', f'{self._start}..{self._end}' if self._start else self._end).stdout - commits = {} + commits, reverts = {}, {} fixes = defaultdict(list) lines = iter(result.splitlines(False)) for i, commit_hash in enumerate(lines): @@ -300,6 +301,11 @@ class CommitRange: logger.debug(f'Reached Release commit, breaking: {commit}') break + revert_match = self.REVERT_RE.fullmatch(commit.short) + if revert_match: + reverts[revert_match.group(1)] = commit + continue + fix_match = self.FIXES_RE.search(commit.short) if fix_match: commitish = fix_match.group(1) @@ -307,6 +313,13 @@ class CommitRange: commits[commit.hash] = commit + for commitish, revert_commit in reverts.items(): + reverted = commits.pop(commitish, None) + if reverted: + logger.debug(f'{commit} fully reverted {reverted}') + else: + commits[revert_commit.hash] = revert_commit + for commitish, fix_commits in fixes.items(): if commitish in commits: hashes = ', '.join(commit.hash[:HASH_LENGTH] for commit in fix_commits) -- cgit v1.2.3 From 337734d4a8a6500bc65434843db346b5cbd05e81 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 6 Jul 2023 20:09:42 +0530 Subject: [cleanup] Misc --- devscripts/make_changelog.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index eb0e3082f..3ad4c5408 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -55,6 +55,7 @@ class CommitGroup(enum.Enum): 'dependencies', 'jsinterp', 'outtmpl', + 'formats', 'plugins', 'update', 'upstream', @@ -68,9 +69,9 @@ class CommitGroup(enum.Enum): 'misc', 'test', }, - cls.EXTRACTOR: {'extractor'}, - cls.DOWNLOADER: {'downloader'}, - cls.POSTPROCESSOR: {'postprocessor'}, + cls.EXTRACTOR: {'extractor', 'ie'}, + cls.DOWNLOADER: {'downloader', 'fd'}, + cls.POSTPROCESSOR: {'postprocessor', 'pp'}, }.items() for name in names } -- cgit v1.2.3 From c365dba8430ee33abda85d31f95128605bf240eb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 15 Jul 2023 14:30:08 +0530 Subject: [networking] Add module (#2861) No actual changes - code is only moved around --- devscripts/make_changelog.py | 1 + 1 file changed, 1 insertion(+) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 3ad4c5408..157c66126 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -54,6 +54,7 @@ class CommitGroup(enum.Enum): 'core', 'dependencies', 'jsinterp', + 'networking', 'outtmpl', 'formats', 'plugins', -- cgit v1.2.3 From 62b5c94cadaa5f596dc1a7083db9db12efe357be Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 22 Jul 2023 09:08:12 +0530 Subject: [cleanup] Misc fixes Closes #7528 --- devscripts/make_changelog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devscripts/make_changelog.py') diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 157c66126..84f72d52f 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -53,10 +53,10 @@ class CommitGroup(enum.Enum): 'cookies', 'core', 'dependencies', + 'formats', 'jsinterp', 'networking', 'outtmpl', - 'formats', 'plugins', 'update', 'upstream', @@ -254,7 +254,7 @@ class CommitRange: (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? ''', re.VERBOSE | re.DOTALL) EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) - REVERT_RE = re.compile(r'(?i:Revert)\s+([\da-f]{40})') + REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})') FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})') UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') -- cgit v1.2.3