From 5b61ea818ef2ab93bb4f43d5b0bbe3d140f720ce Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sat, 12 Mar 2022 11:12:32 +0200 Subject: [PATCH 01/10] pr: support output to stdout With "-o -" write mbox with no cover to stdout. Signed-off-by: Amir Goldstein --- b4/command.py | 2 ++ b4/pr.py | 7 +++++++ man/b4.5.rst | 2 ++ 3 files changed, 11 insertions(+) diff --git a/b4/command.py b/b4/command.py index 3a2d58f3..a06d5ac7 100644 --- a/b4/command.py +++ b/b4/command.py @@ -191,6 +191,8 @@ def cmd(): 'the identity must match a [sendemail "identity"] config section')) sp_pr.add_argument('--dry-run', dest='dryrun', action='store_true', default=False, help='Force a --dry-run on git-send-email invocation (use with -s)') + sp_pr.add_argument('--no-cover', dest='nocover', action='store_true', default=False, + help='Do not save the cover letter (on by default when using -o -)') sp_pr.add_argument('msgid', nargs='?', help='Message ID to process, or pipe a raw message') sp_pr.set_defaults(func=cmd_pr) diff --git a/b4/pr.py b/b4/pr.py index 5a661805..f7088ef7 100644 --- a/b4/pr.py +++ b/b4/pr.py @@ -569,6 +569,13 @@ def main(cmdargs): logger.info(out) sys.exit(ecode) + # With "-o -" write mbox with no cover to stdout + if cmdargs.outmbox == '-': + b4.save_git_am_mbox(msgs[1:], sys.stdout) + sys.exit(0) + elif cmdargs.nocover: + msgs = msgs[1:] + config = b4.get_main_config() if config.get('save-maildirs', 'no') == 'yes': save_maildir = True diff --git a/man/b4.5.rst b/man/b4.5.rst index 19e96af4..ad7103b9 100644 --- a/man/b4.5.rst +++ b/man/b4.5.rst @@ -163,6 +163,8 @@ optional arguments: -l, --retrieve-links Attempt to retrieve any Link: URLs (use with -e) -f MAILFROM, --from-addr MAILFROM Use this From: in exploded messages (use with -e) + --no-cover + Do not save the cover letter (on by default when using -o -) *Example*: b4 pr 202003292120.2BDCB41@keescook From 925fab9971ea47e66500e46dbc7387b719bf5ec5 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Mon, 14 Mar 2022 14:37:10 +0200 Subject: [PATCH 02/10] pr: support pr-tracker reply msgid b4 pr cannot auto-discover merge-base on a merged pull request. In order to explode a merged PR into patches mbox, provide the msgid of the PR tracker reply, which contains the merge commit id. Don't try to read msgid's from stdin if provided as argument. Signed-off-by: Amir Goldstein --- b4/__init__.py | 2 ++ b4/pr.py | 61 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/b4/__init__.py b/b4/__init__.py index 0d506bba..5d4fc7c0 100644 --- a/b4/__init__.py +++ b/b4/__init__.py @@ -104,6 +104,7 @@ def _dkim_log_filter(record): 'linkmask': LOREADDR + '/r/%s', 'trailer-order': DEFAULT_TRAILER_ORDER, 'listid-preference': '*.feeds.kernel.org,*.linux.dev,*.kernel.org,*', + 'pr-tracker-email': 'pr-tracker-bot@kernel.org', 'save-maildirs': 'no', # off: do not bother checking attestation # check: print an attaboy when attestation is found @@ -860,6 +861,7 @@ def __init__(self, msg): self.pr_repo = None self.pr_ref = None self.pr_tip_commit = None + self.pr_merge_commit = None self.pr_remote_tip_commit = None # Patchwork hash diff --git a/b4/pr.py b/b4/pr.py index f7088ef7..55b4d56f 100644 --- a/b4/pr.py +++ b/b4/pr.py @@ -46,6 +46,10 @@ re.compile(r'^\s*([\w+-]+(?:://|@)[\w/.@~-]+)\s*$', re.M | re.I), ] +# PR tracker +PULL_BODY_MERGE_COMMIT_ID_RE = [ + re.compile(r'^https://git.kernel.org/torvalds/c/([0-9a-f]{5,40})$', re.M | re.I), +] def format_addrs(pairs): return ', '.join([utils.formataddr(pair) for pair in pairs]) @@ -85,7 +89,30 @@ def git_get_commit_id_from_repo_ref(repo, ref): return commit_id -def parse_pr_data(msg): +def parse_pr_tracker_data(lmsg, gitdir): + for merge_cid_re in PULL_BODY_MERGE_COMMIT_ID_RE: + matches = merge_cid_re.search(lmsg.body) + if matches: + merge_cid = matches.groups()[0] + break + + if merge_cid is None: + logger.debug('did not find merge commit-id, ignoring pull request') + return None + + gitargs = ['rev-parse', '%s^2' % merge_cid] + ecode, out = b4.git_run_command(gitdir, gitargs) + if ecode > 0: + logger.debug('invalid merge commit: %s', merge_cid) + return None + + lmsg.pr_remote_tip_commit = out.split('\n')[0] + lmsg.pr_merge_commit = merge_cid + logger.debug('success, merge commit-ids: %s <- %s', merge_cid, lmsg.pr_remote_tip_commit) + return lmsg + + +def parse_pr_data(msg, gitdir): lmsg = b4.LoreMessage(msg) if lmsg.body is None: logger.critical('Could not find a plain part in the message body') @@ -93,6 +120,10 @@ def parse_pr_data(msg): logger.info('Looking at: %s', lmsg.full_subject) + config = b4.get_main_config() + if lmsg.fromemail == config['pr-tracker-email']: + return parse_pr_tracker_data(lmsg, gitdir) + for since_re in PULL_BODY_SINCE_ID_RE: matches = since_re.search(lmsg.body) if matches: @@ -258,15 +289,19 @@ def thanks_record_pr(lmsg): def explode(gitdir, lmsg, mailfrom=None, retrieve_links=True, fpopts=None): - ecode = fetch_remote(gitdir, lmsg, check_sig=False, ty_track=False) - if ecode > 0: - raise RuntimeError('Fetching unsuccessful') + if lmsg.pr_repo and lmsg.pr_ref: + ecode = fetch_remote(gitdir, lmsg, check_sig=False, ty_track=False) + if ecode > 0: + raise RuntimeError('Fetching unsuccessful') if not lmsg.pr_base_commit: - # Use git merge-base between HEAD and FETCH_HEAD to find + # Use git merge-base between HEAD and PR tip to find # where we should start logger.info('Running git merge-base to find common ancestry') - gitargs = ['merge-base', 'HEAD', 'FETCH_HEAD'] + head = 'HEAD' + if lmsg.pr_merge_commit: + head = '%s^' % lmsg.pr_merge_commit + gitargs = ['merge-base', head, lmsg.pr_tip_commit] ecode, out = b4.git_run_command(gitdir, gitargs, logstderr=True) if ecode > 0: logger.critical('Could not find common ancestry.') @@ -313,7 +348,7 @@ def explode(gitdir, lmsg, mailfrom=None, retrieve_links=True, fpopts=None): # of the archived threads. linked_ids.add(lmsg.msgid) - with b4.git_format_patches(gitdir, lmsg.pr_base_commit, 'FETCH_HEAD', prefixes=prefixes, extraopts=fpopts) as pdir: + with b4.git_format_patches(gitdir, lmsg.pr_base_commit, lmsg.pr_tip_commit, prefixes=prefixes, extraopts=fpopts) as pdir: if pdir is None: raise RuntimeError('Could not run format-patches') @@ -341,6 +376,9 @@ def explode(gitdir, lmsg, mailfrom=None, retrieve_links=True, fpopts=None): cmsg.add_header('From', mailfrom) cmsg.add_header('Subject', '[' + ' '.join(msubj.prefixes) + '] ' + lmsg.subject) cmsg.add_header('Date', lmsg.msg.get('Date')) + # For PR tracker reply, include a referenece to original PR + if lmsg.fromemail == config['pr-tracker-email'] and lmsg.in_reply_to: + cmsg.add_header('In-Reply-To', '<%s>' % lmsg.in_reply_to) cmsg.set_charset('utf-8') cmsg.replace_header('Content-Transfer-Encoding', '8bit') @@ -505,11 +543,11 @@ def main(cmdargs): gitdir = cmdargs.gitdir lmsg = None - if not sys.stdin.isatty(): + if not cmdargs.msgid and not sys.stdin.isatty(): logger.debug('Getting PR message from stdin') msg = email.message_from_bytes(sys.stdin.buffer.read()) cmdargs.msgid = b4.LoreMessage.get_clean_msgid(msg) - lmsg = parse_pr_data(msg) + lmsg = parse_pr_data(msg, gitdir) else: if cmdargs.msgid and 'github.com' in cmdargs.msgid and '/pull/' in cmdargs.msgid: logger.debug('Getting PR info from Github') @@ -524,7 +562,7 @@ def main(cmdargs): for msg in msgs: mmsgid = b4.LoreMessage.get_clean_msgid(msg) if mmsgid == msgid: - lmsg = parse_pr_data(msg) + lmsg = parse_pr_data(msg, gitdir) break if lmsg is None or lmsg.pr_remote_tip_commit is None: @@ -626,4 +664,5 @@ def main(cmdargs): logger.info('Pull request does not appear to be in this tree.') sys.exit(0) - fetch_remote(gitdir, lmsg, branch=cmdargs.branch) + if lmsg.pr_repo and lmsg.pr_ref: + fetch_remote(gitdir, lmsg, branch=cmdargs.branch) From 5c27ba5bde28f75dde506309089c0ab7f2acfd1e Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 29 Mar 2022 21:48:42 +0300 Subject: [PATCH 03/10] mbox: try harder to find cover letter With add_message(..., needcover=True) try harder to find the cover letter of the message by looking at in-reply-to. Regardless of needcover, always accept cover letter with numbering 0/N but without diffstat, like this one: https://lore.kernel.org/all/20220317053907.164160-1-david@fromorbit.com/ Signed-off-by: Amir Goldstein --- b4/__init__.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/b4/__init__.py b/b4/__init__.py index 5d4fc7c0..5020c005 100644 --- a/b4/__init__.py +++ b/b4/__init__.py @@ -275,7 +275,7 @@ def get_series(self, revision=None, sloppytrailers=False, reroll=True): for member in lser.patches: if member is not None and member.in_reply_to is not None: potential = self.get_by_msgid(member.in_reply_to) - if potential is not None and potential.has_diffstat and not potential.has_diff: + if potential is not None and potential.maybe_cover(): # This is *probably* the cover letter lser.patches[0] = potential lser.has_cover = True @@ -351,7 +351,7 @@ def get_series(self, revision=None, sloppytrailers=False, reroll=True): return lser - def add_message(self, msg): + def add_message(self, msg, needcover=False): msgid = LoreMessage.get_clean_msgid(msg) if msgid in self.msgid_map: logger.debug('Already have a message with this msgid, skipping %s', msgid) @@ -367,7 +367,7 @@ def add_message(self, msg): self.followups.append(lmsg) return - if lmsg.counter == 0 and (not lmsg.counters_inferred or lmsg.has_diffstat): + if lmsg.likely_cover(): # Cover letter # Add it to covers -- we'll deal with them later logger.debug(' adding as v%s cover letter', lmsg.revision) @@ -375,12 +375,12 @@ def add_message(self, msg): return if lmsg.has_diff: - if lmsg.revision not in self.series: + if lmsg.revision not in self.series or needcover: if lmsg.revision_inferred and lmsg.in_reply_to: # We have an inferred revision here. # Do we have an upthread cover letter that specifies a revision? irt = self.get_by_msgid(lmsg.in_reply_to) - if irt is not None and irt.has_diffstat and not irt.has_diff: + if irt is not None and irt.likely_cover(): # Yes, this is very likely our cover letter logger.debug(' fixed revision to v%s', irt.revision) lmsg.revision = irt.revision @@ -968,6 +968,12 @@ def __init__(self, msg): if trailer[0].lower() not in badtrailers: self.trailers.append(trailer) + def maybe_cover(self): + return self.has_diffstat and not self.has_diff + + def likely_cover(self): + return self.counter == 0 and (not self.counters_inferred or self.mayby_cover()) + def get_trailers(self, sloppy=False): trailers = list() mismatches = set() From 1cef0cdcb01f28ac9835f2c526f5d70e3f4246dd Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 30 Mar 2022 10:20:06 +0300 Subject: [PATCH 04/10] mbox: fix calling arguments to get_extra_series() nocache was not passed into get_extra_series() by callers and wantvers was passed in as wrong type. Signed-off-by: Amir Goldstein --- b4/diff.py | 3 ++- b4/mbox.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/b4/diff.py b/b4/diff.py index b21e25f8..524072db 100644 --- a/b4/diff.py +++ b/b4/diff.py @@ -44,7 +44,8 @@ def diff_same_thread_series(cmdargs): if not msgs: logger.critical('Unable to retrieve thread: %s', msgid) return - msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers, useproject=cmdargs.useproject) + msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers, + nocache=cmdargs.nocache, useproject=cmdargs.useproject) if os.path.exists(cachedir): shutil.rmtree(cachedir) pathlib.Path(cachedir).mkdir(parents=True) diff --git a/b4/mbox.py b/b4/mbox.py index 05d46b5f..a857f6ea 100644 --- a/b4/mbox.py +++ b/b4/mbox.py @@ -480,7 +480,7 @@ def save_as_quilt(am_msgs, q_dirname): sfh.write('%s\n' % patch_filename) -def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = None, nocache: bool = False, +def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = None, nocache: bool = False, useproject: Optional[str] = None) -> list: base_msg = None latest_revision = None @@ -725,7 +725,8 @@ def main(cmdargs): return if len(msgs) and cmdargs.checknewer: - msgs = get_extra_series(msgs, direction=1, useproject=cmdargs.useproject) + msgs = get_extra_series(msgs, direction=1, nocache=cmdargs.nocache, + useproject=cmdargs.useproject) if cmdargs.subcmd in ('am', 'shazam'): make_am(msgs, cmdargs, msgid) From 0127e9e3317f254f17b0201aecb35e567a985b9f Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 30 Mar 2022 10:20:47 +0300 Subject: [PATCH 05/10] mbox: query latest revision of patch series by patch subject With get_extra_series([], base_msg=base_msg, direction=0, ...), get the latest revision of series from public-inbox with a match to base_msg subject. This will be used when base_msg refer to a patch in a local mbox where neither the revision nor the date nor author of the local patch is relevant for the search. Signed-off-by: Amir Goldstein --- b4/mbox.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/b4/mbox.py b/b4/mbox.py index a857f6ea..036e706a 100644 --- a/b4/mbox.py +++ b/b4/mbox.py @@ -479,11 +479,15 @@ def save_as_quilt(am_msgs, q_dirname): for patch_filename in patch_filenames: sfh.write('%s\n' % patch_filename) - +# Get older/newer revisions of a patch series from public-inbox +# +# @direction: +# 1 - look for newer revisions (default) +# -1 - look for older revisions +# 0 - look for latest revision in public-inbox (regardless of @base_msg) def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = None, nocache: bool = False, - useproject: Optional[str] = None) -> list: - base_msg = None - latest_revision = None + base_msg = None, useproject: Optional[str] = None) -> list: + latest_revision = 0 seen_msgids = set() seen_covers = set() obsoleted = list() @@ -552,7 +556,7 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = else: # Get subject info from base_msg again lsub = b4.LoreSubject(base_msg['Subject']) - if not len(lsub.prefixes): + if direction > 0 and not len(lsub.prefixes): logger.debug('Not checking for new revisions: no prefixes on the cover letter.') return msgs if direction < 0 and latest_revision <= 1: @@ -569,10 +573,15 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = q = 's:"%s" AND f:"%s" AND d:%s..' % (lsub.subject.replace('"', ''), fromeml, startdate) queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A', 'o': '-1'})) logger.critical('Checking for newer revisions on %s', listarc) - else: + elif direction < 0: q = 's:"%s" AND f:"%s" AND d:..%s' % (lsub.subject.replace('"', ''), fromeml, startdate) queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A', 'o': '1'})) logger.critical('Checking for older revisions on %s', listarc) + else: + # Find latest revision in public-inbox to match base_msg subject + q = 's:"%s"' % (lsub.subject.replace('"', '')) + queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A'})) + logger.debug('Checking for revisions on %s', listarc) logger.debug('Query URL: %s', queryurl) session = b4.get_requests_session() @@ -592,11 +601,11 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = for entry in entries: title = entry.find('atom:title', ns).text lsub = b4.LoreSubject(title) - if lsub.reply or lsub.counter > 1: + if lsub.reply or (direction != 0 and lsub.counter > 1): logger.debug('Ignoring result (not interesting): %s', title) continue link = entry.find('atom:link', ns).get('href') - if direction > 0 and lsub.revision <= latest_revision: + if direction >= 0 and lsub.revision <= latest_revision: logger.debug('Ignoring result (not new revision): %s', title) continue elif direction < 0 and lsub.revision >= latest_revision: @@ -610,13 +619,13 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = continue if lsub.revision == 1 and lsub.revision == latest_revision: # Someone sent a separate message with an identical title but no new vX in the subject line - if direction > 0: + if direction >= 0: # It's *probably* a new revision. logger.debug('Likely a new revision: %s', title) else: # It's *probably* an older revision. logger.debug('Likely an older revision: %s', title) - elif direction > 0 and lsub.revision > latest_revision: + elif direction >= 0 and lsub.revision > latest_revision: logger.debug('Definitely a new revision [v%s]: %s', lsub.revision, title) elif direction < 0 and lsub.revision < latest_revision: logger.debug('Definitely an older revision [v%s]: %s', lsub.revision, title) From fb46481420332a732d2f609912b040b9125c133b Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 30 Mar 2022 10:18:39 +0300 Subject: [PATCH 06/10] mbox: cache get_extra_series() query results When query starts with an empty msgs list, try to use cached results from previous query of the same base_msg. When base_msg is a git am formatted message without a message id use commit id as cache key. Signed-off-by: Amir Goldstein --- b4/__init__.py | 13 ++++++++++++- b4/mbox.py | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/b4/__init__.py b/b4/__init__.py index 5020c005..3398172e 100644 --- a/b4/__init__.py +++ b/b4/__init__.py @@ -1293,6 +1293,17 @@ def get_clean_msgid(msg, header='Message-Id'): msgid = matches.groups()[0] return msgid + # Get commit id from git am formatted patch + @staticmethod + def get_commit_id(msg): + commitid = None + unixhdr = msg.get_unixfrom() + if unixhdr: + matches = re.search(r'^From ([0-9a-f]+)', unixhdr) + if matches: + commitid = matches.groups()[0] + return commitid + @staticmethod def get_preferred_duplicate(msg1, msg2): config = get_main_config() @@ -2055,7 +2066,7 @@ def get_cache_dir(appname: str = 'b4') -> str: fullpath = os.path.join(cachedir, entry) st = os.stat(fullpath) if st.st_mtime < expage: - logger.debug('Cleaning up cache: %s', entry) + logger.debug('Cleaning up cache: %s mtime=%d < %d', entry, st.st_mtime, expage) if os.path.isdir(fullpath): shutil.rmtree(fullpath) else: diff --git a/b4/mbox.py b/b4/mbox.py index 036e706a..26d5f5ea 100644 --- a/b4/mbox.py +++ b/b4/mbox.py @@ -525,6 +525,36 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = logger.debug('Could not find cover of 1st patch in mbox') return msgs + # For query by @base_msg, check if we have a cache of this lookup + base_msgid = b4.LoreMessage.get_clean_msgid(base_msg) + identifier = base_msgid + # Use commit id as key to cache of git am formatted @base_msg + if not identifier: + identifier = b4.LoreMessage.get_commit_id(base_msg) + if identifier is None: + logger.debug('Could not find find base msgid for series') + return msgs + + cachedir = None + if identifier and len(msgs) == 0 and not wantvers: + if useproject: + identifier += '-' + useproject + if direction > 0: + identifier += '+' + elif direction < 0: + identifier += '-' + cachedir = b4.get_cache_file(identifier, suffix='extra.msgs') + + if cachedir and os.path.exists(cachedir) and not nocache: + logger.debug('Using cached copy of %s at %s', identifier, cachedir) + msgs = list() + for msg in os.listdir(cachedir): + with open(os.path.join(cachedir, msg), 'rb') as fh: + msgs.append(email.message_from_binary_file(fh)) + return msgs + else: + logger.debug('No cached copy for %s', identifier) + config = b4.get_main_config() loc = urllib.parse.urlparse(config['midmask']) if not useproject: @@ -565,7 +595,6 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = if direction < 0 and wantvers is None: wantvers = [latest_revision - 1] - base_msgid = b4.LoreMessage.get_clean_msgid(base_msg) fromeml = email.utils.getaddresses(base_msg.get_all('from', []))[0][1] msgdate = email.utils.parsedate_tz(str(base_msg['Date'])) startdate = time.strftime('%Y%m%d', msgdate[:9]) @@ -642,6 +671,13 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = nt_msgs += potentials logger.info(' Added %s messages from that thread', len(potentials)) + # Write results of @base_msg query to cache + if cachedir: + if os.path.exists(cachedir): + shutil.rmtree(cachedir) + pathlib.Path(cachedir).mkdir(parents=True) + at = 0 + # Append all of these to the existing mailbox for nt_msg in nt_msgs: nt_msgid = b4.LoreMessage.get_clean_msgid(nt_msg) @@ -652,6 +688,10 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = logger.debug('Adding: %s', nt_subject) msgs.append(nt_msg) seen_msgids.add(nt_msgid) + if cachedir: + with open(os.path.join(cachedir, '%04d' % at), 'wb') as fh: + fh.write(nt_msg.as_bytes(policy=b4.emlpolicy)) + at += 1 return msgs From f5966362a524182206d5c5e8a4f96fba5d4c92ca Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Fri, 11 Mar 2022 09:29:40 +0200 Subject: [PATCH 07/10] Introduce command "b4 rn" to produce release for pull requests Associate patches in patch queue to patch series and for every patch series provide a lore link to latest revision. Can be used to produce release notes draft to add to a pull request: git format-patch --stdout | b4 rn -m - To produce release notes before merging a pull request: b4 pr -e -o - | b4 rn -m - And to produce release notes from a merged pull request: b4 pr -e -o - | b4 rn -m - See tests/linux_rn.sh for examples. Signed-off-by: Amir Goldstein --- b4/command.py | 14 ++++++ b4/rn.py | 107 ++++++++++++++++++++++++++++++++++++++++ man/b4.5.rst | 23 ++++++++- tests/linux_rn.sh | 86 ++++++++++++++++++++++++++++++++ tests/xfs-5.10..5.17.in | 24 +++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 b4/rn.py create mode 100755 tests/linux_rn.sh create mode 100644 tests/xfs-5.10..5.17.in diff --git a/b4/command.py b/b4/command.py index a06d5ac7..b9e5aed5 100644 --- a/b4/command.py +++ b/b4/command.py @@ -103,6 +103,11 @@ def cmd_diff(cmdargs): b4.diff.main(cmdargs) +def cmd_rn(cmdargs): + import b4.rn + b4.rn.main(cmdargs) + + def cmd(): # noinspection PyTypeChecker parser = argparse.ArgumentParser( @@ -250,6 +255,15 @@ def cmd(): help='Show all developer keys found in a thread') sp_kr.set_defaults(func=cmd_kr) + # b4 rn + sp_rn = subparsers.add_parser('rn', help='Generate release notes from pull request') + cmd_retrieval_common_opts(sp_rn) + sp_rn.add_argument('-g', '--gitdir', default=None, + help='Operate on this git tree instead of current dir') + sp_rn.add_argument('-o', '--output-file', dest='outfile', default=None, + help='Write release notes into this file instead of outputting to stdout') + sp_rn.set_defaults(func=cmd_rn) + cmdargs = parser.parse_args() logger.setLevel(logging.DEBUG) diff --git a/b4/rn.py b/b4/rn.py new file mode 100644 index 00000000..5affda28 --- /dev/null +++ b/b4/rn.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022 CTERA Networks. All Rights Reserved. +# +__author__ = 'Amir Goldstein ' + +import os +import sys +import b4 +import b4.mbox +import mailbox +import email +import shutil +import pathlib + +logger = b4.logger + +def note_series(lser, notes, fh): + cover = None + if lser.has_cover: + cover = lser.patches[0] + elif len(lser.patches) > 1: + cover = lser.patches[1] + + if not cover: + logger.critical('No cover letter found for patch series') + return + + if cover.msgid in notes: + logger.debug('Duplicate series: %s', cover.subject) + return + + notes[cover.msgid] = cover + config = b4.get_main_config() + fh.write('\n- %s\n' % cover.full_subject) + fh.write(' [%s]\n' % (config['linkmask'] % cover.msgid)) + + +def note_latest_series(msgs, notes, fh): + count = len(msgs) + logger.debug('---') + logger.debug('Analyzing %s messages in the thread', count) + lmbx = b4.LoreMailbox() + # Add covers of all revisions first, so we are sure to find the right cover + # when we add the message + for msg in msgs: + lmbx.add_message(msg, needcover=True) + + lser = lmbx.get_series() + if lser is None or len(lser.patches) == 0: + logger.critical('No posted patches found') + return None + + note_series(lser, notes, fh) + + +# Breakup patch queue into series and report notes for every series +def release_notes(msgs, cmdargs, fh): + fh.write('\n---\n') + notes = {} + + for msg in msgs: + # Strip prefixes from subject + lsub = b4.LoreSubject(msg['Subject']) + if not lsub.subject: + continue; + logger.debug('Message: %s', lsub.subject) + if lsub.counter == 0: + # Cover letter of PR + prsub = lsub.subject + if prsub.startswith('Re: '): + prsub = prsub[4:] + fh.write('Changes in %s:\n' % prsub) + prmsgid = b4.LoreMessage.get_clean_msgid(msg, header='In-Reply-To') + if prmsgid: + config = b4.get_main_config() + fh.write(' [%s]\n' % (config['linkmask'] % prmsgid)) + continue; + + # Find public-inbox series whose first patch matches this msg subject + ser_msgs = b4.mbox.get_extra_series([], base_msg=msg, direction=0, + nocache=cmdargs.nocache, + useproject=cmdargs.useproject) + # Report notes for found series + if len(ser_msgs) > 0: + note_latest_series(ser_msgs, notes, fh) + + if not notes: + logger.critical('No posted patches found') + + fh.write('\n---\n') + + +def main(cmdargs): + msgid, msgs = b4.mbox.get_msgs(cmdargs) + if not msgs: + logger.critical('Unable to retrieve messages') + sys.exit(1) + + if cmdargs.outfile is not None: + logger.info('Writing %s', cmdargs.outfile) + fh = open(cmdargs.outfile, 'w') + else: + fh = sys.stdout + + release_notes(msgs, cmdargs, fh) diff --git a/man/b4.5.rst b/man/b4.5.rst index ad7103b9..f6ca7f88 100644 --- a/man/b4.5.rst +++ b/man/b4.5.rst @@ -13,7 +13,7 @@ Work with code submissions in a public-inbox archive SYNOPSIS -------- -b4 {mbox,am,shazam,attest,pr,ty,diff} [options] +b4 {mbox,am,shazam,attest,pr,rn,ty,diff} [options] DESCRIPTION ----------- @@ -31,6 +31,7 @@ SUBCOMMANDS * *b4 am*: Create an mbox file that is ready to git-am * *b4 shazam*: Similar to *am*, but lets you apply patches directly * *b4 pr*: Work with pull requests +* *b4 rn*: Generate release notes * *b4 diff*: Show range-diff style diffs between patch versions * *b4 ty*: Create templated replies for processed patches and pull requests * *b4 attest*: (EXPERIMENTAL) Add cryptographic attestation to patches @@ -249,6 +250,26 @@ optional arguments: *Example*: b4 kr --show-keys 20210521184811.617875-1-konstantin@linuxfoundation.org +b4 rn +~~~~~ +usage: + command.py rn [-h] [-g GITDIR] [-p USEPROJECT] [-C] [-c] [msgid] + +positional arguments: + msgid Message ID to process, or pipe a raw message + +optional arguments: + -h, --help show this help message and exit + -g GITDIR, --gitdir GITDIR + Operate on this git tree instead of current dir + -p USEPROJECT, --use-project USEPROJECT + Use a specific project instead of guessing (linux-mm, linux-hardening, etc) + -o OUTFILE, --output-file OUTFILE + Write release notes into this file instead of outputting to stdout + -C, --no-cache Do not use local cache + +*Example*: b4 rn 202003292120.2BDCB41@keescook + CONFIGURATION ------------- B4 configuration is handled via git-config(1), so you can store it in diff --git a/tests/linux_rn.sh b/tests/linux_rn.sh new file mode 100755 index 00000000..29374cd4 --- /dev/null +++ b/tests/linux_rn.sh @@ -0,0 +1,86 @@ +#!/bin/sh +# +# b4 rn examples to be run from linux source tree +# +# Usage: +# +# T=$PWD/tests/ +# cd ~/src/linux +# $T/linux_rn.sh -d 2>rn.log +# $T/linux_rn.sh -d $T/xfs-5.10..5.17.in -p linux-xfs +# + +if ! (git rev-parse v2.6.12-rc2 2> /dev/null | \ + grep -q 9e734775f7c22d2f89943ad6c745571f1930105f); then + echo "Please run this test from a linux source tree" + exit 1 +fi + +RNMBX=/tmp/linux_rn.mbx +RNOUT=/tmp/linux_rn.out + +if [ "$1" = "-d" ]; then + RNDEBUG=$1 + shift +fi +RNOPTS="$*" + +git_fixes_rn() +{ + local descr="$1" + local range="$2" + local path="$3" + + echo "---" + echo "GIT log - $descr" + echo "---" + git log -p --grep Fixes: --pretty=email $range -- $path | \ + b4 $RNDEBUG rn $RNOPTS -m - +} + +pr_tracker_rn() +{ + local descr="$1" + local msgid="$2" + + rm -rf $RNMBX $RNOUT + + echo "---" + echo "PR tracker - $descr" + echo "---" + b4 $RNDEBUG pr -e -o $RNMBX "$msgid" + b4 $RNDEBUG rn $RNOPTS -m $RNMBX -o $RNOUT + cat $RNOUT +} + + +# Process PR list from input file +if [ -f "$1" ]; then + PRFILE="$1" + shift + RNOPTS="$*" + cat "$PRFILE" | while read pr name; do + echo "Writing release notes of $pr to $name.out..." + >"$name".out 2>"$name".log Date: Sun, 10 Apr 2022 09:48:58 +0300 Subject: [PATCH 08/10] rn: generate release notes in reStructuredText format If output file suffux is .rst, generate release notes in reStructuredText format. Signed-off-by: Amir Goldstein --- b4/rn.py | 36 ++++++++++++++++++++++++------------ tests/linux_rn.sh | 14 +++++++++++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/b4/rn.py b/b4/rn.py index 5affda28..cf0cf385 100644 --- a/b4/rn.py +++ b/b4/rn.py @@ -16,7 +16,7 @@ logger = b4.logger -def note_series(lser, notes, fh): +def note_series(lser, notes, fh, rst): cover = None if lser.has_cover: cover = lser.patches[0] @@ -33,11 +33,14 @@ def note_series(lser, notes, fh): notes[cover.msgid] = cover config = b4.get_main_config() - fh.write('\n- %s\n' % cover.full_subject) - fh.write(' [%s]\n' % (config['linkmask'] % cover.msgid)) + link = (config['linkmask'] % cover.msgid) + if rst: + fh.write('\n- `%s <%s>`_\n' % (cover.full_subject, link)) + else: + fh.write('\n- %s\n [%s]\n' % (cover.full_subject, link)) -def note_latest_series(msgs, notes, fh): +def note_latest_series(msgs, notes, fh, rst): count = len(msgs) logger.debug('---') logger.debug('Analyzing %s messages in the thread', count) @@ -52,12 +55,14 @@ def note_latest_series(msgs, notes, fh): logger.critical('No posted patches found') return None - note_series(lser, notes, fh) + note_series(lser, notes, fh, rst) # Breakup patch queue into series and report notes for every series -def release_notes(msgs, cmdargs, fh): - fh.write('\n---\n') +def release_notes(msgs, cmdargs, fh, rst): + fh.write('\n'); + if not rst: + fh.write('---\n') notes = {} for msg in msgs: @@ -71,11 +76,14 @@ def release_notes(msgs, cmdargs, fh): prsub = lsub.subject if prsub.startswith('Re: '): prsub = prsub[4:] - fh.write('Changes in %s:\n' % prsub) prmsgid = b4.LoreMessage.get_clean_msgid(msg, header='In-Reply-To') if prmsgid: config = b4.get_main_config() - fh.write(' [%s]\n' % (config['linkmask'] % prmsgid)) + link = (config['linkmask'] % prmsgid) + if rst: + fh.write('`%s: <%s>`_\n\n' % (prsub, link)) + else: + fh.write('Changes in %s:\n [%s]\n' % (prsub, link)) continue; # Find public-inbox series whose first patch matches this msg subject @@ -84,12 +92,14 @@ def release_notes(msgs, cmdargs, fh): useproject=cmdargs.useproject) # Report notes for found series if len(ser_msgs) > 0: - note_latest_series(ser_msgs, notes, fh) + note_latest_series(ser_msgs, notes, fh, rst) if not notes: logger.critical('No posted patches found') - fh.write('\n---\n') + fh.write('\n'); + if not rst: + fh.write('---\n') def main(cmdargs): @@ -98,10 +108,12 @@ def main(cmdargs): logger.critical('Unable to retrieve messages') sys.exit(1) + rst = False if cmdargs.outfile is not None: logger.info('Writing %s', cmdargs.outfile) fh = open(cmdargs.outfile, 'w') + rst = cmdargs.outfile.endswith('.rst') else: fh = sys.stdout - release_notes(msgs, cmdargs, fh) + release_notes(msgs, cmdargs, fh, rst) diff --git a/tests/linux_rn.sh b/tests/linux_rn.sh index 29374cd4..c89f55bd 100755 --- a/tests/linux_rn.sh +++ b/tests/linux_rn.sh @@ -17,7 +17,11 @@ if ! (git rev-parse v2.6.12-rc2 2> /dev/null | \ fi RNMBX=/tmp/linux_rn.mbx -RNOUT=/tmp/linux_rn.out +# Default to release notes in reStructuredText format +RNFMT=rst +# Uncomment to generate release notes in plain text format +#RNFMT=txt +RNOUT=/tmp/linux_rn.$RNFMT if [ "$1" = "-d" ]; then RNDEBUG=$1 @@ -59,10 +63,14 @@ if [ -f "$1" ]; then PRFILE="$1" shift RNOPTS="$*" + + rnoutall=`basename ${PRFILE%.*}`-rn.$RNFMT + echo "# Release notes auto-generated by 'b4 rn'" > $rnoutall cat "$PRFILE" | while read pr name; do - echo "Writing release notes of $pr to $name.out..." - >"$name".out 2>"$name".log "$name".$RNFMT 2>"$name".log > $rnoutall done exit fi From 3ea3360c4a05649c16d250cb3b03891665d24d52 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 10 Apr 2022 09:50:25 +0300 Subject: [PATCH 09/10] rn: print references to fstests in release notes If any of the messages in the series has references a fstests test name, print the tests in release notes. Signed-off-by: Amir Goldstein --- b4/rn.py | 22 ++++++++++++++++++++-- tests/linux_rn.sh | 3 +++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/b4/rn.py b/b4/rn.py index cf0cf385..a43d62de 100644 --- a/b4/rn.py +++ b/b4/rn.py @@ -13,10 +13,15 @@ import email import shutil import pathlib +import re logger = b4.logger -def note_series(lser, notes, fh, rst): +MSG_BODY_TEST_REF_RE = [ + re.compile(r'\b(btrfs|ceph|cifs|ext4|f2fs|generic|nfs|ocfs2|overlay|perf|shared|udf|xfs)/([0-9]{3})\b'), +] + +def note_series(lser, notes, tests, fh, rst): cover = None if lser.has_cover: cover = lser.patches[0] @@ -38,6 +43,8 @@ def note_series(lser, notes, fh, rst): fh.write('\n- `%s <%s>`_\n' % (cover.full_subject, link)) else: fh.write('\n- %s\n [%s]\n' % (cover.full_subject, link)) + if tests: + fh.write(' Tests: %s\n' % ' '.join(sorted(tests))) def note_latest_series(msgs, notes, fh, rst): @@ -45,17 +52,28 @@ def note_latest_series(msgs, notes, fh, rst): logger.debug('---') logger.debug('Analyzing %s messages in the thread', count) lmbx = b4.LoreMailbox() + tests = set() # Add covers of all revisions first, so we are sure to find the right cover # when we add the message for msg in msgs: + lmsg = b4.LoreMessage(msg) + if lmsg.body is None: + logger.critical('Could not find a plain part in the message body') + continue lmbx.add_message(msg, needcover=True) + for tests_re in MSG_BODY_TEST_REF_RE: + for match in re.finditer(tests_re, lmsg.body): + test = match.group(0) + if not test in tests: + logger.debug('Found reference to test: %s\n' % test) + tests.add(test) lser = lmbx.get_series() if lser is None or len(lser.patches) == 0: logger.critical('No posted patches found') return None - note_series(lser, notes, fh, rst) + note_series(lser, notes, tests, fh, rst) # Breakup patch queue into series and report notes for every series diff --git a/tests/linux_rn.sh b/tests/linux_rn.sh index c89f55bd..77863a53 100755 --- a/tests/linux_rn.sh +++ b/tests/linux_rn.sh @@ -87,6 +87,9 @@ pr_tracker_rn "individual patch posted" \ pr_tracker_rn "patch series posted" \ 164408216661.7836.4930013315804213982.pr-tracker-bot@kernel.org +pr_tracker_rn "series with fstest reference" \ + 162784611031.1186.18214929758593020802.pr-tracker-bot@kernel.org + pr_tracker_rn "patch series including partial reroll" \ 164817214223.9489.12483808836905609419.pr-tracker-bot@kernel.org From 369aadef5ac567643d91aa346a3061cc82adba17 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 26 Apr 2022 10:20:19 +0300 Subject: [PATCH 10/10] rn: add release notes for patches not posted When a patch is found in PR whose subject is not found in public inbox, list the patch is release notes as: - [PATCH ?/?] ... Because we do not know if it is part of a patch series and we have no link to public inbox. Signed-off-by: Amir Goldstein --- b4/rn.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/b4/rn.py b/b4/rn.py index a43d62de..c9958eb7 100644 --- a/b4/rn.py +++ b/b4/rn.py @@ -30,11 +30,11 @@ def note_series(lser, notes, tests, fh, rst): if not cover: logger.critical('No cover letter found for patch series') - return + return False if cover.msgid in notes: logger.debug('Duplicate series: %s', cover.subject) - return + return True notes[cover.msgid] = cover config = b4.get_main_config() @@ -45,7 +45,7 @@ def note_series(lser, notes, tests, fh, rst): fh.write('\n- %s\n [%s]\n' % (cover.full_subject, link)) if tests: fh.write(' Tests: %s\n' % ' '.join(sorted(tests))) - + return True def note_latest_series(msgs, notes, fh, rst): count = len(msgs) @@ -71,9 +71,9 @@ def note_latest_series(msgs, notes, fh, rst): lser = lmbx.get_series() if lser is None or len(lser.patches) == 0: logger.critical('No posted patches found') - return None + return False - note_series(lser, notes, tests, fh, rst) + return note_series(lser, notes, tests, fh, rst) # Breakup patch queue into series and report notes for every series @@ -109,8 +109,11 @@ def release_notes(msgs, cmdargs, fh, rst): nocache=cmdargs.nocache, useproject=cmdargs.useproject) # Report notes for found series + found = False if len(ser_msgs) > 0: - note_latest_series(ser_msgs, notes, fh, rst) + found = note_latest_series(ser_msgs, notes, fh, rst) + if not found: + fh.write('\n- [PATH ?/?] %s\n' % lsub.subject) if not notes: logger.critical('No posted patches found')