Initial patchwork integration support

A lot of maintainers use patchwork alongside b4, to make it easier to
track patches and rely on some CI integration. This commit adds some
basic patchwork integration:

- on "b4 am", "b4 shazam", "b4 pr" we will mark the relevant patchwork
  entries as "Under Review"
- on "b4 ty" we can set these patches as "Accepted"
- on "b4 ty -d" we can set them as "Deferred"

To make it work, the following entries must be present in the repository
used with b4:

    [b4]
      pw-key = (your API token)
      pw-url = https://patchwork.kernel.org
      pw-project = (your project, e.g. linux-usb)
      pw-review-state = under-review
      pw-accept-state = accepted
      pw-discard-state = deferred

To get your patchwork API token, go to your patchwork profile page.

The pw-accept-state and pw-discard-state can be overridden using the
--pw-set-state flag to "b4 ty". E.g. if you wanted to mark the patches
as "Not applicable":

    b4 ty -d 5 --pw-set-state not-applicable

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

5 files changed, 124 insertions, 6 deletions

diff --git a/b4/__init__.py b/b4/__init__.py
index ebd4f2e..cfbe6a6 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -14,8 +14,8 @@ import email.header
 import email.generator
 import tempfile
 import pathlib
+import argparse
 
-import requests
 import urllib.parse
 import datetime
 import time
@@ -25,6 +25,8 @@ import mailbox
 # noinspection PyCompatibility
 import pwd
 
+import requests
+
 from contextlib import contextmanager
 from typing import Optional, Tuple, Set, List, TextIO
 
@@ -44,7 +46,8 @@ try:
 except ModuleNotFoundError:
     can_patatt = False
 
-__VERSION__ = '0.9.0'
+__VERSION__ = '0.10.0-dev'
+PW_REST_API_VERSION = '1.2'
 
 
 def _dkim_log_filter(record):
@@ -1867,7 +1870,7 @@ class LoreAttestorPatatt(LoreAttestor):
             self.have_key = True
 
 
-def _run_command(cmdargs: list, stdin: Optional[bytes] = None) -> Tuple[int, bytes, bytes]:
+def _run_command(cmdargs: List[str], stdin: Optional[bytes] = None) -> Tuple[int, bytes, bytes]:
     logger.debug('Running %s' % ' '.join(cmdargs))
     sp = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
     (output, error) = sp.communicate(input=stdin)
@@ -2113,7 +2116,7 @@ def get_requests_session():
     return REQSESSION
 
 
-def get_msgid_from_stdin():
+def get_msgid_from_stdin() -> Optional[str]:
     if not sys.stdin.isatty():
         from email.parser import BytesParser
         message = BytesParser().parsebytes(
@@ -2122,7 +2125,7 @@ def get_msgid_from_stdin():
     return None
 
 
-def get_msgid(cmdargs) -> Optional[str]:
+def get_msgid(cmdargs: argparse.Namespace) -> Optional[str]:
     if not cmdargs.msgid:
         logger.debug('Getting Message-ID from stdin')
         msgid = get_msgid_from_stdin()
@@ -2134,6 +2137,15 @@ def get_msgid(cmdargs) -> Optional[str]:
 
     msgid = msgid.strip('<>')
     # Handle the case when someone pastes a full URL to the message
+    # Is this a patchwork URL?
+    matches = re.search(r'^https?://.*/project/.*/patch/([^/]+@[^/]+)', msgid, re.IGNORECASE)
+    if matches:
+        logger.debug('Looks like a patchwork URL')
+        chunks = matches.groups()
+        msgid = urllib.parse.unquote(chunks[0])
+        return msgid
+
+    # Is it a lore URL?
     matches = re.search(r'^https?://[^/]+/([^/]+)/([^/]+@[^/]+)', msgid, re.IGNORECASE)
     if matches:
         chunks = matches.groups()
@@ -2588,3 +2600,67 @@ def get_smtp(identity: Optional[str] = None):
         smtp = smtplib.SMTP(server, port)
 
     return smtp, fromaddr
+
+
+def get_patchwork_session(pwkey: str, pwurl: str) -> Tuple[requests.Session, str]:
+    session = requests.session()
+    session.headers.update({
+        'User-Agent': 'b4/%s' % __VERSION__,
+        'Authorization': f'Token {pwkey}',
+    })
+    url = '/'.join((pwurl.rstrip('/'), 'api', PW_REST_API_VERSION))
+    logger.debug('pw url=%s', url)
+    return session, url
+
+
+def patchwork_set_state(msgids: List[str], state: str) -> bool:
+    # Do we have a pw-key defined in config?
+    config = get_main_config()
+    pwkey = config.get('pw-key')
+    pwurl = config.get('pw-url')
+    pwproj = config.get('pw-project')
+    if not (pwkey and pwurl and pwproj):
+        logger.debug('Patchwork support not configured')
+        return False
+    pses, url = get_patchwork_session(pwkey, pwurl)
+    patches_url = '/'.join((url, 'patches'))
+    tochange = list()
+    for msgid in msgids:
+        # Two calls, first to look up the patch-id, second to update its state
+        params = [
+            ('project', pwproj),
+            ('archived', 'false'),
+            ('msgid', msgid),
+        ]
+        try:
+            logger.debug('looking up patch_id of msgid=%s', msgid)
+            rsp = pses.get(patches_url, params=params, stream=False)
+            rsp.raise_for_status()
+            pdata = rsp.json()
+            for entry in pdata:
+                patch_id = entry.get('id')
+                if patch_id:
+                    title = entry.get('name')
+                    if entry.get('state') != state:
+                        tochange.append((patch_id, title))
+        except requests.exceptions.RequestException as ex:
+            logger.debug('Patchwork REST error: %s', ex)
+
+    if tochange:
+        logger.info('---')
+        loc = urllib.parse.urlparse(pwurl)
+        logger.info('Patchwork: setting state on %s/%s', loc.netloc, pwproj)
+        for patch_id, title in tochange:
+            patchid_url = '/'.join((patches_url, str(patch_id), ''))
+            logger.debug('patchid_url=%s', patchid_url)
+            data = [
+                ('state', state),
+            ]
+            try:
+                rsp = pses.patch(patchid_url, data=data, stream=False)
+                rsp.raise_for_status()
+                newdata = rsp.json()
+                if newdata.get('state') == state:
+                    logger.info(' -> %s : %s', state, title)
+            except requests.exceptions.RequestException as ex:
+                logger.debug('Patchwork REST error: %s', ex)
diff --git a/b4/command.py b/b4/command.py
index 92ffd65..d746efb 100644
--- a/b4/command.py
+++ b/b4/command.py
@@ -224,6 +224,8 @@ def cmd():
                        help='Send email instead of writing out .thanks files')
     sp_ty.add_argument('--dry-run', action='store_true', dest='dryrun', default=False,
                        help='Print out emails instead of sending them')
+    sp_ty.add_argument('--pw-set-state', default=None,
+                       help='Set this patchwork state instead of default (use with -a, -t or -d)')
     sp_ty.set_defaults(func=cmd_ty)
 
     # b4 diff
diff --git a/b4/mbox.py b/b4/mbox.py
index 1cc58c7..b9578ef 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -20,6 +20,7 @@ import pathlib
 import tempfile
 import io
 import shlex
+import argparse
 
 import urllib.parse
 import xml.etree.ElementTree
@@ -417,6 +418,7 @@ def thanks_record_am(lser, cherrypick=None):
     filename = '%s.am' % slug
 
     patches = list()
+    msgids = list()
     at = 0
     padlen = len(str(lser.expected))
     lmsg = None
@@ -425,6 +427,7 @@ def thanks_record_am(lser, cherrypick=None):
         if pmsg is None:
             at += 1
             continue
+        msgids.append(pmsg.msgid)
 
         if lmsg is None:
             lmsg = pmsg
@@ -472,6 +475,11 @@ def thanks_record_am(lser, cherrypick=None):
         json.dump(out, fh, ensure_ascii=False, indent=4)
         logger.debug('Wrote %s for thanks tracking', filename)
 
+    config = b4.get_main_config()
+    pwstate = config.get('pw-review-state')
+    if pwstate:
+        b4.patchwork_set_state(msgids, pwstate)
+
 
 def save_as_quilt(am_msgs, q_dirname):
     if os.path.exists(q_dirname):
@@ -670,7 +678,7 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
     return msgs
 
 
-def get_msgs(cmdargs) -> Tuple[Optional[str], Optional[list]]:
+def get_msgs(cmdargs: argparse.Namespace) -> Tuple[Optional[str], Optional[list]]:
     msgid = None
     if not cmdargs.localmbox:
         msgid = b4.get_msgid(cmdargs)
diff --git a/b4/pr.py b/b4/pr.py
index 6918a6d..825391e 100644
--- a/b4/pr.py
+++ b/b4/pr.py
@@ -256,6 +256,11 @@ def thanks_record_pr(lmsg):
         json.dump(out, fh, ensure_ascii=False, indent=4)
         logger.debug('Wrote %s for thanks tracking', filename)
 
+    config = b4.get_main_config()
+    pwstate = config.get('pw-review-state')
+    if pwstate:
+        b4.patchwork_set_state(lmsg.msgid, pwstate)
+
 
 def explode(gitdir, lmsg, mailfrom=None, retrieve_links=True, fpopts=None):
     ecode = fetch_remote(gitdir, lmsg, check_sig=False, ty_track=False)
diff --git a/b4/ty.py b/b4/ty.py
index dddeca8..95477ff 100644
--- a/b4/ty.py
+++ b/b4/ty.py
@@ -454,6 +454,7 @@ def send_messages(listing, branch, cmdargs):
         signature = '%s <%s>' % (usercfg['name'], usercfg['email'])
 
     outgoing = 0
+    msgids = list()
     for jsondata in listing:
         jsondata['myname'] = usercfg['name']
         jsondata['myemail'] = usercfg['email']
@@ -468,6 +469,10 @@ def send_messages(listing, branch, cmdargs):
         if msg is None:
             continue
 
+        msgids.append(jsondata['msgid'])
+        for pdata in jsondata.get('patches', list()):
+            msgids.append(pdata[2])
+
         outgoing += 1
         msg.set_charset('utf-8')
         msg.replace_header('Content-Transfer-Encoding', '8bit')
@@ -516,13 +521,24 @@ def send_messages(listing, branch, cmdargs):
         logger.info('No thanks necessary.')
         return
 
+    config = b4.get_main_config()
+    pwstate = cmdargs.pw_set_state
+    if not pwstate:
+        pwstate = config.get('pw-accept-state')
+
     if cmdargs.sendemail:
         if cmdargs.dryrun:
             logger.info('DRYRUN: generated %s thank-you letters', outgoing)
         else:
             logger.info('Sent %s thank-you letters', outgoing)
+            if pwstate:
+                b4.patchwork_set_state(msgids, pwstate)
         smtp.quit()
     else:
+        if pwstate:
+            print(msgids, pwstate)
+            b4.patchwork_set_state(msgids, pwstate)
+            logger.info('---')
         logger.debug('Wrote %s thank-you letters', outgoing)
         logger.info('You can now run:')
         logger.info('  git send-email %s/*.thanks', cmdargs.outdir)
@@ -621,10 +637,21 @@ def discard_selected(cmdargs):
 
     datadir = b4.get_data_dir()
     logger.info('Discarding %s messages', len(listing))
+    msgids = list()
     for jsondata in listing:
         fullpath = os.path.join(datadir, jsondata['trackfile'])
         os.rename(fullpath, '%s.discarded' % fullpath)
         logger.info('  Discarded: %s', jsondata['subject'])
+        msgids.append(jsondata['msgid'])
+        for pdata in jsondata.get('patches', list()):
+            msgids.append(pdata[2])
+
+    config = b4.get_main_config()
+    pwstate = cmdargs.pw_set_state
+    if not pwstate:
+        pwstate = config.get('pw-discard-state')
+    if pwstate:
+        b4.patchwork_set_state(msgids, pwstate)
 
     sys.exit(0)