Don't depend on List-Archive lore header

The newer version of public-inbox is not injecting its own List-Archive
headers, so stop relying on it for any purpose.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

3 files changed, 67 insertions, 53 deletions

diff --git a/b4/__init__.py b/b4/__init__.py
index f13a599..c39a905 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -179,56 +179,20 @@ class LoreMailbox:
                     break
         logger.info('---')
         logger.info('Thread incomplete, attempting to backfill')
-        cachedir = get_cache_dir()
-        listmap = os.path.join(cachedir, 'lists.map.lookup')
-        if not os.path.exists(listmap):
-            # lists.map is a custom service running on lore.kernel.org, so it is
-            # meaningless to make this a configurable URL
-            session = get_requests_session()
-            resp = session.get('https://lore.kernel.org/lists.map')
-            if resp.status_code != 200:
-                logger.debug('Unable to retrieve lore.kernel.org/lists.map')
-                return
-            content = resp.content.decode('utf-8')
-            with open(listmap, 'w') as fh:
-                fh.write(content)
-        else:
-            with open(listmap, 'r') as fh:
-                content = fh.read()
-
-        projmap = dict()
-        for line in content.split('\n'):
-            if line.find(':') <= 0:
+        for project in get_lore_projects_from_msg(patch.msg):
+            projurl = 'https://lore.kernel.org/%s/' % project
+            # Try to backfill from that project
+            backfills = get_pi_thread_by_msgid(patch.msgid, useproject=project)
+            if not backfills:
                 continue
-            chunks = line.split(':')
-            projmap[chunks[0]] = chunks[1].strip()
-
-        allto = email.utils.getaddresses([str(x) for x in patch.msg.get_all('to', [])])
-        allto += email.utils.getaddresses([str(x) for x in patch.msg.get_all('cc', [])])
-        listarc = patch.msg.get_all('list-archive', [])
-        for entry in allto:
-            if entry[1] in projmap:
-                projurl = 'https://lore.kernel.org/%s/' % projmap[entry[1]]
-                # Make sure we don't re-query the same project we just used
-                reused = False
-                for arcurl in listarc:
-                    if projurl in arcurl:
-                        reused = True
-                        break
-                if reused:
-                    continue
-                # Try to backfill from that project
-                backfills = get_pi_thread_by_msgid(patch.msgid, useproject=projmap[entry[1]])
-                if not backfills:
-                    return
-                was = len(self.msgid_map)
-                for msg in backfills:
-                    self.add_message(msg)
-                if len(self.msgid_map) > was:
-                    logger.info('Loaded %s messages from %s', len(self.msgid_map)-was, projurl)
-                if self.series[revision].complete:
-                    logger.info('Successfully backfilled missing patches')
-                    break
+            was = len(self.msgid_map)
+            for msg in backfills:
+                self.add_message(msg)
+            if len(self.msgid_map) > was:
+                logger.info('Loaded %s messages from %s', len(self.msgid_map)-was, projurl)
+            if self.series[revision].complete:
+                logger.info('Successfully backfilled missing patches')
+                break
 
     def partial_reroll(self, revision, sloppytrailers, backfill):
         # Is it a partial reroll?
@@ -2301,3 +2265,38 @@ def save_git_am_mbox(msgs: list, dest):
         bmsg = bmsg.replace(b'From mboxrd@z ', b'From git@z ')
         bmsg = bmsg.rstrip(b'\r\n') + b'\n\n'
         dest.write(bmsg)
+
+
+def get_lore_projects_from_msg(msg) -> list:
+    cachedir = get_cache_dir()
+    listmap = os.path.join(cachedir, 'lists.map.lookup')
+    if not os.path.exists(listmap):
+        # lists.map is a custom service running on lore.kernel.org, so it is
+        # meaningless to make this a configurable URL
+        session = get_requests_session()
+        resp = session.get('https://lore.kernel.org/lists.map')
+        if resp.status_code != 200:
+            logger.debug('Unable to retrieve lore.kernel.org/lists.map')
+            return list()
+        content = resp.content.decode()
+        with open(listmap, 'w') as fh:
+            fh.write(content)
+    else:
+        with open(listmap, 'r') as fh:
+            content = fh.read()
+
+    projmap = dict()
+    for line in content.split('\n'):
+        if line.find(':') <= 0:
+            continue
+        chunks = line.split(':')
+        projmap[chunks[0]] = chunks[1].strip()
+
+    allto = email.utils.getaddresses([str(x) for x in msg.get_all('to', [])])
+    allto += email.utils.getaddresses([str(x) for x in msg.get_all('cc', [])])
+    projects = list()
+    for entry in allto:
+        if entry[1] in projmap:
+            projects.append(projmap[entry[1]])
+
+    return projects
diff --git a/b4/diff.py b/b4/diff.py
index abd5fec..299ebf2 100644
--- a/b4/diff.py
+++ b/b4/diff.py
@@ -44,7 +44,7 @@ def diff_same_thread_series(cmdargs):
         if not msgs:
             logger.critical('Unable to retrieve thread: %s', msgid)
             return
-        msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers)
+        msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers, useproject=cmdargs.useproject)
         if os.path.exists(cachedir):
             shutil.rmtree(cachedir)
         pathlib.Path(cachedir).mkdir(parents=True)
diff --git a/b4/mbox.py b/b4/mbox.py
index bf1a586..f575f94 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -387,7 +387,8 @@ def save_as_quilt(am_msgs, q_dirname):
             sfh.write('%s\n' % patch_filename)
 
 
-def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = None, nocache: bool = False) -> list:
+def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = None, nocache: bool = False,
+                     useproject: Optional[str] = None) -> list:
     base_msg = None
     latest_revision = None
     seen_msgids = set()
@@ -427,7 +428,21 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
         logger.debug('Could not find cover of 1st patch in mbox')
         return msgs
 
-    listarc = base_msg.get_all('List-Archive')[-1].strip('<>')
+    config = b4.get_main_config()
+    loc = urllib.parse.urlparse(config['midmask'])
+    if not useproject:
+        projects = b4.get_lore_projects_from_msg(base_msg)
+        if not projects:
+            logger.info('Unable to figure out list archive location')
+            return msgs
+        useproject = projects[0]
+
+    listarc = '%s://%s/%s/' % (loc.scheme, loc.netloc, useproject)
+
+    if not listarc:
+        logger.info('Unable to figure out list archive location')
+        return msgs
+
     nt_msgs = list()
     if len(obsoleted):
         for nt_msgid in obsoleted:
@@ -601,7 +616,7 @@ def main(cmdargs):
         return
 
     if len(msgs) and cmdargs.checknewer:
-        msgs = get_extra_series(msgs, direction=1)
+        msgs = get_extra_series(msgs, direction=1, useproject=cmdargs.useproject)
 
     if cmdargs.subcmd == 'am':
         make_am(msgs, cmdargs, msgid)