Display range-diff by default

Don't make developers do copy-pasting unnecessarily. Switch to
outputting the diff by default, with flag options to save to file or
just show what needs to be done.

Additionally, adds caching to lookups and remember previously generated
fake-am ranges so we don't continuously generate loose objects on repeat
runs.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

4 files changed, 105 insertions, 24 deletions

diff --git a/b4/__init__.py b/b4/__init__.py
index 064f8b8..414f669 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -1844,7 +1844,13 @@ def save_strict_thread(in_mbx, out_mbx, msgid):
         logger.info('Reduced thread to strict matches only (%s->%s)', len(in_mbx), len(out_mbx))
 
 
-def get_pi_thread_by_url(t_mbx_url, savefile):
+def get_pi_thread_by_url(t_mbx_url, savefile, nocache=False):
+    cachedir = get_cache_dir()
+    cachefile = os.path.join(cachedir, '%s.pi.mbx' % urllib.parse.quote_plus(t_mbx_url))
+    if os.path.exists(cachefile) and not nocache:
+        logger.debug('Using cached copy: %s', cachefile)
+        shutil.copyfile(cachefile, savefile)
+        return savefile
     session = get_requests_session()
     resp = session.get(t_mbx_url)
     if resp.status_code != 200:
@@ -1858,22 +1864,13 @@ def get_pi_thread_by_url(t_mbx_url, savefile):
     with open(savefile, 'wb') as fh:
         logger.debug('Saving %s', savefile)
         fh.write(t_mbox)
+    shutil.copyfile(savefile, cachefile)
     return savefile
 
 
 def get_pi_thread_by_msgid(msgid, savefile, useproject=None, nocache=False):
     qmsgid = urllib.parse.quote_plus(msgid)
     config = get_main_config()
-    cachedir = get_cache_dir()
-    base = msgid
-    if useproject:
-        base = '%s-%s' % (useproject, msgid)
-    cachefile = os.path.join(cachedir, '%s.pi.mbx' % urllib.parse.quote_plus(base))
-    if os.path.exists(cachefile) and not nocache:
-        logger.debug('Using cached copy: %s', cachefile)
-        shutil.copyfile(cachefile, savefile)
-        return savefile
-
     # Grab the head from lore, to see where we are redirected
     midmask = config['midmask'] % qmsgid
     loc = urllib.parse.urlparse(midmask)
@@ -1894,7 +1891,7 @@ def get_pi_thread_by_msgid(msgid, savefile, useproject=None, nocache=False):
     logger.debug('t_mbx_url=%s', t_mbx_url)
 
     logger.critical('Grabbing thread from %s', projurl.split('://')[1])
-    in_mbxf = get_pi_thread_by_url(t_mbx_url, '%s-loose' % savefile)
+    in_mbxf = get_pi_thread_by_url(t_mbx_url, '%s-loose' % savefile, nocache=nocache)
     if not in_mbxf:
         return None
     in_mbx = mailbox.mbox(in_mbxf)
@@ -1903,7 +1900,6 @@ def get_pi_thread_by_msgid(msgid, savefile, useproject=None, nocache=False):
     in_mbx.close()
     out_mbx.close()
     os.unlink(in_mbxf)
-    shutil.copyfile(savefile, cachefile)
     return savefile
 
 
diff --git a/b4/command.py b/b4/command.py
index 12900ff..80b072a 100644
--- a/b4/command.py
+++ b/b4/command.py
@@ -181,6 +181,12 @@ def cmd():
                          help='Do not use local cache')
     sp_diff.add_argument('-v', '--compare-versions', dest='wantvers', type=int, default=None, nargs='+',
                          help='Compare specific versions instead of latest and one before that, e.g. -v 3 5')
+    sp_diff.add_argument('-n', '--no-diff', dest='nodiff', action='store_true', default=False,
+                         help='Do not generate a diff, just show the command to do it')
+    sp_diff.add_argument('-o', '--output-diff', dest='outdiff', default=None,
+                         help='Save diff into this file instead of outputting to stdout')
+    sp_diff.add_argument('-c', '--color', dest='color', action='store_true', default=False,
+                         help='Force color output even when writing to file')
     sp_diff.set_defaults(func=cmd_diff)
 
     cmdargs = parser.parse_args()
diff --git a/b4/diff.py b/b4/diff.py
index 0ce2283..36bdfd6 100644
--- a/b4/diff.py
+++ b/b4/diff.py
@@ -10,6 +10,9 @@ import sys
 import b4
 import b4.mbox
 import mailbox
+import shutil
+import urllib.parse
+
 from tempfile import mkstemp
 
 
@@ -17,6 +20,45 @@ logger = b4.logger
 
 
 def make_fake_commit_range(gitdir, lser):
+    start_commit = end_commit = None
+    # Do we have it in cache already?
+    cachedir = b4.get_cache_dir()
+    # Use the msgid of the first non-None patch in the series
+    msgid = None
+    for lmsg in lser.patches:
+        if lmsg is not None:
+            msgid = lmsg.msgid
+            break
+    if msgid is None:
+        logger.critical('Cannot operate on an empty series')
+        return None, None
+    cachefile = os.path.join(cachedir, '%s.fakeam' % urllib.parse.quote_plus(msgid))
+    if os.path.exists(cachefile):
+        stalecache = False
+        with open(cachefile, 'r') as fh:
+            cachedata = fh.read()
+            chunks = cachedata.strip().split()
+            if len(chunks) == 2:
+                start_commit, end_commit = chunks
+            else:
+                stalecache = True
+        if start_commit is not None and end_commit is not None:
+            # Make sure they are still there
+            ecode, out = b4.git_run_command(gitdir, ['cat-file', '-e', start_commit])
+            if ecode > 0:
+                stalecache = True
+            else:
+                ecode, out = b4.git_run_command(gitdir, ['cat-file', '-e', end_commit])
+                if ecode > 0:
+                    stalecache = True
+                else:
+                    logger.debug('Using previously generated range')
+                    return start_commit, end_commit
+
+        if stalecache:
+            logger.debug('Stale cache for [v%s] %s', lser.revision, lser.subject)
+            os.unlink(cachefile)
+
     logger.info('Preparing fake-am for v%s: %s', lser.revision, lser.subject)
     with b4.git_temp_worktree(gitdir):
         # We are in a temporary chdir at this time, so writing to a known file should be safe
@@ -73,6 +115,11 @@ def make_fake_commit_range(gitdir, lser):
         end_commit = out.strip()
         logger.info('  range: %.12s..%.12s', start_commit, end_commit)
 
+    with open(cachefile, 'w') as fh:
+        logger.debug('Saving into cache: %s', cachefile)
+        logger.debug('    %s..%s', start_commit, end_commit)
+        fh.write(f'{start_commit} {end_commit}\n')
+
     return start_commit, end_commit
 
 
@@ -84,12 +131,26 @@ def main(cmdargs):
 
     # start by grabbing the mbox provided
     savefile = mkstemp('b4-diff-to')[1]
-    mboxfile = b4.get_pi_thread_by_msgid(msgid, savefile, useproject=cmdargs.useproject, nocache=cmdargs.nocache)
-    if mboxfile is None:
-        logger.critical('Unable to retrieve thread: %s', msgid)
-        return
-    logger.info('Retrieved %s messages in the thread', len(mboxfile))
-    b4.mbox.get_extra_series(mboxfile, direction=-1, wantvers=cmdargs.wantvers)
+    # Do we have a cache of this lookup?
+    cachedir = b4.get_cache_dir()
+    if cmdargs.wantvers:
+        cachefile = os.path.join(cachedir,
+                                 '%s-%s.diff.mbx' % (urllib.parse.quote_plus(msgid), '-'.join(cmdargs.wantvers)))
+    else:
+        cachefile = os.path.join(cachedir, '%s-latest.diff.mbx' % urllib.parse.quote_plus(msgid))
+    if os.path.exists(cachefile) and not cmdargs.nocache:
+        logger.info('Using cached copy of the lookup')
+        shutil.copyfile(cachefile, savefile)
+        mboxfile = savefile
+    else:
+        mboxfile = b4.get_pi_thread_by_msgid(msgid, savefile, useproject=cmdargs.useproject, nocache=cmdargs.nocache)
+        if mboxfile is None:
+            logger.critical('Unable to retrieve thread: %s', msgid)
+            return
+        logger.info('Retrieved %s messages in the thread', len(mboxfile))
+        b4.mbox.get_extra_series(mboxfile, direction=-1, wantvers=cmdargs.wantvers)
+
+    shutil.copyfile(mboxfile, cachefile)
     mbx = mailbox.mbox(mboxfile)
     count = len(mbx)
     logger.info('---')
@@ -136,6 +197,24 @@ def main(cmdargs):
         os.unlink(mboxfile)
         sys.exit(1)
     logger.info('---')
-    logger.info('Success, you may now run:')
-    logger.info('    git range-diff %.12s..%.12s %.12s..%.12s', lsc, lec, usc, uec)
-
+    grdcmd = 'git range-diff %.12s..%.12s %.12s..%.12s' % (lsc, lec, usc, uec)
+    if cmdargs.nodiff:
+        logger.info('Success, to compare v%s and v%s:', lower, upper)
+        logger.info(f'    {grdcmd}')
+        sys.exit(0)
+    logger.info('Running: %s', grdcmd)
+    gitargs = ['range-diff', f'{lsc}..{lec}', f'{usc}..{uec}']
+    if cmdargs.outdiff is None or cmdargs.color:
+        gitargs.append('--color')
+    ecode, rdiff = b4.git_run_command(cmdargs.gitdir, gitargs)
+    if ecode > 0:
+        logger.critical('Unable to generate diff')
+        logger.critical('Try running it yourself:')
+        logger.critical(f'    {grdcmd}')
+        sys.exit(1)
+    if cmdargs.outdiff is not None:
+        logger.info('Writing %s', cmdargs.outdiff)
+        fh = open(cmdargs.outdiff, 'w')
+    else:
+        fh = sys.stdout
+    fh.write(rdiff)
diff --git a/b4/mbox.py b/b4/mbox.py
index 9ff88a6..e03ef6c 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -310,7 +310,7 @@ def am_mbox_to_quilt(am_mbx, q_dirname):
             sfh.write('%s\n' % patch_filename)
 
 
-def get_extra_series(mboxfile, direction=1, wantvers=None):
+def get_extra_series(mboxfile, direction=1, wantvers=None, nocache=False):
     # Open the mbox and find the latest series mentioned in it
     mbx = mailbox.mbox(mboxfile)
     base_msg = None
@@ -415,7 +415,7 @@ def get_extra_series(mboxfile, direction=1, wantvers=None):
             continue
         t_mbx_url = '%st.mbox.gz' % link
         savefile = mkstemp('b4-get')[1]
-        nt_mboxfile = b4.get_pi_thread_by_url(t_mbx_url, savefile)
+        nt_mboxfile = b4.get_pi_thread_by_url(t_mbx_url, savefile, nocache=nocache)
         nt_mbx = mailbox.mbox(nt_mboxfile)
         # Append all of these to the existing mailbox
         new_adds = 0