ez: initial rework of web submission endpoint

Reimplement initial enrolment with the web submission endpoint. A lot
more work is required before this is useful, but we're at least able to
authenticate received messages.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

4 files changed, 245 insertions, 122 deletions

diff --git a/b4/command.py b/b4/command.py
index a434d30..25615e3 100644
--- a/b4/command.py
+++ b/b4/command.py
@@ -282,7 +282,7 @@ def cmd():
                         help='Apply trailers without email address match checking')
     sp_trl.set_defaults(func=cmd_trailers)
 
-    # ez-send
+    # b4 send
     sp_send = subparsers.add_parser('send', help='Submit your work for review on the mailing lists')
     sp_send.add_argument('-d', '--dry-run', dest='dryrun', action='store_true', default=False,
                          help='Do not send, just dump out raw smtp messages to the stdout')
@@ -298,6 +298,11 @@ def cmd():
                          help='Remove yourself from the To: or Cc: list')
     sp_send.add_argument('--no-sign', action='store_true', default=False,
                          help='Do not cryptographically sign your patches with patatt')
+    ag_sendh = sp_send.add_argument_group('Web submission', 'Authenticate with the web submission endpoint')
+    ag_sendh.add_argument('--web-auth-new', dest='auth_new', action='store_true', default=False,
+                          help='Initiate a new web authentication request')
+    ag_sendh.add_argument('--web-auth-verify', dest='auth_verify', metavar='VERIFY_TOKEN',
+                          help='Submit the token received via verification email')
     sp_send.set_defaults(func=cmd_send)
 
     cmdargs = parser.parse_args()
diff --git a/b4/ez.py b/b4/ez.py
index 5aeb6be..6e3409e 100644
--- a/b4/ez.py
+++ b/b4/ez.py
@@ -24,9 +24,6 @@ import base64
 import textwrap
 import gzip
 
-# from nacl.signing import SigningKey
-# from nacl.encoding import Base64Encoder
-
 from typing import Optional, Tuple, List
 from email import utils
 from string import Template
@@ -71,76 +68,146 @@ Changes in v${newrev}:
 
 """
 
-# def auth_new(cmdargs: argparse.Namespace) -> None:
-#     # Check if we have a patatt signingkey already defined
-#     endpoint, name, email, ptskey = get_configs()
-#     skey, pkey = get_patatt_ed25519keys(ptskey)
-#     logger.info('Will submit a new email authorization request to:')
-#     logger.info('  Endpoint: %s', endpoint)
-#     logger.info('      Name: %s', name)
-#     logger.info('     Email: %s', email)
-#     logger.info('       Key: %s (%s)', pkey, ptskey)
-#     logger.info('---')
-#     confirm = input('Confirm selection [y/N]: ')
-#     if confirm != 'y':
-#         logger.info('Exiting')
-#         sys.exit(0)
-#     req = {
-#         'action': 'auth-new',
-#         'name': name,
-#         'email': email,
-#         'key': pkey,
-#     }
-#     ses = b4.get_requests_session()
-#     res = ses.post(endpoint, json=req)
-#     logger.info('---')
-#     if res.status_code == 200:
-#         try:
-#             rdata = res.json()
-#             if rdata.get('result') == 'success':
-#                 logger.info('Challenge generated and sent to %s', email)
-#                 logger.info('Once you receive it, run b4 submit --web-auth-verify [challenge-string]')
-#             sys.exit(0)
-#
-#         except Exception as ex:  # noqa
-#             logger.critical('Odd response from the endpoint: %s', res.text)
-#             sys.exit(1)
-#
-#     logger.critical('500 response from the endpoint: %s', res.text)
-#     sys.exit(1)
-#
-#
-# def auth_verify(cmdargs: argparse.Namespace) -> None:
-#     endpoint, name, email, ptskey = get_configs()
-#     skey, pkey = get_patatt_ed25519keys(ptskey)
-#     challenge = cmdargs.auth_verify
-#     logger.info('Signing challenge using key %s', ptskey)
-#     sk = SigningKey(skey.encode(), encoder=Base64Encoder)
-#     bdata = sk.sign(challenge.encode(), encoder=Base64Encoder)
-#     req = {
-#         'action': 'auth-verify',
-#         'name': name,
-#         'email': email,
-#         'challenge': challenge,
-#         'sigdata': bdata.decode(),
-#     }
-#     ses = b4.get_requests_session()
-#     res = ses.post(endpoint, json=req)
-#     logger.info('---')
-#     if res.status_code == 200:
-#         try:
-#             rdata = res.json()
-#             if rdata.get('result') == 'success':
-#                 logger.info('Challenge successfully verified for %s', email)
-#                 logger.info('You may now use this endpoint for submitting patches.')
-#             sys.exit(0)
-#
-#         except Exception as ex:  # noqa
-#             logger.critical('Odd response from the endpoint: %s', res.text)
-#             sys.exit(1)
-#
-#     logger.critical('500 response from the endpoint: %s', res.text)
-#     sys.exit(1)
+
+def get_auth_configs() -> Tuple[str, str, str, str, str, str]:
+    config = b4.get_main_config()
+    endpoint = config.get('send-endpoint-web')
+    if not endpoint:
+        raise RuntimeError('No web submission endpoint defined, set b4.send-endpoint-web')
+
+    usercfg = b4.get_user_config()
+    myemail = usercfg.get('email')
+    if not myemail:
+        raise RuntimeError('No email configured, set user.email')
+    myname = usercfg.get('name')
+    pconfig = patatt.get_main_config()
+    selector = pconfig.get('selector', 'default')
+    algo, keydata = patatt.get_algo_keydata(pconfig)
+    return endpoint, myname, myemail, selector, algo, keydata
+
+
+def auth_new(cmdargs: argparse.Namespace) -> None:
+    try:
+        endpoint, myname, myemail, selector, algo, keydata = get_auth_configs()
+    except patatt.NoKeyError as ex:
+        logger.critical('CRITICAL: no usable signing key configured')
+        logger.critical('          %s', ex)
+        sys.exit(1)
+    except RuntimeError as ex:
+        logger.critical('CRITICAL: unable to set up web authentication')
+        logger.critical('          %s', ex)
+        sys.exit(1)
+
+    if algo == 'openpgp':
+        gpgargs = ['--export', '--export-options', 'export-minimal', '-a', keydata]
+        ecode, out, err = b4.gpg_run_command(gpgargs)
+        if ecode > 0:
+            logger.critical('CRITICAL: unable to get PGP public key for %s:%s', algo, keydata)
+            sys.exit(1)
+        pubkey = out.decode()
+    elif algo == 'ed25519':
+        from nacl.signing import SigningKey
+        from nacl.encoding import Base64Encoder
+        sk = SigningKey(keydata, encoder=Base64Encoder)
+        pubkey = base64.b64encode(sk.verify_key.encode()).decode()
+    else:
+        logger.critical('CRITICAL: algorithm %s not currently supported for web endpoint submission', algo)
+        sys.exit(1)
+
+    logger.info('Will submit a new email authorization request to:')
+    logger.info('  Endpoint: %s', endpoint)
+    logger.info('      Name: %s', myname)
+    logger.info('  Identity: %s', myemail)
+    logger.info('  Selector: %s', selector)
+    if algo == 'openpgp':
+        logger.info('    Pubkey: %s:%s', algo, keydata)
+    else:
+        logger.info('    Pubkey: %s:%s', algo, pubkey)
+    logger.info('---')
+    try:
+        input('Press Enter to confirm or Ctrl-C to abort')
+    except KeyboardInterrupt:
+        logger.info('')
+        sys.exit(130)
+
+    req = {
+        'action': 'auth-new',
+        'name': myname,
+        'identity': myemail,
+        'selector': selector,
+        'pubkey': pubkey,
+    }
+    logger.info('Submitting new auth request to %s', endpoint)
+    ses = b4.get_requests_session()
+    try:
+        res = ses.post(endpoint, json=req)
+        res.raise_for_status()
+    except Exception as ex:
+        logger.critical('CRITICAL: unable to send endpoint request')
+        logger.critical('          %s', ex)
+        sys.exit(1)
+    logger.info('---')
+    if res.status_code == 200:
+        try:
+            rdata = res.json()
+            if rdata.get('result') == 'success':
+                logger.info('Challenge generated and sent to %s', myemail)
+                logger.info('Once you receive it, run b4 send --web-auth-verify [challenge-string]')
+            sys.exit(0)
+
+        except Exception as ex:  # noqa
+            logger.critical('Odd response from the endpoint: %s', res.text)
+            sys.exit(1)
+
+    logger.critical('500 response from the endpoint: %s', res.text)
+    sys.exit(1)
+
+
+def auth_verify(cmdargs: argparse.Namespace) -> None:
+    vstr = cmdargs.auth_verify
+    endpoint, myname, myemail, selector, algo, keydata = get_auth_configs()
+    logger.info('Signing challenge')
+    # Create a minimal message
+    cmsg = email.message.EmailMessage()
+    cmsg.add_header('From', myemail)
+    cmsg.add_header('Subject', 'b4-send-verify')
+    cmsg.set_payload(f'verify:{vstr}\n')
+    bdata = cmsg.as_bytes(policy=b4.emlpolicy)
+    try:
+        bdata = patatt.rfc2822_sign(bdata).decode()
+    except patatt.SigningError as ex:
+        logger.critical('CRITICAL: Unable to sign verification message')
+        logger.critical('          %s', ex)
+        sys.exit(1)
+
+    req = {
+        'action': 'auth-verify',
+        'msg': bdata.encode(),
+    }
+    logger.info('Submitting verification to %s', endpoint)
+    ses = b4.get_requests_session()
+    try:
+        res = ses.post(endpoint, json=req)
+        res.raise_for_status()
+    except Exception as ex:
+        logger.critical('CRITICAL: unable to send endpoint request')
+        logger.critical('          %s', ex)
+        sys.exit(1)
+    logger.info('---')
+    if res.status_code == 200:
+        try:
+            rdata = res.json()
+            if rdata.get('result') == 'success':
+                logger.info('Challenge successfully verified for %s', myemail)
+                logger.info('You may now use this endpoint for submitting patches.')
+            sys.exit(0)
+
+        except Exception as ex:  # noqa
+            logger.critical('Odd response from the endpoint: %s', res.text)
+            sys.exit(1)
+
+    logger.critical('500 response from the endpoint: %s', res.text)
+    sys.exit(1)
 
 
 def get_rev_count(revrange: str, maxrevs: Optional[int] = 500) -> int:
@@ -865,6 +932,12 @@ def format_patch(output_dir: str) -> None:
 
 
 def cmd_send(cmdargs: argparse.Namespace) -> None:
+    if cmdargs.auth_new:
+        auth_new(cmdargs)
+        return
+    if cmdargs.auth_verify:
+        auth_verify(cmdargs)
+        return
     # Check if the cover letter has 'EDITME' in it
     cover, tracking = load_cover(strip_comments=True)
     if 'EDITME' in cover:
diff --git a/misc/send-receive.py b/misc/send-receive.py
index 7b47798..c102508 100644
--- a/misc/send-receive.py
+++ b/misc/send-receive.py
@@ -5,10 +5,7 @@ import os
 import logging
 import json
 import sqlalchemy as sa
-
-from nacl.signing import VerifyKey
-from nacl.encoding import Base64Encoder
-from nacl.exceptions import BadSignatureError
+import patatt
 
 DB_VERSION = 1
 
@@ -34,19 +31,13 @@ class SendReceiveListener(object):
         auth = sa.Table('auth', md,
                         sa.Column('auth_id', sa.Integer(), primary_key=True),
                         sa.Column('created', sa.DateTime(), nullable=False, server_default=sa.sql.func.now()),
-                        sa.Column('email', sa.Text(), nullable=False),
-                        sa.Column('name', sa.Text(), nullable=False),
+                        sa.Column('identity', sa.Text(), nullable=False),
+                        sa.Column('selector', sa.Text(), nullable=False),
                         sa.Column('pubkey', sa.Text(), nullable=False),
+                        sa.Column('challenge', sa.Text(), nullable=True),
+                        sa.Column('verified', sa.Integer(), nullable=False),
                         )
-        sa.Index('idx_email_pubkey', auth.c.pubkey, auth.c.email, unique=True)
-        challenge = sa.Table('challenge', md,
-                             sa.Column('challenge_id', sa.Integer(), primary_key=True),
-                             sa.Column('created', sa.DateTime(), nullable=False, server_default=sa.sql.func.now()),
-                             sa.Column('pubkey', sa.Text(), nullable=False),
-                             sa.Column('email', sa.Text(), nullable=False),
-                             sa.Column('challenge', sa.Text(), nullable=False),
-                             )
-        sa.Index('idx_uniq_challenge', challenge.c.pubkey, challenge.c.email, challenge.c.challenge, unique=True)
+        sa.Index('idx_identity_selector', auth.c.identity, auth.c.selector, unique=True)
         md.create_all(self._engine)
         q = sa.insert(meta).values(version=DB_VERSION)
         conn.execute(q)
@@ -68,56 +59,107 @@ class SendReceiveListener(object):
         # Is it already authorized?
         conn = self._engine.connect()
         md = sa.MetaData()
+        identity = jdata.get('identity')
+        selector = jdata.get('selector')
+        pubkey = jdata.get('pubkey')
         t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
-        email = jdata.get('email')
-        pubkey = jdata.get('key')
-        q = sa.select([t_auth.c.auth_id]).where(t_auth.c.email == email, t_auth.c.pubkey == pubkey)
+        q = sa.select([t_auth.c.auth_id]).where(t_auth.c.identity == identity, t_auth.c.selector == selector,
+                                                t_auth.c.verified == 1)
         rp = conn.execute(q)
         if len(rp.fetchall()):
-            self.send_error(resp, message='%s:%s is already authorized' % (email, pubkey))
+            self.send_error(resp, message='i=%s;s=%s is already authorized' % (identity, selector))
             return
         # delete any existing challenges for this and create a new one
-        t_challenge = sa.Table('challenge', md, autoload=True, autoload_with=self._engine)
-        q = sa.delete(t_challenge).where(t_challenge.c.email == email, t_challenge.c.pubkey == pubkey)
+        q = sa.delete(t_auth).where(t_auth.c.identity == identity, t_auth.c.selector == selector,
+                                    t_auth.c.verified == 0)
         conn.execute(q)
         # create new challenge
         import uuid
         cstr = str(uuid.uuid4())
-        q = sa.insert(t_challenge).values(pubkey=pubkey, email=email, challenge=cstr)
+        q = sa.insert(t_auth).values(identity=identity, selector=selector, pubkey=pubkey, challenge=cstr,
+                                     verified=0)
         conn.execute(q)
         # TODO: Actual mail sending
         logger.info('Challenge: %s', cstr)
         self.send_success(resp, message='Challenge generated')
 
+    def validate_message(self, conn, t_auth, bdata, verified=1):
+        # Returns auth_id of the matching record
+        pm = patatt.PatattMessage(bdata)
+        if not pm.signed:
+            return None
+        auth_id = identity = pubkey = None
+        for ds in pm.get_sigs():
+            selector = 'default'
+            identity = ''
+            i = ds.get_field('i')
+            if i:
+                identity = i.decode()
+            s = ds.get_field('s')
+            if s:
+                selector = s.decode()
+            logger.debug('i=%s; s=%s', identity, selector)
+            q = sa.select([t_auth.c.auth_id, t_auth.c.pubkey]).where(t_auth.c.identity == identity,
+                                                                     t_auth.c.selector == selector,
+                                                                     t_auth.c.verified == verified)
+            rp = conn.execute(q)
+            res = rp.fetchall()
+            if res:
+                auth_id, pubkey = res[0]
+                break
+
+        logger.debug('auth_id=%s', auth_id)
+        if not auth_id:
+            return None
+        try:
+            pm.validate(identity, pubkey.encode())
+        except Exception as ex:
+            logger.debug('Validation failed: %s', ex)
+            return None
+
+        return auth_id
+
     def auth_verify(self, jdata, resp):
-        # Do we have a record for this email/challenge?
+        msg = jdata.get('msg')
+        if msg.find('\nverify:') < 0:
+            self.send_error(resp, message='Invalid verification message')
+            return
         conn = self._engine.connect()
         md = sa.MetaData()
-        t_challenge = sa.Table('challenge', md, autoload=True, autoload_with=self._engine)
-        email = jdata.get('email', '')
-        challenge = jdata.get('challenge', '')
-        sigdata = jdata.get('sigdata', '')
-        q = sa.select([t_challenge.c.pubkey]).where(t_challenge.c.email == email, t_challenge.c.challenge == challenge)
+        t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
+        bdata = msg.encode()
+        auth_id = self.validate_message(conn, t_auth, bdata, verified=0)
+        if auth_id is None:
+            self.send_error(resp, message='Signature validation failed')
+            return
+        # Now compare the challenge to what we received
+        q = sa.select([t_auth.c.challenge]).where(t_auth.c.auth_id == auth_id)
         rp = conn.execute(q)
-        qres = rp.fetchall()
-        if not len(qres):
-            self.send_error(resp, message='No such challenge for %s' % email)
+        res = rp.fetchall()
+        challenge = res[0][0]
+        if msg.find(f'\nverify:{challenge}') < 0:
+            self.send_error(resp, message='Invalid verification string')
             return
-        pubkey = qres[0][0]
-        vk = VerifyKey(pubkey.encode(), encoder=Base64Encoder)
-        try:
-            vk.verify(sigdata.encode(), encoder=Base64Encoder)
-        except BadSignatureError:
-            self.send_error(resp, message='Could not validate signature for %s' % email)
+        q = sa.update(t_auth).where(t_auth.c.auth_id == auth_id).values(challenge=None, verified=1)
+        conn.execute(q)
+        self.send_success(resp, message='Challenge verified')
+
+    def auth_delete(self, jdata, resp):
+        msg = jdata.get('msg')
+        if msg.find('\nauth-delete') < 0:
+            self.send_error(resp, message='Invalid key delete message')
             return
-        # validated at this point, so record this as valid auth
-        name = jdata.get('name')
+        conn = self._engine.connect()
+        md = sa.MetaData()
         t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
-        q = sa.insert(t_auth).values(pubkey=pubkey, name=name, email=email)
-        conn.execute(q)
-        q = sa.delete(t_challenge).where(t_challenge.c.email == email, t_challenge.c.challenge == challenge)
+        bdata = msg.encode()
+        auth_id = self.validate_message(conn, t_auth, bdata)
+        if auth_id is None:
+            self.send_error(resp, message='Signature validation failed')
+            return
+        q = sa.delete(t_auth).where(t_auth.c.auth_id == auth_id)
         conn.execute(q)
-        self.send_success(resp, message='Challenge verified')
+        self.send_success(resp, message='Authentication deleted')
 
     def on_post(self, req, resp):
         if not req.content_length:
@@ -133,17 +175,20 @@ class SendReceiveListener(object):
             resp.content_type = falcon.MEDIA_TEXT
             resp.text = 'Failed to parse the request\n'
             return
-        logger.info(jdata)
         action = jdata.get('action')
         if action == 'auth-new':
             self.auth_new(jdata, resp)
+            return
         if action == 'auth-verify':
             self.auth_verify(jdata, resp)
-        else:
-            resp.status = falcon.HTTP_500
-            resp.content_type = falcon.MEDIA_TEXT
-            resp.text = 'Unknown action: %s\n' % action
             return
+        if action == 'auth-delete':
+            self.auth_delete(jdata, resp)
+            return
+
+        resp.status = falcon.HTTP_500
+        resp.content_type = falcon.MEDIA_TEXT
+        resp.text = 'Unknown action: %s\n' % action
 
 
 app = falcon.App()
diff --git a/misc/test.sqlite b/misc/test.sqlite
deleted file mode 100644
index fc7a99b..0000000
--- a/misc/test.sqlite
+++ /dev/null