aboutsummaryrefslogtreecommitdiff
path: root/misc/send-receive.py
blob: c15e12adc98a89a089b11e66a4fe0acd549dd000 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
#!/usr/bin/env python3
# noinspection PyUnresolvedReferences
import falcon
import os
import sys
import logging
import logging.handlers
import json
import sqlalchemy as sa
import patatt
import smtplib
import email
import email.header
import email.policy
import re
import ezpi
import copy

from configparser import ConfigParser, ExtendedInterpolation
from string import Template
from email import utils
from typing import Tuple, Union

from email import charset
charset.add_charset('utf-8', None)
emlpolicy = email.policy.EmailPolicy(utf8=True, cte_type='8bit', max_line_length=None)

DB_VERSION = 1

logger = logging.getLogger('b4-send-receive')
logger.setLevel(logging.DEBUG)


# noinspection PyBroadException, PyMethodMayBeStatic
class SendReceiveListener(object):

    def __init__(self, _engine, _config) -> None:
        self._engine = _engine
        self._config = _config
        # You shouldn't use this in production
        if self._engine.driver == 'pysqlite':
            self._init_sa_db()
        logfile = _config['main'].get('logfile')
        loglevel = _config['main'].get('loglevel', 'info')
        if logfile:
            self._init_logger(logfile, loglevel)

    def _init_logger(self, logfile: str, loglevel: str) -> None:
        global logger
        lch = logging.handlers.WatchedFileHandler(os.path.expanduser(logfile))
        lfmt = logging.Formatter('[%(process)d] %(asctime)s - %(levelname)s - %(message)s')
        lch.setFormatter(lfmt)
        if loglevel == 'critical':
            lch.setLevel(logging.CRITICAL)
        elif loglevel == 'debug':
            lch.setLevel(logging.DEBUG)
        else:
            lch.setLevel(logging.INFO)
        logger.addHandler(lch)

    def _init_sa_db(self) -> None:
        logger.info('Setting up SQLite database')
        conn = self._engine.connect()
        md = sa.MetaData()
        meta = sa.Table('meta', md,
                        sa.Column('version', sa.Integer())
                        )
        auth = sa.Table('auth', md,
                        sa.Column('auth_id', sa.Integer(), primary_key=True),
                        sa.Column('created', sa.DateTime(), nullable=False, server_default=sa.sql.func.now()),
                        sa.Column('identity', sa.Text(), nullable=False),
                        sa.Column('selector', sa.Text(), nullable=False),
                        sa.Column('pubkey', sa.Text(), nullable=False),
                        sa.Column('challenge', sa.Text(), nullable=True),
                        sa.Column('verified', sa.Integer(), nullable=False),
                        )
        sa.Index('idx_identity_selector', auth.c.identity, auth.c.selector, unique=True)
        md.create_all(self._engine)
        q = sa.insert(meta).values(version=DB_VERSION)
        conn.execute(q)
        conn.close()

    def on_get(self, req, resp):  # noqa
        resp.status = falcon.HTTP_200
        resp.content_type = falcon.MEDIA_TEXT
        resp.text = "We don't serve GETs here\n"

    def send_error(self, resp, message: str) -> None:
        resp.status = falcon.HTTP_500
        logger.critical('Returning error: %s', message)
        resp.text = json.dumps({'result': 'error', 'message': message})

    def send_success(self, resp, message: str) -> None:
        resp.status = falcon.HTTP_200
        logger.debug('Returning success: %s', message)
        resp.text = json.dumps({'result': 'success', 'message': message})

    def get_smtp(self) -> Tuple[Union[smtplib.SMTP, smtplib.SMTP_SSL, None], Tuple[str, str]]:
        sconfig = self._config['sendemail']
        server = sconfig.get('smtpserver', 'localhost')
        port = sconfig.get('smtpserverport', 0)
        encryption = sconfig.get('smtpencryption')

        logger.debug('Connecting to %s:%s', server, port)
        # We only authenticate if we have encryption
        if encryption:
            if encryption in ('tls', 'starttls'):
                # We do startssl
                smtp = smtplib.SMTP(server, port)
                # Introduce ourselves
                smtp.ehlo()
                # Start encryption
                smtp.starttls()
                # Introduce ourselves again to get new criteria
                smtp.ehlo()
            elif encryption in ('ssl', 'smtps'):
                # We do TLS from the get-go
                smtp = smtplib.SMTP_SSL(server, port)
            else:
                raise smtplib.SMTPException('Unclear what to do with smtpencryption=%s' % encryption)

            # If we got to this point, we should do authentication.
            auser = sconfig.get('smtpuser')
            apass = sconfig.get('smtppass')
            if auser and apass:
                # Let any exceptions bubble up
                smtp.login(auser, apass)
        else:
            # We assume you know what you're doing if you don't need encryption
            smtp = smtplib.SMTP(server, port)

        frompair = utils.getaddresses([sconfig.get('from')])[0]
        return smtp, frompair

    def auth_new(self, jdata, resp) -> None:
        # Is it already authorized?
        conn = self._engine.connect()
        md = sa.MetaData()
        identity = jdata.get('identity')
        selector = jdata.get('selector')
        logger.info('New authentication request for %s/%s', identity, selector)
        pubkey = jdata.get('pubkey')
        t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
        q = sa.select([t_auth.c.auth_id]).where(t_auth.c.identity == identity, t_auth.c.selector == selector,
                                                t_auth.c.verified == 1)
        rp = conn.execute(q)
        if len(rp.fetchall()):
            self.send_error(resp, message='i=%s;s=%s is already authorized' % (identity, selector))
            return
        # delete any existing challenges for this and create a new one
        q = sa.delete(t_auth).where(t_auth.c.identity == identity, t_auth.c.selector == selector,
                                    t_auth.c.verified == 0)
        conn.execute(q)
        # create new challenge
        import uuid
        cstr = str(uuid.uuid4())
        q = sa.insert(t_auth).values(identity=identity, selector=selector, pubkey=pubkey, challenge=cstr,
                                     verified=0)
        conn.execute(q)
        logger.info('Created new challenge for %s/%s: %s', identity, selector, cstr)
        conn.close()
        smtp, frompair = self.get_smtp()
        cmsg = email.message.EmailMessage()
        fromname, fromaddr = frompair
        if len(fromname):
            cmsg.add_header('From', f'{fromname} <{fromaddr}>')
        else:
            cmsg.add_header('From', fromaddr)
        tpt_subject = self._config['templates']['verify-subject'].strip()
        tpt_body = self._config['templates']['verify-body'].strip()
        signature = self._config['templates']['signature'].strip()
        subject = Template(tpt_subject).safe_substitute({'identity': jdata.get('identity')})
        cmsg.add_header('Subject', subject)
        name = jdata.get('name', 'Anonymous Llama')
        cmsg.add_header('To', f'{name} <{identity}>')
        cmsg.add_header('Message-Id', utils.make_msgid('b4-verify'))
        vals = {
            'name': name,
            'myurl': self._config['main'].get('myurl'),
            'challenge': cstr,
        }
        body = Template(tpt_body).safe_substitute(vals)
        body += '\n-- \n'
        body += Template(signature).safe_substitute(vals)
        body += '\n'
        cmsg.set_payload(body, charset='utf-8')
        bdata = cmsg.as_bytes(policy=emlpolicy)
        destaddrs = [identity]
        alwaysbcc = self._config['main'].get('alwayscc')
        if alwaysbcc:
            destaddrs += [x[1] for x in utils.getaddresses(alwaysbcc)]
        logger.info('Sending challenge to %s', identity)
        smtp.sendmail(fromaddr, [identity], bdata)
        smtp.close()
        self.send_success(resp, message=f'Challenge generated and sent to {identity}')

    def validate_message(self, conn, t_auth, bdata, verified=1) -> Tuple[str, str, int]:
        # Returns auth_id of the matching record
        pm = patatt.PatattMessage(bdata)
        if not pm.signed:
            raise patatt.ValidationError('Message is not signed')

        auth_id = identity = selector = pubkey = None
        for ds in pm.get_sigs():
            selector = 'default'
            identity = ''
            i = ds.get_field('i')
            if i:
                identity = i.decode()
            s = ds.get_field('s')
            if s:
                selector = s.decode()
            logger.debug('i=%s; s=%s', identity, selector)
            q = sa.select([t_auth.c.auth_id, t_auth.c.pubkey]).where(t_auth.c.identity == identity,
                                                                     t_auth.c.selector == selector,
                                                                     t_auth.c.verified == verified)
            rp = conn.execute(q)
            res = rp.fetchall()
            if res:
                auth_id, pubkey = res[0]
                break

        if not auth_id:
            logger.debug('Did not find a matching identity!')
            raise patatt.NoKeyError('No match for this identity')

        logger.debug('Found matching %s/%s with auth_id=%s', identity, selector, auth_id)
        pm.validate(identity, pubkey.encode())

        return identity, selector, auth_id

    def auth_verify(self, jdata, resp) -> None:
        msg = jdata.get('msg')
        if msg.find('\nverify:') < 0:
            self.send_error(resp, message='Invalid verification message')
            return
        conn = self._engine.connect()
        md = sa.MetaData()
        t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
        bdata = msg.encode()
        try:
            identity, selector, auth_id = self.validate_message(conn, t_auth, bdata, verified=0)
        except Exception as ex:
            self.send_error(resp, message='Signature validation failed: %s' % ex)
            return
        logger.debug('Message validation passed for %s/%s with auth_id=%s', identity, selector, auth_id)

        # Now compare the challenge to what we received
        q = sa.select([t_auth.c.challenge]).where(t_auth.c.auth_id == auth_id)
        rp = conn.execute(q)
        res = rp.fetchall()
        challenge = res[0][0]
        if msg.find(f'\nverify:{challenge}') < 0:
            self.send_error(resp, message='Challenge verification for %s/%s did not match' % (identity, selector))
            return
        logger.info('Successfully verified challenge for %s/%s with auth_id=%s', identity, selector, auth_id)
        q = sa.update(t_auth).where(t_auth.c.auth_id == auth_id).values(challenge=None, verified=1)
        conn.execute(q)
        conn.close()
        self.send_success(resp, message='Challenge verified for %s/%s' % (identity, selector))

    def auth_delete(self, jdata, resp) -> None:
        msg = jdata.get('msg')
        if msg.find('\nauth-delete') < 0:
            self.send_error(resp, message='Invalid key delete message')
            return
        conn = self._engine.connect()
        md = sa.MetaData()
        t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
        bdata = msg.encode()
        try:
            identity, selector, auth_id = self.validate_message(conn, t_auth, bdata)
        except Exception as ex:
            self.send_error(resp, message='Signature validation failed: %s' % ex)
            return

        logger.info('Deleting record for %s/%s with auth_id=%s', identity, selector, auth_id)
        q = sa.delete(t_auth).where(t_auth.c.auth_id == auth_id)
        conn.execute(q)
        conn.close()
        self.send_success(resp, message='Record deleted for %s/%s' % (identity, selector))

    def clean_header(self, hdrval: str) -> str:
        if hdrval is None:
            return ''

        decoded = ''
        for hstr, hcs in email.header.decode_header(hdrval):
            if hcs is None:
                hcs = 'utf-8'
            try:
                decoded += hstr.decode(hcs, errors='replace')
            except LookupError:
                # Try as utf-u
                decoded += hstr.decode('utf-8', errors='replace')
            except (UnicodeDecodeError, AttributeError):
                decoded += hstr
        new_hdrval = re.sub(r'\n?\s+', ' ', decoded)
        return new_hdrval.strip()

    def receive(self, jdata, resp) -> None:
        servicename = self._config['main'].get('myname')
        if not servicename:
            servicename = 'Web Endpoint'
        umsgs = jdata.get('messages')
        if not umsgs:
            self.send_error(resp, message='Missing the messages array')
            return
        logger.debug('Received a request for %s messages', len(umsgs))

        diffre = re.compile(r'^(---.*\n\+\+\+|GIT binary patch|diff --git \w/\S+ \w/\S+)', flags=re.M | re.I)
        diffstatre = re.compile(r'^\s*\d+ file.*\d+ (insertion|deletion)', flags=re.M | re.I)

        msgs = list()
        conn = self._engine.connect()
        md = sa.MetaData()
        t_auth = sa.Table('auth', md, autoload=True, autoload_with=self._engine)
        mustdest = self._config['main'].get('mustdest')
        # First, validate all messages
        seenid = identity = selector = None
        for umsg in umsgs:
            try:
                identity, selector, auth_id = self.validate_message(conn, t_auth, umsg.encode())
            except patatt.NoKeyError as ex:  # noqa
                self.send_error(resp, message='No matching record found, maybe you need to auth-verify first?')
                return
            except Exception as ex:
                self.send_error(resp, message='Signature validation failed: %s' % ex)
                return

            # Make sure only a single auth_id is used within a receive session
            if seenid is None:
                seenid = auth_id
            elif seenid != auth_id:
                self.send_error(resp, message='We only support a single signing identity across patch series.')
                return

            msg = email.message_from_string(umsg)
            logger.debug('Checking sanity on message: %s', msg.get('Subject'))
            # Some quick sanity checking:
            # - Subject has to start with [PATCH
            # - Content-type may ONLY be text/plain
            # - Has to include a diff or a diffstat
            passes = True
            if not msg.get('Subject', '').startswith('[PATCH '):
                passes = False
            if passes:
                cte = msg.get_content_type()
                if cte.lower() != 'text/plain':
                    passes = False
            if passes:
                payload = msg.get_payload()
                if not (diffre.search(payload) or diffstatre.search(payload)):
                    passes = False

            if not passes:
                self.send_error(resp, message='This service only accepts patches')
                return

            # Make sure that From, Date, Subject, and Message-Id headers exist
            if not msg.get('From') or not msg.get('Date') or not msg.get('Subject') or not msg.get('Message-Id'):
                self.send_error(resp, message='Message is missing some required headers.')
                return

            # Check that To/Cc have a mailing list we recognize
            alldests = utils.getaddresses([str(x) for x in msg.get_all('to', [])])
            alldests += utils.getaddresses([str(x) for x in msg.get_all('cc', [])])
            destaddrs = {x[1] for x in alldests}
            if mustdest:
                matched = False
                for destaddr in destaddrs:
                    if re.search(mustdest, destaddr, flags=re.I):
                        matched = True
                        break
                if not matched:
                    self.send_error(resp, message='Destinations must include a mailing list we recognize.')
                    return
            msg.add_header('X-Endpoint-Received', f'by {servicename} for {identity}/{selector} with auth_id={auth_id}')
            msgs.append((msg, destaddrs))

        conn.close()
        # All signatures verified. Prepare messages for sending.
        cfgdomains = self._config['main'].get('mydomains')
        if cfgdomains is not None:
            mydomains = [x.strip() for x in cfgdomains.split(',')]
        else:
            mydomains = list()

        smtp, frompair = self.get_smtp()
        bccaddrs = set()
        _bcc = self._config['main'].get('alwaysbcc')
        if _bcc:
            bccaddrs.update([x[1] for x in utils.getaddresses([_bcc])])

        repo = listid = None
        if 'public-inbox' in self._config and self._config['public-inbox'].get('repo'):
            repo = self._config['public-inbox'].get('repo')
            listid = self._config['public-inbox'].get('listid')
            if not os.path.isdir(repo):
                repo = None

        logger.info('Sending %s messages for %s/%s', len(msgs), identity, selector)
        for msg, destaddrs in msgs:
            subject = self.clean_header(msg.get('Subject'))
            if repo:
                pmsg = copy.deepcopy(msg)
                if pmsg.get('List-Id'):
                    pmsg.replace_header('List-Id', listid)
                else:
                    pmsg.add_header('List-Id', listid)
                ezpi.add_rfc822(repo, pmsg)
                logger.debug('Wrote %s to public-inbox at %s', subject, repo)

            origfrom = self.clean_header(msg.get('From'))
            origpair = utils.getaddresses([origfrom])[0]
            origaddr = origpair[1]
            # Does it match one of our domains
            mydomain = False
            for _domain in mydomains:
                if origaddr.endswith(f'@{_domain}'):
                    mydomain = True
                    break
            if mydomain:
                logger.debug('%s matches mydomain, no substitution required', origaddr)
                fromaddr = origaddr
            else:
                logger.debug('%s does not match mydomain, substitution required', origaddr)
                fromaddr = frompair[1]
                # We can't just send this as-is due to DMARC policies. Therefore, we set
                # Reply-To and X-Original-From.
                origname = origpair[0]
                if not origname:
                    origname = origpair[1]
                msg.replace_header('From', f'{origname} via {servicename} <{fromaddr}>')

                if msg.get('X-Original-From'):
                    msg.replace_header('X-Original-From', origfrom)
                else:
                    msg.add_header('X-Original-From', origfrom)
                if msg.get('Reply-To'):
                    msg.replace_header('Reply-To', f'<{origpair[1]}>')
                else:
                    msg.add_header('Reply-To', f'<{origpair[1]}>')

                body = msg.get_payload()
                # Parse it as a message and see if we get a From: header
                cmsg = email.message_from_string(body)
                if cmsg.get('From') is None:
                    cmsg.add_header('From', origfrom)
                    msg.set_payload(cmsg.as_string(policy=emlpolicy, maxheaderlen=0), charset='utf-8')

            if bccaddrs:
                destaddrs.update(bccaddrs)

            bdata = msg.as_string(policy=emlpolicy).encode()

            if not self._config['main'].getboolean('dryrun'):
                smtp.sendmail(fromaddr, list(destaddrs), bdata)
                logger.info('Sent: %s', subject)
            else:
                logger.info('---DRYRUN MSG START---')
                logger.info(msg)
                logger.info('---DRYRUN MSG END---')

        smtp.close()
        if repo:
            # run it once after writing all messages
            logger.debug('Running public-inbox repo hook (if present)')
            ezpi.run_hook(repo)
        logger.info('Sent %s messages for %s/%s', len(msgs), identity, selector)
        self.send_success(resp, message=f'Sent {len(msgs)} messages for {identity}/{selector}')

    def on_post(self, req, resp):
        if not req.content_length:
            resp.status = falcon.HTTP_500
            resp.content_type = falcon.MEDIA_TEXT
            resp.text = 'Payload required\n'
            return
        raw = req.bounded_stream.read()
        try:
            jdata = json.loads(raw)
        except:
            resp.status = falcon.HTTP_500
            resp.content_type = falcon.MEDIA_TEXT
            resp.text = 'Failed to parse the request\n'
            return
        action = jdata.get('action')
        if not action:
            logger.critical('Action not set from %s', req.remote_addr)

        logger.info('Action: %s; from: %s', action, req.remote_addr)
        if action == 'auth-new':
            self.auth_new(jdata, resp)
            return
        if action == 'auth-verify':
            self.auth_verify(jdata, resp)
            return
        if action == 'auth-delete':
            self.auth_delete(jdata, resp)
            return
        if action == 'receive':
            self.receive(jdata, resp)
            return

        resp.status = falcon.HTTP_500
        resp.content_type = falcon.MEDIA_TEXT
        resp.text = 'Unknown action: %s\n' % action


parser = ConfigParser(interpolation=ExtendedInterpolation())
cfgfile = os.getenv('CONFIG')
if not cfgfile or not os.path.exists(cfgfile):
    sys.stderr.write('CONFIG env var is not set or is not valid')
    sys.exit(1)

parser.read(cfgfile)

gpgbin = parser['main'].get('gpgbin')
if gpgbin:
    patatt.GPGBIN = gpgbin

dburl = parser['main'].get('dburl')
# By default, recycle db connections after 5 min
db_pool_recycle = parser['main'].getint('dbpoolrecycle', 300)
engine = sa.create_engine(dburl, pool_recycle=db_pool_recycle)
srl = SendReceiveListener(engine, parser)
app = falcon.App()
mp = os.getenv('MOUNTPOINT', '/_b4_submit')
app.add_route(mp, srl)


if __name__ == '__main__':
    from wsgiref.simple_server import make_server
    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    ch.setLevel(logging.DEBUG)
    logger.addHandler(ch)

    with make_server('', 8000, app) as httpd:
        logger.info('Serving on port 8000...')

        # Serve until process is killed
        httpd.serve_forever()