Add TLS keylog AES-GCM decryption support

README.rst

@@ -50,6 +50,19 @@ Want to go to coffee shop and see http(s) requests floating about?
    TLSRecord(length=64, version=771, type=22, data="l\xd0\xce\x96\xf5\x1a\xf8\xcf\xcc\x1...
    TLSRecord(length=560, version=771, type=23, data='\x1d\x942K\xfb\x87\x19v\xba\x13\x14...
 
+TLS key log decryption
+----------------------
+
+``urlwatch`` can use an NSS ``SSLKEYLOGFILE`` to decrypt common TLS 1.2
+AES-GCM application records after the ClientHello and ServerHello have been
+seen in the flow::
+
+   $ urlwatch --pcap captured.pcap --keylog captured.keylog
+
+This supports the ``CLIENT_RANDOM`` key log format used by browsers and tools
+that write ``SSLKEYLOGFILE``. TLS 1.3 traffic-secret reconstruction and full
+cross-flow HTTP request/response pairing are separate follow-up steps.
+
 Documentation
 -------------
 

chains/links/tls_meta.py

@@ -1,21 +1,30 @@
-"""TLSMeta, Pull out HTTP meta data from incoming flow data"""
+"""TLSMeta, Pull out TLS meta data from incoming flow data"""
 from __future__ import print_function
 import dpkt
 
 # Local imports
 from chains.links import link
-from chains.utils import file_utils, data_utils, log_utils
+from chains.utils import file_utils, log_utils, tls_keylog
 logger = log_utils.get_logger()
 
+
 class TLSMeta(link.Link):
     """Pull out application meta data from incoming flow data"""
 
-    def __init__(self):
+    def __init__(self, keylog_path=None, keylog_entries=None):
         """Initialize TLSMeta Class"""
 
         # Call super class init
         super(TLSMeta, self).__init__()
 
+        if keylog_entries is not None:
+            self.keylog_entries = keylog_entries
+        elif keylog_path:
+            self.keylog_entries = tls_keylog.load_keylog_file(keylog_path)
+        else:
+            self.keylog_entries = {}
+        self._tls_sessions = {}
+
         # Set my output
         self.output_stream = self.tls_meta_data()
 
@@ -39,7 +48,10 @@ def tls_meta_data(self):
                         logger.warning('Incomplete TLS record at the end...')
 
                     # Process the TLS records
-                    flow['tls'] = {'type':'TLS_CTS', 'data':{'tls_records': tls_records, 'uri':None, 'headers':None}}
+                    flow['tls'] = {
+                        'type': 'TLS_CTS',
+                        'data': {'tls_records': tls_records, 'uri': None, 'headers': None},
+                    }
                 except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, dpkt.ssl.SSL3Exception):
                     flow['tls'] = None
 
@@ -51,13 +63,69 @@ def tls_meta_data(self):
                         logger.warning('Incomplete TLS record at the end...')
 
                     # Process the TLS records
-                    flow['tls'] = {'type':'TLS_STC', 'data':{'tls_records': tls_records, 'uri':None, 'headers':None}}
+                    flow['tls'] = {
+                        'type': 'TLS_STC',
+                        'data': {'tls_records': tls_records, 'uri': None, 'headers': None},
+                    }
                 except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, dpkt.ssl.SSL3Exception):
                     flow['tls'] = None
 
+            if flow['tls']:
+                self._add_keylog_decryption(flow)
+
             # All done
             yield flow
 
+    def _add_keylog_decryption(self, flow):
+        """Attach TLS keylog metadata and decrypted records when possible."""
+
+        handshake = tls_keylog.extract_handshake_metadata(flow['payload'])
+        flow['tls']['data']['handshake'] = tls_keylog.handshake_metadata_for_output(handshake)
+        flow['tls']['data']['decrypted_records'] = []
+
+        if not self.keylog_entries:
+            return
+
+        session = self._session_for_flow(flow)
+        session.update(handshake)
+        if not self._session_ready(session):
+            return
+
+        direction = 'client' if flow['direction'] == 'CTS' else 'server'
+        try:
+            key_material = tls_keylog.key_material_from_keylog(
+                self.keylog_entries,
+                session['client_random'],
+                session['server_random'],
+                session['cipher_suite'],
+            )
+            records = tls_keylog.decrypt_tls12_aes_gcm_records(
+                flow['payload'],
+                key_material,
+                direction,
+            )
+            flow['tls']['data']['decrypted_records'] = tls_keylog.decrypted_records_for_output(records)
+            flow['tls']['data']['cipher_suite'] = key_material['cipher_suite_name']
+        except tls_keylog.TLSKeyLogError as error:
+            logger.warning('TLS keylog decryption skipped: %s', error)
+
+    def _session_for_flow(self, flow):
+        """Return the client/server normalized session metadata dict."""
+
+        if flow['direction'] == 'CTS':
+            session_id = (flow['src'], flow['sport'], flow['dst'], flow['dport'])
+        else:
+            session_id = (flow['dst'], flow['dport'], flow['src'], flow['sport'])
+        return self._tls_sessions.setdefault(session_id, {})
+
+    @staticmethod
+    def _session_ready(session):
+        return (
+            session.get('client_random') and
+            session.get('server_random') and
+            session.get('cipher_suite') is not None
+        )
+
     def ssl_handshake_processing(tls_records):
         """Process a set of TLS records for a SSL handshake
            In general the order of messages should be the following:
@@ -104,5 +172,6 @@ def test():
             logger.info('Could not find TLS in Flow:')
             flows.print_flow_info(item)
 
+
 if __name__ == '__main__':
     test()

chains/links/tls_meta_keylog_test.py

@@ -0,0 +1,115 @@
+"""Tests for TLSMeta keylog-backed decryption."""
+import binascii
+import struct
+
+from Crypto.Cipher import AES
+
+from chains.links.tls_meta import TLSMeta
+from chains.utils import tls_keylog
+
+
+TLS_VERSION_12 = 0x0303
+CIPHER_SUITE = 0xc02f
+CLIENT_RANDOM = b'\x11' * 32
+SERVER_RANDOM = b'\x22' * 32
+MASTER_SECRET = b'\x33' * 48
+
+
+def test_tls_meta_adds_decrypted_records_from_keylog():
+    key_material = tls_keylog.derive_tls12_key_material(
+        MASTER_SECRET,
+        CLIENT_RANDOM,
+        SERVER_RANDOM,
+        CIPHER_SUITE,
+    )
+    plaintext = b'HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n'
+    server_payload = (
+        _record(22, _handshake(2, _server_hello_body(SERVER_RANDOM, CIPHER_SUITE))) +
+        _record(20, b'\x01') +
+        _encrypted_record(
+            plaintext,
+            key_material['server_write_key'],
+            key_material['server_write_iv'],
+            sequence=0,
+        )
+    )
+    random_hex = binascii.hexlify(CLIENT_RANDOM).decode('ascii')
+    tls_meta = TLSMeta(keylog_entries={
+        random_hex: {
+            'client_random': CLIENT_RANDOM,
+            'CLIENT_RANDOM': MASTER_SECRET,
+        },
+    })
+    tls_meta.input_stream = iter([
+        _flow('CTS', _record(22, _handshake(1, _client_hello_body(CLIENT_RANDOM)))),
+        _flow('STC', server_payload),
+    ])
+    tls_meta.output_stream = tls_meta.tls_meta_data()
+
+    output = list(tls_meta.output_stream)
+
+    decrypted = output[1]['tls']['data']['decrypted_records']
+    assert len(decrypted) == 1
+    assert decrypted[0]['sequence'] == 0
+    assert decrypted[0]['plaintext'] == plaintext
+    assert output[1]['tls']['data']['cipher_suite'] == 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256'
+
+
+def _flow(direction, payload):
+    if direction == 'CTS':
+        src, sport, dst, dport = '10.0.0.2', 50505, '93.184.216.34', 443
+    else:
+        src, sport, dst, dport = '93.184.216.34', 443, '10.0.0.2', 50505
+    return {
+        'protocol': 'TCP',
+        'direction': direction,
+        'src': src,
+        'sport': sport,
+        'dst': dst,
+        'dport': dport,
+        'payload': payload,
+    }
+
+
+def _encrypted_record(plaintext, key, fixed_iv, sequence):
+    explicit_nonce = b'\x44' * 8
+    nonce = fixed_iv + explicit_nonce
+    aad = struct.pack('!Q B H H', sequence, 23, TLS_VERSION_12, len(plaintext))
+    cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+    cipher.update(aad)
+    ciphertext, tag = cipher.encrypt_and_digest(plaintext)
+    return _record(23, explicit_nonce + ciphertext + tag)
+
+
+def _record(content_type, fragment):
+    return struct.pack('!BHH', content_type, TLS_VERSION_12, len(fragment)) + fragment
+
+
+def _handshake(message_type, body):
+    return struct.pack('!B', message_type) + _uint24(len(body)) + body
+
+
+def _client_hello_body(client_random):
+    cipher_suites = struct.pack('!H', CIPHER_SUITE)
+    return (
+        struct.pack('!H', TLS_VERSION_12) +
+        client_random +
+        b'\x00' +
+        struct.pack('!H', len(cipher_suites)) +
+        cipher_suites +
+        b'\x01\x00'
+    )
+
+
+def _server_hello_body(server_random, cipher_suite):
+    return (
+        struct.pack('!H', TLS_VERSION_12) +
+        server_random +
+        b'\x00' +
+        struct.pack('!H', cipher_suite) +
+        b'\x00'
+    )
+
+
+def _uint24(value):
+    return struct.pack('!BBB', (value >> 16) & 0xff, (value >> 8) & 0xff, value & 0xff)