bin/djbdns-logparse.py

   1 #!/usr/bin/python3
   2 """
   3 Convert tinydns and dnscache logs to human-readable form
   4 """
   5
   6 import re
   7 from struct import pack
   8 from time import strftime, gmtime
   9 from subprocess import Popen, PIPE
  10
  11
  12 ## Regular expressions for matching tinydns/dnscache log lines. We
  13 ## compile these once here rather than within the corresponding
  14 ## matching functions, because the latter get executed repeatedly.
  15
  16 # This first pattern is used to match the timestamp format that the
  17 # tai64nlocal program produces. It appears in both dnscache and
  18 # tinydns lines, after they've been piped through tai64nlocal, of
  19 # course.
  20 timestamp_pat = r'[\d-]+ [\d:\.]+'
  21
  22 # The regex to match dnscache log lines.
  23 dnscache_log_re = re.compile(fr'({timestamp_pat}) (\w+)(.*)')
  24
  25 # The "hex4" pattern matches a string of four hexadecimal digits. This
  26 # is used, for example, by tinydns to encode the query type
  27 # identifier.
  28 hex4_pat = r'[0-9a-f]{4}'
  29
  30 # The IP pattern matches a string of either 8 or 32 hexadecimal
  31 # characters, which correspond to IPv4 and IPv6 addresses,
  32 # respectively, in tinydns logs.
  33 ip_pat = r'[0-9a-f]{8,32}'
  34
  35 # The regex to match tinydns log lines.
  36 tinydns_log_re = re.compile(
  37     rf'({timestamp_pat}) ({ip_pat}):({hex4_pat}):({hex4_pat}) ([\+\-IC/]) ({hex4_pat}) (.*)'
  38 )
  39
  40 # A dictionary mapping query type identifiers, in decimal, to their
  41 # friendly names for tinydns. Reference:
  42 #
  43 #   https://en.wikipedia.org/wiki/List_of_DNS_record_types
  44 #
  45 # Note that mapping here is non-exhaustive, and that tinydns will
  46 # log responses for record types that it does not know about.
  47 query_type = {
  48       1: "a",
  49       2: "ns",
  50       5: "cname",
  51       6: "soa",
  52      12: "ptr",
  53      13: "hinfo",
  54      15: "mx",
  55      16: "txt",
  56      17: "rp",
  57      24: "sig",
  58      25: "key",
  59      28: "aaaa",
  60      33: "srv",
  61      35: "naptr",
  62      38: "a6",
  63      48: "dnskey",
  64      52: "tlsa",
  65      65: "https",
  66     252: "axfr",
  67     255: "any",
  68     257: "caa"
  69 }
  70
  71 # tinydns can drop a query for one of three reasons; this dictionary
  72 # maps the symbol that gets logged in each case to a human-readable
  73 # reason.
  74 query_drop_reason = {
  75     "-": "no authority",
  76     "I": "invalid query",
  77     "C": "invalid class",
  78 }
  79
  80
  81 def convert_ip(ip : str):
  82     """
  83     Convert a hex string representing an IP address to conventional
  84     human-readable form, ie. dotted-quad decimal for IPv4, and
  85     8 colon-separated hex shorts for IPv6.
  86
  87     Examples
  88     --------
  89
  90         >>> convert_ip("7f000001")
  91         '127.0.0.1'
  92         >>> convert_ip("00000000000000000000ffff7f000001")
  93         '0000:0000:0000:0000:0000:ffff:7f00:0001'
  94
  95     """
  96     if len(ip) == 8:
  97         # IPv4, eg. "7f000001" -> "7f 00 00 01" -> "127.0.0.1"
  98         return "%d.%d.%d.%d" % tuple(pack(">L", int(ip, 16)))
  99     elif len(ip) == 32:
 100         # IPv6 is actually simpler -- it's just a string-slicing operation.
 101         return ":".join([ip[(4*i) : (4*i+4)] for i in range(8)])
 102
 103
 104 def _cvt_ip(match):
 105     return convert_ip(match.group(1))
 106
 107 def _cvt_port(match):
 108     return ":" + str(int(match.group(1), 16))
 109
 110 def decode_client(words, i):
 111     chunks = words[i].split(":")
 112     if len(chunks) == 2:                # ip:port
 113         words[i] = "%s:%d" % (convert_ip(chunks[0]), int(chunks[1], 16))
 114     elif len(chunks) == 3:
 115         words[i] = "%s:%d (id %d)" % (convert_ip(chunks[0]),
 116                                       int(chunks[1], 16),
 117                                       int(chunks[2], 16))
 118
 119 def decode_ip(words, i):
 120     words[i] = convert_ip(words[i])
 121
 122 def decode_ttl(words, i):
 123     words[i] = "TTL=%s" % words[i]
 124
 125 def decode_serial(words, i):
 126     serial = int(words[i])
 127     words[i] = "#%d" % serial
 128
 129 def decode_type(words, i):
 130     qt = words[i]
 131     words[i] = query_type.get(int(qt), qt)
 132
 133 def handle_dnscache_log(line, match):
 134     (timestamp, event, data) = match.groups()
 135
 136     words = data.split()
 137     if event == "cached":
 138         if words[0] not in ("cname", "ns", "nxdomain"):
 139             decode_type(words, 0)
 140
 141     elif event == "drop":
 142         decode_serial(words, 0)
 143
 144     elif event == "lame":
 145         decode_ip(words, 0)
 146
 147     elif event == "nodata":
 148         decode_ip(words, 0)
 149         decode_ttl(words, 1)
 150         decode_type(words, 2)
 151
 152     elif event == "nxdomain":
 153         decode_ip(words, 0)
 154         decode_ttl(words, 1)
 155
 156     elif event == "query":
 157         decode_serial(words, 0)
 158         decode_client(words, 1)
 159         decode_type(words, 2)
 160
 161     elif event == "rr":
 162         decode_ip(words, 0)
 163         decode_ttl(words, 1)
 164         if words[2] not in ("cname", "mx", "ns", "ptr", "soa"):
 165             decode_type(words, 2)
 166             if words[2] == "a":         # decode answer to an A query
 167                 decode_ip(words, 4)
 168             if words[2] == "txt":       # text record
 169                 response = words[4]
 170                 if response.endswith("..."):
 171                     ellipsis = "..."
 172                     response = response[0:-3]
 173                 else:
 174                     ellipsis = ""
 175                 length = int(response[0:2], 16)
 176                 chars = []
 177                 for i in range(1, len(response)/2):
 178                     chars.append(chr(int(response[2*i : (2*i)+2], 16)))
 179                 words[4] = "%d:\"%s%s\"" % (length, "".join(chars), ellipsis)
 180
 181     elif event == "sent":
 182         decode_serial(words, 0)
 183
 184     elif event == "stats":
 185         words[0] = "count=%s" % words[0]
 186         words[1] = "motion=%s" % words[1]
 187         words[2] = "udp-active=%s" % words[2]
 188         words[3] = "tcp-active=%s" % words[3]
 189
 190     elif event == "tx":
 191         words[0] = "g=%s" % words[0]
 192         decode_type(words, 1)
 193         # words[2] = name
 194         # words[3] = control (domain for which these servers are believed
 195         #            to be authoritative)
 196         for i in range(4, len(words)):
 197             decode_ip(words, i)
 198
 199     elif event in ("tcpopen", "tcpclose"):
 200         decode_client(words, 0)
 201
 202     print(timestamp, event, " ".join(words))
 203
 204
 205 def handle_tinydns_log(line : str, match: re.Match):
 206     """
 207     Handle a line that matched the ``tinydns_log_re`` regex.
 208
 209     Parameters
 210     ----------
 211
 212     line : string
 213         The tinydns log line that matched ``tinydns_log_re``.
 214
 215     match : re.Match
 216         The match object that was returned when ``line`` was
 217         tested against ``tinydns_log_re``.
 218
 219     Examples
 220     --------
 221
 222         >>> line = "2022-09-14 21:04:40.206516500 7f000001:9d61:be69 - 0001 www.example.com"
 223         >>> match = tinydns_log_re.match(line)
 224         >>> handle_tinydns_log(line, match)
 225         2022-09-14 21:04:40.206516500 dropped query (no authority) from 127.0.0.1:40289 (id 48745): a www.example.com
 226
 227     """
 228     (timestamp, ip, port, id, code, type, name) = match.groups()
 229     ip = convert_ip(ip)
 230     port = int(port, 16)
 231     id = int(id, 16)
 232
 233     # Convert the "type" field to a human-readable record type name
 234     # using the query_type dictionary. If the right name isn't present
 235     # in the dictionary, we use the (decimal) type id instead.
 236     type = int(type, 16)                # "001c" -> 28
 237     type = query_type.get(type, type)   # 28 -> "aaaa"
 238
 239     print(timestamp, end=' ')
 240
 241     if code == "+":
 242         print ("sent response to %s:%s (id %s): %s %s"
 243                % (ip, port, id, type, name))
 244     elif code in ("-", "I", "C"):
 245         reason = query_drop_reason[code]
 246         print ("dropped query (%s) from %s:%s (id %s): %s %s"
 247                % (reason, ip, port, id, type, name))
 248     elif code == "/":
 249         print ("dropped query (couldn't parse) from %s:%s"
 250                % (ip, port))
 251     else:
 252         print ("%s from %s:%s (id %s): %s %s"
 253                % (code, ip, port, id, type, name))
 254
 255
 256 def parse_logfile(file):
 257     # Open pipe to tai64nlocal: we will write lines of our input (the
 258     # raw log file) to it, and read log lines with readable timestamps
 259     # from it.
 260     tai = Popen(["tai64nlocal"], stdin=PIPE, stdout=PIPE, text=True, bufsize=0)
 261
 262     for line in file:
 263         tai.stdin.write(line)
 264         line = tai.stdout.readline()
 265
 266         match = tinydns_log_re.match(line)
 267         if match:
 268             handle_tinydns_log(line, match)
 269             continue
 270
 271         match = dnscache_log_re.match(line)
 272         if match:
 273             handle_dnscache_log(line, match)
 274             continue
 275
 276         print(line)
 277
 278 def main():
 279     # Create an argument parser using the file's docsctring as its
 280     # description.
 281     from argparse import ArgumentParser, FileType
 282     parser = ArgumentParser(description = __doc__)
 283
 284     # Parse zero or more positional arguments into a list of
 285     # "logfiles". If none are given, read from stdin instead.
 286     from sys import stdin
 287     parser.add_argument("logfiles",
 288                         metavar="LOGFILE",
 289                         type=FileType("r"),
 290                         nargs="*",
 291                         default=[stdin],
 292                         help="djbdns logfile to process (default: stdin)")
 293
 294     args = parser.parse_args()
 295     for f in args.logfiles:
 296         parse_logfile(f)
 297
 298
 299
 300
 301 if __name__ == "__main__":
 302     main()