bin/djbdns-logparse.py: add comment about open file descriptors.

[djbdns-logparse.git] / bin / djbdns-logparse.py

diff --git a/bin/djbdns-logparse.py b/bin/djbdns-logparse.py
index 838012445c32027f80f1661929b991cbe28cd185..9af836262e78f1e485188a65a6854072d0de084b 100755 (executable)
--- a/bin/djbdns-logparse.py
+++ b/bin/djbdns-logparse.py
@@ -3,7 +3,7 @@
 Convert tinydns and dnscache logs to human-readable form
 """
 
-import re
+import re, typing
 from struct import pack
 from time import strftime, gmtime
 from subprocess import Popen, PIPE
@@ -101,12 +101,6 @@ def convert_ip(ip : str):
         return ":".join([ip[(4*i) : (4*i+4)] for i in range(8)])
 
 
-def _cvt_ip(match):
-    return convert_ip(match.group(1))
-
-def _cvt_port(match):
-    return ":" + str(int(match.group(1), 16))
-
 def decode_client(words, i):
     chunks = words[i].split(":")
     if len(chunks) == 2:                # ip:port
@@ -253,7 +247,17 @@ def handle_tinydns_log(line : str, match: re.Match):
                % (code, ip, port, id, type, name))
 
 
-def parse_logfile(file):
+def parse_logfile(file : typing.TextIO):
+    """
+    Process a single log ``file``.
+
+    Parameters
+    ----------
+
+    file : typing.TextIO
+        An open log file, or stdin.
+
+    """
     # Open pipe to tai64nlocal: we will write lines of our input (the
     # raw log file) to it, and read log lines with readable timestamps
     # from it.
@@ -291,12 +295,19 @@ def main():
                         default=[stdin],
                         help="djbdns logfile to process (default: stdin)")
 
+    # Warning: argparse automatically opens its file arguments here,
+    # and they only get closed when the program terminates. There's no
+    # real benefit to closing them one-at-a-time after calling
+    # parse_logfile(), because the "scarce" resource of open file
+    # descriptors gets consumed immediately, before any processing has
+    # happened. In other words, if you're going to run out of file
+    # descriptors, it's going to happen right now.
+    #
+    # So anyway, don't run this on several million logfiles.
     args = parser.parse_args()
     for f in args.logfiles:
         parse_logfile(f)
 
 
-
-
 if __name__ == "__main__":
     main()