3 ##########################################################################
4 # Amavis-logwatch: written and maintained by:
6 # Mike "MrC" Cappella <mike (at) cappella (dot) us>
7 # http://logreporters.sourceforge.net/
9 # Please send all comments, suggestions, bug reports regarding this
10 # program/module to the email address above. I will respond as quickly
13 # Questions regarding the logwatch program itself should be directed to
14 # the logwatch project at:
15 # http://sourceforge.net/projects/logwatch/support
17 #######################################################
18 ### All work since Dec 12, 2006 (logwatch CVS revision 1.28)
19 ### Copyright (c) 2006-2012 Mike Cappella
21 ### Covered under the included MIT/X-Consortium License:
22 ### http://www.opensource.org/licenses/mit-license.php
23 ### All modifications and contributions by other persons to
24 ### this script are assumed to have been donated to the
25 ### Logwatch project and thus assume the above copyright
26 ### and licensing terms. If you want to make contributions
27 ### under your own copyright or a different license this
28 ### must be explicitly stated in the contribution an the
29 ### Logwatch project reserves the right to not accept such
30 ### contributions. If you have made significant
31 ### contributions to this script and want to claim
32 ### copyright please contact logwatch-devel@lists.sourceforge.net.
33 ##########################################################
35 ##########################################################################
36 # The original amavis logwatch filter was written by
37 # Jim O'Halloran <jim @ kendle.com.au>, and has had many contributors over
40 # CVS log removed: see Changes file for amavis-logwatch at
41 # http://logreporters.sourceforge.net/
42 # or included with the standalone amavis-logwatch distribution
43 ##########################################################################
49 no warnings
"uninitialized";
52 our $Version = '1.51.03';
53 our $progname_prefix = 'amavis';
55 # Specifies the default configuration file for use in standalone mode.
56 my $config_file = "/usr/local/etc/${progname_prefix}-logwatch.conf";
58 #MODULE: ../Logreporters/Utils.pm
59 package Logreporters
::Utils
;
68 use vars
qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION);
71 @EXPORT = qw(&formathost &get_percentiles &get_percentiles2 &get_frequencies &commify &unitize
72 &get_usable_sectvars &add_section &begin_section_group &end_section_group
73 &get_version &unique_list);
74 @EXPORT_OK = qw(&gen_test_log);
77 use subs qw
(@EXPORT @EXPORT_OK);
80 # Formats IP and hostname for even column spacing
86 if (! $Logreporters::Config
::Opts
{'unknown'} and $_[1] eq 'unknown') {
90 return sprintf "%-$Logreporters::Config::Opts{'ipaddr_width'}s %s",
91 $_[0] eq '' ? '*unknown' : $_[0],
92 $_[1] eq '' ? '*unknown' : lc $_[1];
95 # Add a new section to the end of a section table
97 sub add_section
($$$$$;$) {
99 die "Improperly specified Section entry: $_[0]" if !defined $_[3];
108 $entry->{'DIVISOR'} = $_[4] if defined $_[4];
115 # Begin a new section group. Groups can nest.
117 sub begin_section_group
($;@) {
119 my $group_name = shift;
121 CLASS
=> 'GROUP_BEGIN',
123 LEVEL
=> ++$group_level,
129 # Ends a section group.
131 sub end_section_group
($;@) {
133 my $group_name = shift;
135 CLASS
=> 'GROUP_END',
137 LEVEL
=> --$group_level,
144 # Generate and return a list of section table entries or
145 # limiter key names, skipping any formatting entries.
146 # If 'namesonly' is set, limiter key names are returned,
147 # otherwise an array of section array records is returned.
148 sub get_usable_sectvars
(\
@ $) {
149 my ($sectref,$namesonly) = @_;
150 my (@sect_list, %unique_names);
152 foreach my $sref (@$sectref) {
153 #print "get_usable_sectvars: $sref->{NAME}\n";
154 next unless $sref->{CLASS
} eq 'DATA';
156 $unique_names{$sref->{NAME
}} = 1;
159 push @sect_list, $sref;
162 # return list of unique names
164 return keys %unique_names;
169 # Print program and version info, preceeded by an optional string, and exit.
173 print STDOUT
"@_\n" if ($_[0]);
174 print STDOUT
"$Logreporters::progname: $Logreporters::Version\n";
179 # Returns a list of percentile values given a
180 # sorted array of numeric values. Uses the formula:
182 # r = 1 + (p(n-1)/100) = i + d (Excel method)
185 # p = desired percentile
186 # n = number of items
190 # Arg1 is an array ref to the sorted series
191 # Arg2 is a list of percentiles to use
193 sub get_percentiles
(\
@ @) {
194 my ($aref,@plist) = @_;
195 my ($n, $last, $r, $d, $i, @vals, $Yp);
199 #printf "%6d" x $n . "\n", @{$aref};
201 #printf "n: %4d, last: %d\n", $n, $last;
202 foreach my $p (@plist) {
203 $r = 1 + ($p * ($n - 1) / 100.0);
204 $i = int ($r); # integer part
205 # domain: $i = 1 .. n
207 $Yp = $aref->[$last];
211 print "CAN'T HAPPEN: $Yp\n";
214 $d = $r - $i; # decimal part
215 #p = Y[i] + d(Y[i+1] - Y[i]), but since we're 0 based, use i=i-1
216 $Yp = $aref->[$i-1] + ($d * ($aref->[$i] - $aref->[$i-1]));
218 #printf "\np(%6.2f), r: %6.2f, i: %6d, d: %6.2f, Yp: %6d", $p, $r, $i, $d, $Yp;
225 sub get_num_scores
($) {
226 my $scoretab_r = shift;
230 for (my $i = 0; $i < @$scoretab_r; $i += 2) {
231 $totalscores += $scoretab_r->[$i+1]
239 # (score1, n1), (score2, n2), ... (scoreN, nN)
242 # scores are 0 based (0 = 1st score)
243 sub get_nth_score
($ $) {
244 my ($scoretab_r, $n) = @_;
247 my $n_cur_scores = 0;
248 #print "Byscore (", .5 * @$scoretab_r, "): "; for (my $i = 0; $i < $#$scoretab_r / 2; $i++) { printf "%9s (%d) ", $scoretab_r->[$i], $scoretab_r->[$i+1]; } ; print "\n";
250 while ($i < $#$scoretab_r) {
251 #print "Samples_seen: $n_cur_scores\n";
252 $n_cur_scores += $scoretab_r->[$i+1];
253 if ($n_cur_scores >= $n) {
254 #printf "range: %s %s %s\n", $i >= 2 ? $scoretab_r->[$i - 2] : '<begin>', $scoretab_r->[$i], $i+2 > $#$scoretab_r ? '<end>' : $scoretab_r->[$i + 2];
255 #printf "n: $n, i: %8d, n_cur_scores: %8d, score: %d x %d hits\n", $i, $n_cur_scores, $scoretab_r->[$i], $scoretab_r->[$i+1];
256 return $scoretab_r->[$i];
261 print "returning last score $scoretab_r->[$i]\n";
262 return $scoretab_r->[$i];
265 sub get_percentiles2
(\
@ @) {
266 my ($scoretab_r, @plist) = @_;
267 my ($n, $last, $r, $d, $i, @vals, $Yp);
269 #$last = $#$scoretab_r - 1;
270 $n = get_num_scores
($scoretab_r);
271 #printf "\n%6d" x $n . "\n", @{$scoretab_r};
273 #printf "\n\tn: %4d, @$scoretab_r\n", $n;
274 foreach my $p (@plist) {
275 ###print "\nPERCENTILE: $p\n";
276 $r = 1 + ($p * ($n - 1) / 100.0);
277 $i = int ($r); # integer part
280 #$Yp = $scoretab_r->[$last];
281 $Yp = get_nth_score
($scoretab_r, $n);
284 #$Yp = $scoretab_r->[0];
285 print "1st: CAN'T HAPPEN\n";
286 $Yp = get_nth_score
($scoretab_r, 1);
289 $d = $r - $i; # decimal part
290 #p = Y[i] + d(Y[i+1] - Y[i]), but since we're 0 based, use i=i-1
291 my $ithvalprev = get_nth_score
($scoretab_r, $i);
292 my $ithval = get_nth_score
($scoretab_r, $i+1);
293 $Yp = $ithvalprev + ($d * ($ithval - $ithvalprev));
295 #printf "p(%6.2f), r: %6.2f, i: %6d, d: %6.2f, Yp: %6d\n", $p, $r, $i, $d, $Yp;
304 # Returns a list of frequency distributions given an incrementally sorted
305 # set of sorted scores, and an incrementally sorted list of buckets
307 # Arg1 is an array ref to the sorted series
308 # Arg2 is a list of frequency buckets to use
309 sub get_frequencies
(\
@ @) {
310 my ($aref,@blist) = @_;
312 my @vals = ( 0 ) x
(@blist);
313 my @sorted_blist = sort { $a <=> $b } @blist;
314 my $bucket_index = 0;
316 OUTER
: foreach my $score (@$aref) {
317 #print "Score: $score\n";
318 for my $i ($bucket_index .. @sorted_blist - 1) {
319 #print "\tTrying Bucket[$i]: $sorted_blist[$i]\n";
320 if ($score > $sorted_blist[$i]) {
324 #printf "\t\tinto Bucket[%d]\n", $bucket_index;
325 $vals[$bucket_index]++;
329 #printf "\t\tinto Bucket[%d]\n", $bucket_index - 1;
330 $vals[$bucket_index - 1]++;
336 # Inserts commas in numbers for easier readability
339 return undef if ! defined ($_[0]);
341 my $text = reverse $_[0];
342 $text =~ s/(\d\d\d)(?=\d)(?!\d*\.)/$1,/g;
343 return scalar reverse $text;
346 # Unitize a number, and return appropriate printf formatting string
349 my ($num, $fmt) = @_;
350 my $kilobyte = 2**10;
351 my $megabyte = 2**20;
352 my $gigabyte = 2**30;
353 my $terabyte = 2**40;
355 if ($num >= $terabyte) {
358 } elsif ($num >= $gigabyte) {
361 } elsif ($num >= $megabyte) {
364 } elsif ($num >= $kilobyte) {
374 # Returns a sublist of the supplied list of elements in an unchanged order,
375 # where only the first occurrence of each defined element is retained
376 # and duplicates removed
378 # Borrowed from amavis 2.6.2
381 my ($r) = @_ == 1 && ref($_[0]) ? $_[0] : \
@_; # accept list, or a list ref
383 my (@unique) = grep { defined($_) && !$seen{$_}++ } @$r;
388 # Generate a test maillog file from the '#TD' test data lines
389 # The test data file is placed in /var/tmp/maillog.autogen
391 # arg1: "postfix" or "amavis"
392 # arg2: path to postfix-logwatch or amavis-logwatch from which to read '#TD' data
395 # TD<service><QID>(<count>) log entry
397 sub gen_test_log
($) {
398 my $scriptpath = shift;
400 my $toolname = $Logreporters::progname_prefix
;
401 my $datafile = "/var/tmp/maillog-${toolname}.autogen";
403 die "gen_test_log: invalid toolname $toolname" if ($toolname !~ /^(postfix|amavis)$/);
406 require Sys
::Hostname
;
408 } or die "Unable to create test data file: required module(s) not found\n$@";
410 my $syslogtime = localtime;
411 $syslogtime =~ s/^....(.*) \d{4}$/$1/;
413 my ($hostname) = split /\./, Sys
::Hostname
::hostname
();
416 # delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
418 my $flags = &Fcntl
::O_CREAT
|&Fcntl
::O_WRONLY
|&Fcntl
::O_TRUNC
;
419 sysopen(FH
, $datafile, $flags) or die "Can't create test data file: $!";
420 print "Generating test log data file from $scriptpath: $datafile\n";
423 @ARGV = ($scriptpath);
424 if ($toolname eq 'postfix') {
449 $id = 'postfix/smtp[12345]';
452 if (/^\s*#TD([a-zA-Z]*[NQ]?)(\d+)?(?:\(([^)]+)\))? (.*)$/) {
453 my ($service,$count,$qid,$line) = ($1, $2, $3, $4);
455 #print "SERVICE: %s, QID: %s, COUNT: %s, line: %s\n", $service, $qid, $count, $line;
457 if ($service eq '') {
460 die ("No such service: \"$service\": line \"$_\"") if (!exists $services{$service});
462 $id = $services{$service} . '[123]';
463 $id = 'postfix/' . $id unless $services{$service} eq 'postgrey';
464 #print "searching for service: \"$service\"\n\tFound $id\n";
465 if ($service =~ /N$/) { $id .= ': NOQUEUE'; }
466 elsif ($service =~ /Q$/) { $id .= $qid ? $qid : ': DEADBEEF'; }
470 #print "$syslogtime $hostname $id: \"$line\"\n" x ($count ? $count : 1);
471 print FH
"$syslogtime $hostname $id: $line\n" x
($count ? $count : 1);
481 if (/^\s*#TD([a-z]*)(\d+)? (.*)$/) {
482 my ($service,$count,$line) = ($1, $2, $3);
483 if ($service eq '') {
486 die ("No such service: \"$service\": line \"$_\"") if (!exists $services{$service});
487 $id = $services{$service} . '[123]:';
488 if ($services{$service} eq 'amavis') {
491 print FH
"$syslogtime $hostname $id $line\n" x
($count ? $count : 1)
496 close FH
or die "Can't close $datafile: $!";
501 #MODULE: ../Logreporters/Config.pm
502 package Logreporters
::Config
;
512 use vars
qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION);
515 @EXPORT = qw(&init_run_mode &add_option &get_options &init_cmdline &get_vars_from_file
516 &process_limiters &process_debug_opts &init_getopts_table_common &zero_opts
517 @Optspec %Opts %Configvars @Limiters %line_styles $fw1 $fw2 $sep1 $sep2
518 &D_CONFIG &D_ARGS &D_VARS &D_TREE &D_SECT &D_UNMATCHED &D_TEST &D_ALL
524 our @Optspec = (); # options table used by Getopts
526 our %Opts = (); # program-wide options
527 our %Configvars = (); # configuration file variables
530 # Report separator characters and widths
531 our ($fw1,$fw2) = (22, 10);
532 our ($sep1,$sep2) = ('=', '-');
538 import Logreporters
::Utils
qw(&get_usable_sectvars);
547 sub init_run_mode
($);
548 sub confighash_to_cmdline
(\
%);
549 sub get_vars_from_file
(\
% $);
550 sub process_limiters
(\
@);
553 sub init_getopts_table_common
(@);
554 sub set_supplemental_reports
($$);
556 sub D_CONFIG
() { 1<<0 }
557 sub D_ARGS () { 1<<1 }
558 sub D_VARS () { 1<<2 }
559 sub D_TREE () { 1<<3 }
560 sub D_SECT () { 1<<4 }
561 sub D_UNMATCHED () { 1<<5 }
563 sub D_TEST () { 1<<30 }
564 sub D_ALL () { 1<<31 }
572 unmatched => D_UNMATCHED,
578 # Clears %Opts hash and initializes basic running mode options in
579 # %Opts hash by setting keys: 'standalone', 'detail', and 'debug'.
582 sub init_run_mode($) {
583 my $config_file = shift;
586 # Logwatch passes a filter's options via environment variables.
587 # When running standalone (w/out logwatch), use command line options
588 $Opts{'standalone'} = exists ($ENV{LOGWATCH_DETAIL_LEVEL}) ? 0 : 1;
590 # Show summary section by default
591 $Opts{'summary'} = 1;
593 if ($Opts{'standalone'}) {
594 process_debug_opts($ENV{'LOGREPORTERS_DEBUG'}) if exists ($ENV{'LOGREPORTERS_DEBUG'});
597 $Opts{'detail'} = $ENV{'LOGWATCH_DETAIL_LEVEL'};
599 #process_debug_opts($ENV{'LOGWATCH_DEBUG'}) if exists ($ENV{'LOGWATCH_DEBUG'});
602 # first process --debug, --help, and --version options
603 add_option ('debug=s', sub { process_debug_opts($_[1]); 1});
604 add_option ('version', sub { &Logreporters::Utils::get_version(); 1;});
607 # now process --config_file, so that all config file vars are read first
608 add_option ('config_file|f=s', sub { get_vars_from_file(%Configvars, $_[1]); 1;});
611 # if no config file vars were read
612 if ($Opts{'standalone'} and ! keys(%Configvars) and -f $config_file) {
613 print "Using default config file: $config_file\n" if $Opts{'debug'} & D_CONFIG;
614 get_vars_from_file(%Configvars, $config_file);
619 my $pass_through = shift;
620 #$SIG{__WARN__} = sub { print "*** $_[0]*** options error\n" };
621 # ensure we're called after %Opts is initialized
622 die "get_options: program error: %Opts is emtpy" unless exists $Opts{'debug'};
624 my $p = new Getopt::Long::Parser;
627 $p->configure(qw(pass_through permute));
630 $p->configure(qw(no_pass_through no_permute));
632 #$p->configure(qw(debug));
634 if ($Opts{'debug'} & D_ARGS
) {
635 print "\nget_options($pass_through): enter\n";
636 printf "\tARGV(%d): ", scalar @ARGV;
638 print "\t$_ ", defined $Opts{$_} ? "=> $Opts{$_}\n" : "\n" foreach sort keys %Opts;
641 if ($p->getoptions(\
%Opts, @Optspec) == 0) {
642 print STDERR
"Use ${Logreporters::progname} --help for options\n";
645 if ($Opts{'debug'} & D_ARGS
) {
646 print "\t$_ ", defined $Opts{$_} ? "=> $Opts{$_}\n" : "\n" foreach sort keys %Opts;
647 printf "\tARGV(%d): ", scalar @ARGV;
649 print "get_options: exit\n";
657 # untaint string, borrowed from amavisd-new
662 if (defined($_[0])) {
663 local($1); # avoid Perl taint bug: tainted global $1 propagates taintedness
664 $str = $1 if $_[0] =~ /^(.*)$/;
670 sub init_getopts_table_common
(@) {
671 my @supplemental_reports = @_;
673 print "init_getopts_table_common: enter\n" if $Opts{'debug'} & D_ARGS
;
675 add_option
('help', sub { print STDOUT Logreporters
::usage
(undef); exit 0 });
676 add_option
('gen_test_log=s', sub { Logreporters
::Utils
::gen_test_log
($_[1]); exit 0; });
677 add_option
('detail=i');
678 add_option
('nodetail', sub {
679 # __none__ will set all limiters to 0 in process_limiters
680 # since they are not known (Sections table is not yet built).
681 push @Limiters, '__none__';
682 # 0 = disable supplemental_reports
683 set_supplemental_reports
(0, \
@supplemental_reports);
685 add_option
('max_report_width=i');
686 add_option
('summary!');
687 add_option
('show_summary=i', sub { $Opts{'summary'} = $_[1]; 1; });
688 # untaint ipaddr_width for use w/sprintf() in Perl v5.10
689 add_option
('ipaddr_width=i', sub { $Opts{'ipaddr_width'} = untaint
($_[1]); 1; });
691 add_option
('sect_vars!');
692 add_option
('show_sect_vars=i', sub { $Opts{'sect_vars'} = $_[1]; 1; });
694 add_option
('syslog_name=s');
695 add_option
('wrap', sub { $Opts{'line_style'} = $line_styles{$_[0]}; 1; });
696 add_option
('full', sub { $Opts{'line_style'} = $line_styles{$_[0]}; 1; });
697 add_option
('truncate', sub { $Opts{'line_style'} = $line_styles{$_[0]}; 1; });
698 add_option
('line_style=s', sub {
699 my $style = lc($_[1]);
700 my @list = grep (/^$style/, keys %line_styles);
702 print STDERR
"Invalid line_style argument \"$_[1]\"\n";
703 print STDERR
"Option line_style argument must be one of \"wrap\", \"full\", or \"truncate\".\n";
704 print STDERR
"Use $Logreporters::progname --help for options\n";
707 $Opts{'line_style'} = $line_styles{lc($list[0])};
711 add_option
('limit|l=s', sub {
712 my ($limiter,$lspec) = split(/=/, $_[1]);
713 if (!defined $lspec) {
714 printf STDERR
"Limiter \"%s\" requires value (ex. --limit %s=10)\n", $_[1],$_[1];
717 foreach my $val (split(/(?:\s+|\s*,\s*)/, $lspec)) {
718 if ($val !~ /^\d+$/ and
719 $val !~ /^(\d*)\.(\d+)$/ and
720 $val !~ /^::(\d+)$/ and
721 $val !~ /^:(\d+):(\d+)?$/ and
722 $val !~ /^(\d+):(\d+)?:(\d+)?$/)
724 printf STDERR
"Limiter value \"$val\" invalid in \"$limiter=$lspec\"\n";
728 push @Limiters, lc $_[1];
731 print "init_getopts_table_common: exit\n" if $Opts{'debug'} & D_ARGS
;
734 sub get_option_names
() {
737 if (ref($_) eq '') { # process only the option names
740 $spec =~ s/([^|]+)\!$/$1|no$1/g;
741 @tmp = split /[|]/, $spec;
742 #print "PUSHING: @tmp\n";
749 # Set values for the configuration variables passed via hashref.
750 # Variables are of the form ${progname_prefix}_KEYNAME.
752 # Because logwatch lowercases all config file entries, KEYNAME is
756 my ($href, $configvar, $value, $var);
758 # logwatch passes all config vars via environment variables
759 $href = $Opts{'standalone'} ? \
%Configvars : \
%ENV;
761 # XXX: this is cheeze: need a list of valid limiters, but since
762 # the Sections table is not built yet, we don't know what is
763 # a limiter and what is an option, as there is no distinction in
764 # variable names in the config file (perhaps this should be changed).
765 my @valid_option_names = get_option_names
();
766 die "Options table not yet set" if ! scalar @valid_option_names;
768 print "confighash_to_cmdline: @valid_option_names\n" if $Opts{'debug'} & D_ARGS
;
770 while (($configvar, $value) = each %$href) {
771 if ($configvar =~ s/^${Logreporters::progname_prefix}_//o) {
772 # distinguish level limiters from general options
773 # would be easier if limiters had a unique prefix
774 $configvar = lc $configvar;
775 my $ret = grep (/^$configvar$/i, @valid_option_names);
777 print "\tLIMITER($ret): $configvar = $value\n" if $Opts{'debug'} & D_ARGS
;
778 push @cmdline, '-l', "$configvar" . "=$value";
781 print "\tOPTION($ret): $configvar = $value\n" if $Opts{'debug'} & D_ARGS
;
782 unshift @cmdline, $value if defined ($value);
783 unshift @cmdline, "--$configvar";
787 unshift @ARGV, @cmdline;
790 # Obtains the variables from a logwatch-style .conf file, for use
791 # in standalone mode. Returns an ENV-style hash of key/value pairs.
793 sub get_vars_from_file
(\
% $) {
794 my ($href, $file) = @_;
797 print "get_vars_from_file: enter: processing file: $file\n" if $Opts{'debug'} & D_CONFIG
;
800 my $ret = stat ($file);
801 if ($ret == 0) { $message = $!; }
802 elsif (! -r _
) { $message = "Permission denied"; }
803 elsif ( -d _
) { $message = "Is a directory"; }
804 elsif (! -f _
) { $message = "Not a regular file"; }
807 print STDERR
"Configuration file \"$file\": $message\n";
811 my $prog = $Logreporters::progname_prefix
;
812 open FILE
, '<', "$file" or die "unable to open configuration file $file: $!";
815 next if (/^\s*$/); # ignore all whitespace lines
816 next if (/^\*/); # ignore logwatch's *Service lines
817 next if (/^\s*#/); # ignore comment lines
818 if (/^\s*\$(${prog}_[^=\s]+)\s*=\s*"?([^"]+)"?$/o) {
819 ($var,$val) = ($1,$2);
820 if ($val =~ /^(?:no|false)$/i) { $val = 0; }
821 elsif ($val =~ /^(?:yes|true)$/i) { $val = 1; }
822 elsif ($val eq '') { $var =~ s/${prog}_/${prog}_no/; $val = undef; }
824 print "\t\"$var\" => \"$val\"\n" if $Opts{'debug'} & D_CONFIG
;
826 $href->{$var} = $val;
829 close FILE
or die "failed to close configuration handle for $file: $!";
830 print "get_vars_from_file: exit\n" if $Opts{'debug'} & D_CONFIG
;
833 sub process_limiters
(\
@) {
836 my ($limiter, $var, $val, @errors);
837 my @l = get_usable_sectvars
(@$sectref, 1);
839 if ($Opts{'debug'} & D_VARS
) {
840 print "process_limiters: enter\n";
841 print "\tLIMITERS: @Limiters\n";
843 while ($limiter = shift @Limiters) {
846 printf "\t%-30s ",$limiter if $Opts{'debug'} & D_VARS
;
847 # disable all limiters when limiter is __none__: see 'nodetail' cmdline option
848 if ($limiter eq '__none__') {
849 $Opts{$_} = 0 foreach @l;
853 ($var,$val) = split /=/, $limiter;
856 push @errors, "Limiter \"$var\" requires value (ex. --limit limiter=10)";
860 # try exact match first, then abbreviated match next
861 if (scalar (@matched = grep(/^$var$/, @l)) == 1 or scalar (@matched = grep(/^$var/, @l)) == 1) {
862 $limiter = $matched[0]; # unabbreviate limiter
863 print "MATCH: $var: $limiter => $val\n" if $Opts{'debug'} & D_VARS
;
864 # XXX move limiters into section hash entry...
865 $Opts{$limiter} = $val;
868 print "matched=", scalar @matched, ": @matched\n" if $Opts{'debug'} & D_VARS
;
870 push @errors, "Limiter \"$var\" is " . (scalar @matched == 0 ? "invalid" : "ambiguous: @matched");
872 print "\n" if $Opts{'debug'} & D_VARS
;
875 print STDERR
"$_\n" foreach @errors;
879 # Set the default value of 10 for each section if no limiter exists.
880 # This allows output for each section should there be no configuration
881 # file or missing limiter within the configuration file.
883 $Opts{$_} = 10 unless exists $Opts{$_};
886 # Enable collection for each section if a limiter is non-zero.
889 #print "DETAIL: $Opts{'detail'}, OPTS: $Opts{$_}\n";
890 $Logreporters::TreeData
::Collecting
{$_} = (($Opts{'detail'} >= 5) && $Opts{$_}) ? 1 : 0;
892 #print "OPTS: \n"; map { print "$_ => $Opts{$_}\n"} keys %Opts;
893 #print "COLLECTING: \n"; map { print "$_ => $Logreporters::TreeData::Collecting{$_}\n"} keys %Logreporters::TreeData::Collecting;
896 # Enable/disable supplemental reports
898 # arg2,...: list of supplemental report keywords
899 sub set_supplemental_reports
($$) {
900 my ($onoff,$aref) = @_;
902 $Opts{$_} = $onoff foreach (@$aref);
905 sub process_debug_opts
($) {
906 my $optstring = shift;
909 foreach (split(/\s*,\s*/, $optstring)) {
911 my @matched = grep (/^$word/, keys %debug_words);
913 if (scalar @matched == 1) {
914 $Opts{'debug'} |= $debug_words{$matched[0]};
918 if (scalar @matched == 0) {
919 push @errors, "Unknown debug keyword \"$word\"";
922 push @errors, "Ambiguous debug keyword abbreviation \"$word\": (matches: @matched)";
926 print STDERR
"$_\n" foreach @errors;
927 print STDERR
"Debug keywords: ", join (' ', sort keys %debug_words), "\n";
932 # Zero the options controlling level specs and those
933 # any others passed via Opts key.
935 # Zero the options controlling level specs in the
936 # Detailed section, and set all other report options
937 # to disabled. This makes it easy via command line to
938 # disable the entire summary section, and then re-enable
939 # one or more sections for specific reports.
941 # eg. progname --nodetail --limit forwarded=2
943 sub zero_opts
($ @) {
945 # remaining args: list of Opts keys to zero
947 map { $Opts{$_} = 0; print "zero_opts: $_ => 0\n" if $Opts{'debug'} & D_VARS
;} @_;
948 map { $Opts{$_} = 0 } get_usable_sectvars
(@$sectref, 1);
953 #MODULE: ../Logreporters/TreeData.pm
954 package Logreporters
::TreeData
;
960 no warnings
"uninitialized";
964 use vars
qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION);
967 @EXPORT = qw(%Totals %Counts %Collecting $END_KEY);
968 @EXPORT_OK = qw(&printTree &buildTree);
975 import Logreporters
::Config
qw(%line_styles);
978 # Totals and Counts are the log line accumulator hashes.
979 # Totals: maintains per-section grand total tallies for use in Summary section
980 # Counts: is a multi-level hash, which maintains per-level key totals.
981 our (%Totals, %Counts);
983 # The Collecting hash determines which sections will be captured in
984 # the Counts hash. Counts are collected only if a section is enabled,
985 # and this hash obviates the need to test both existence and
986 # non-zero-ness of the Opts{'keyname'} (either of which cause capture).
987 # XXX The Opts hash could be used ....
988 our %Collecting = ();
990 sub buildTree
(\
% $ $ $ $ $);
991 sub printTree
($ $ $ $ $);
995 which would be interpreted as follows:
997 a = show level a detail
998 b = show at most b items at this level
999 c = minimun count that will be shown
1002 sub printTree
($ $ $ $ $) {
1003 my ($treeref, $lspecsref, $line_style, $max_report_width, $debug) = @_;
1005 my $cutlength = $max_report_width - 3;
1008 foreach $entry (sort bycount
@$treeref) {
1009 ref($entry) ne "HASH" and die "Unexpected entry in tree: $entry\n";
1011 #print "LEVEL: $entry->{LEVEL}, TOTAL: $entry->{TOTAL}, HASH: $entry, DATA: $entry->{DATA}\n";
1013 # Once the top N lines have been printed, we're done
1014 if ($lspecsref->[$entry->{LEVEL
}]{topn
}) {
1015 if ($topn++ >= $lspecsref->[$entry->{LEVEL
}]{topn
} ) {
1016 print ' ', ' ' x
($entry->{LEVEL
} + 3), "...\n"
1017 unless ($debug) and do {
1018 $line = ' ' . ' ' x
($entry->{LEVEL
} + 3) . '...';
1019 printf "%-130s L%d: topn reached(%d)\n", $line, $entry->{LEVEL
} + 1, $lspecsref->[$entry->{LEVEL
}]{topn
};
1025 # Once the item's count falls below the given threshold, we're done at this level
1026 # unless a top N is specified, as threshold has lower priority than top N
1027 elsif ($lspecsref->[$entry->{LEVEL
}]{threshold
}) {
1028 if ($entry->{TOTAL
} <= $lspecsref->[$entry->{LEVEL
}]{threshold
}) {
1029 print ' ', ' ' x
($entry->{LEVEL
} + 3), "...\n"
1030 unless ($debug) and do {
1031 $line = ' ' . (' ' x
($entry->{LEVEL
} + 3)) . '...';
1032 printf "%-130s L%d: threshold reached(%d)\n", $line, $entry->{LEVEL
} + 1, $lspecsref->[$entry->{LEVEL
}]{threshold
};
1038 $line = sprintf "%8d%s%s", $entry->{TOTAL
}, ' ' x
($entry->{LEVEL
} + 2), $entry->{DATA
};
1041 printf "%-130s %-60s\n", $line, $entry->{DEBUG
};
1044 # line_style full, or lines < max_report_width
1046 #printf "MAX: $max_report_width, LEN: %d, CUTLEN $cutlength\n", length($line);
1047 if ($line_style == $line_styles{'full'} or length($line) <= $max_report_width) {
1050 elsif ($line_style == $line_styles{'truncate'}) {
1051 print substr ($line,0,$cutlength), '...', "\n";
1053 elsif ($line_style == $line_styles{'wrap'}) {
1054 my $leader = ' ' x
8 . ' ' x
($entry->{LEVEL
} + 2);
1055 print substr ($line, 0, $max_report_width, ''), "\n";
1056 while (length($line)) {
1057 print $leader, substr ($line, 0, $max_report_width - length($leader), ''), "\n";
1061 die ('unexpected line style');
1064 printTree
($entry->{CHILDREF
}, $lspecsref, $line_style, $max_report_width, $debug) if (exists $entry->{CHILDREF
});
1068 my $re_IP_strict = qr/\b(25[0-5]|2[0-4]\d|[01]?\d{1,2})\.(25[0-5]|2[0-4]\d|[01]?\d{1,2})\.(25[0-5]|2[0-4]\d|[01]?\d{1,2})\.(25[0-5]|2[0-4]\d|[01]?\d{1,2})\b/;
1069 # XXX optimize this using packed default sorting. Analysis shows speed isn't an issue though
1071 # Sort by totals, then IP address if one exists, and finally by data as a string
1073 local $SIG{__WARN__
} = sub { print "*** PLEASE REPORT:\n*** $_[0]*** Unexpected: \"$a->{DATA}\", \"$b->{DATA}\"\n" };
1075 $b->{TOTAL
} <=> $a->{TOTAL
}
1079 pack('C4' => $a->{DATA
} =~ /^$re_IP_strict/o) cmp pack('C4' => $b->{DATA
} =~ /^$re_IP_strict/o)
1083 $a->{DATA
} cmp $b->{DATA
}
1087 # Builds a tree of REC structures from the multi-key %Counts hashes
1090 # Hash: A multi-key hash, with keys being used as category headings, and leaf data
1091 # being tallies for that set of keys
1092 # Level: This current recursion level. Call with 0.
1095 # Listref: A listref, where each item in the list is a rec record, described as:
1096 # DATA: a string: a heading, or log data
1097 # TOTAL: an integer: which is the subtotal of this item's children
1098 # LEVEL: an integer > 0: representing this entry's level in the tree
1099 # CHILDREF: a listref: references a list consisting of this node's children
1100 # Total: The cummulative total of items found for a given invocation
1102 # Use the special key variable $END_KEY, which is "\a\a" (two ASCII bell's) to end a,
1103 # nested hash early, or the empty string '' may be used as the last key.
1105 our $END_KEY = "\a\a";
1107 sub buildTree
(\
% $ $ $ $ $) {
1108 my ($href, $max_level_section, $levspecref, $max_level_global, $recurs_level, $show_unique, $debug) = @_;
1109 my ($subtotal, $childList, $rec);
1114 foreach my $item (sort keys %$href) {
1115 if (ref($href->{$item}) eq "HASH") {
1116 #print " " x ($recurs_level * 4), "HASH: LEVEL $recurs_level: Item: $item, type: \"", ref($href->{$item}), "\"\n";
1118 ($subtotal, $childList) = buildTree
(%{$href->{$item}}, $max_level_section, $levspecref, $max_level_global, $recurs_level + 1, $debug);
1120 if ($recurs_level < $max_level_global and $recurs_level < $max_level_section) {
1125 LEVEL
=> $recurs_level,
1126 CHILDREF
=> $childList,
1130 $rec->{DEBUG
} = sprintf "L%d: levelspecs: %2d/%2d/%2d/%2d, Count: %10d",
1131 $recurs_level + 1, $max_level_global, $max_level_section,
1132 $levspecref->[$recurs_level]{topn
}, $levspecref->[$recurs_level]{threshold
}, $subtotal;
1134 push (@treeList, $rec);
1138 if ($item ne '' and $item ne $END_KEY and $recurs_level < $max_level_global and $recurs_level < $max_level_section) {
1141 TOTAL
=> $href->{$item},
1142 LEVEL
=> $recurs_level,
1146 $rec->{DEBUG
} = sprintf "L%d: levelspecs: %2d/%2d/%2d/%2d, Count: %10d",
1147 $recurs_level, $max_level_global, $max_level_section,
1148 $levspecref->[$recurs_level]{topn
}, $levspecref->[$recurs_level]{threshold
}, $href->{$item};
1150 push (@treeList, $rec);
1152 $subtotal = $href->{$item};
1155 $total += $subtotal;
1158 #print " " x ($recurs_level * 4), "LEVEL $recurs_level: Returning from recurs_level $recurs_level\n";
1160 return ($total, \
@treeList);
1165 #MODULE: ../Logreporters/Reports.pm
1166 package Logreporters
::Reports
;
1172 no warnings
"uninitialized";
1176 use vars
qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION);
1178 @ISA = qw(Exporter);
1179 @EXPORT = qw(&inc_unmatched &print_unmatched_report &print_percentiles_report2
1180 &print_summary_report &print_detail_report);
1184 use subs
@EXPORT_OK;
1187 import Logreporters
::Config
qw(%Opts $fw1 $fw2 $sep1 $sep2 &D_UNMATCHED &D_TREE);
1188 import Logreporters
::Utils
qw(&commify &unitize &get_percentiles &get_percentiles2);
1189 import Logreporters
::TreeData
qw(%Totals %Counts &buildTree &printTree);
1192 my (%unmatched_list);
1194 our $origline; # unmodified log line, for error reporting and debug
1196 sub inc_unmatched
($) {
1198 $unmatched_list{$origline}++;
1199 print "UNMATCHED($id): \"$origline\"\n" if $Opts{'debug'} & D_UNMATCHED
;
1202 # Print unmatched lines
1204 sub print_unmatched_report
() {
1205 return unless (keys %unmatched_list);
1207 print "\n\n**Unmatched Entries**\n";
1208 foreach my $line (sort {$unmatched_list{$b}<=>$unmatched_list{$a} } keys %unmatched_list) {
1209 printf "%8d %s\n", $unmatched_list{$line}, $line;
1214 ****** Summary ********************************************************
1215 2 Miscellaneous warnings
1217 20621 Total messages scanned ---------------- 100.00%
1218 662.993M Total bytes scanned 695,198,092
1219 ======== ================================================
1221 19664 Ham ----------------------------------- 95.36%
1222 19630 Clean passed 95.19%
1223 34 Bad header passed 0.16%
1225 942 Spam ---------------------------------- 4.57%
1226 514 Spam blocked 2.49%
1227 428 Spam discarded (no quarantine) 2.08%
1229 15 Malware ------------------------------- 0.07%
1230 15 Malware blocked 0.07%
1233 1978 SpamAssassin bypassed
1234 18 Released from quarantine
1238 51 Bad header (debug supplemental)
1239 28 Extra code modules loaded at runtime
1241 # Prints the Summary report section
1243 sub print_summary_report
(\
@) {
1244 my ($sections) = @_;
1245 my ($keyname,$cur_level);
1248 my $expand_header_footer = sub {
1251 foreach my $horf (@_) {
1252 # print blank line if keyname is newline
1253 if ($horf eq "\n") {
1256 elsif (my ($sepchar) = ($horf =~ /^(.)$/o)) {
1257 $line .= sprintf "%s %s\n", $sepchar x
8, $sepchar x
50;
1260 die "print_summary_report: unsupported header or footer type \"$horf\"";
1266 if ($Opts{'detail'} >= 5) {
1267 my $header = "****** Summary ";
1268 print $header, '*' x
($Opts{'max_report_width'} - length $header), "\n\n";
1272 foreach my $sref (@$sections) {
1273 # headers and separators
1274 die "Unexpected Section $sref" if (ref($sref) ne 'HASH');
1276 # Start of a new section group.
1277 # Expand and save headers to output at end of section group.
1278 if ($sref->{CLASS
} eq 'GROUP_BEGIN') {
1279 $cur_level = $sref->{LEVEL
};
1280 $headers[$cur_level] = &$expand_header_footer(@{$sref->{HEADERS
}});
1283 elsif ($sref->{CLASS
} eq 'GROUP_END') {
1284 my $prev_level = $sref->{LEVEL
};
1286 # If this section had lines to output, tack on headers and footers,
1287 # removing extraneous newlines.
1288 if ($lines[$cur_level]) {
1289 # squish multiple blank lines
1290 if ($headers[$cur_level] and substr($headers[$cur_level],0,1) eq "\n") {
1291 if ( ! defined $lines[$prev_level][-1] or $lines[$prev_level][-1] eq "\n") {
1292 $headers[$cur_level] =~ s/^\n+//;
1296 push @{$lines[$prev_level]}, $headers[$cur_level] if $headers[$cur_level];
1297 push @{$lines[$prev_level]}, @{$lines[$cur_level]};
1298 my $f = &$expand_header_footer(@{$sref->{FOOTERS
}});
1299 push @{$lines[$prev_level]}, $f if $f;
1300 $lines[$cur_level] = undef;
1303 $headers[$cur_level] = undef;
1304 $cur_level = $prev_level;
1307 elsif ($sref->{CLASS
} eq 'DATA') {
1309 $keyname = $sref->{NAME
};
1310 if ($Totals{$keyname} > 0) {
1311 my ($numfmt, $desc, $divisor) = ($sref->{FMT
}, $sref->{TITLE
}, $sref->{DIVISOR
});
1314 my $extra = ' %25s';
1315 my $total = $Totals{$keyname};
1317 # Z format provides unitized or unaltered totals, as appropriate
1318 if ($numfmt eq 'Z') {
1319 ($total, $fmt) = unitize
($total, $fmt);
1326 if ($divisor and $$divisor) {
1327 # XXX generalize this
1328 if (ref ($desc) eq 'ARRAY') {
1329 $desc = @$desc[0] . ' ' . @$desc[1] x
(42 - 2 - length(@$desc[0]));
1332 push @{$lines[$cur_level]},
1333 sprintf "$fmt %-42s %6.2f%%\n", $total, $desc,
1334 $$divisor == $Totals{$keyname} ? 100.00 : $Totals{$keyname} * 100 / $$divisor;
1339 $new_line = sprintf("$fmt %-23s \n", $total, $desc);
1342 $new_line = sprintf("$fmt %-23s $extra\n",
1345 commify
($Totals{$keyname}));
1347 push @{$lines[$cur_level]}, $new_line
1352 die "print_summary_report: unexpected control...";
1359 # Prints the Detail report section
1361 # Note: side affect; deletes each key in Totals/Counts
1362 # after printout. Only the first instance of a key in
1363 # the Section table will result in Detail output.
1364 sub print_detail_report
(\
@) {
1365 my ($sections) = @_;
1366 my $header_printed = 0;
1368 return unless (keys %Counts);
1370 #use Devel::Size qw(size total_size);
1372 foreach my $sref ( @$sections ) {
1373 next unless $sref->{CLASS
} eq 'DATA';
1374 # only print detail for this section if DETAIL is enabled
1375 # and there is something in $Counts{$keyname}
1376 next unless $sref->{DETAIL
};
1377 next unless exists $Counts{$sref->{NAME
}};
1379 my $keyname = $sref->{NAME
};
1380 my $max_level = undef;
1381 my $print_this_key = 0;
1383 my @levelspecs = ();
1384 clear_level_specs
($max_level, \
@levelspecs);
1385 if (exists $Opts{$keyname}) {
1386 $max_level = create_level_specs
($Opts{$keyname}, $Opts{'detail'}, \
@levelspecs);
1387 $print_this_key = 1 if ($max_level);
1390 $print_this_key = 1;
1392 #print_level_specs($max_level,\@levelspecs);
1394 # at detail 5, print level 1, detail 6: level 2, ...
1396 #print STDERR "building: $keyname\n";
1397 my ($count, $treeref) =
1398 buildTree
(%{$Counts{$keyname}}, defined ($max_level) ? $max_level : 11,
1399 \
@levelspecs, $Opts{'detail'} - 4, 0, $Opts{'debug'} & D_TREE
);
1402 if ($print_this_key) {
1403 my $desc = $sref->{TITLE
};
1406 if (! $header_printed) {
1407 my $header = "****** Detail ($max_level) ";
1408 print $header, '*' x
($Opts{'max_report_width'} - length $header), "\n";
1409 $header_printed = 1;
1411 printf "\n%8d %s %s\n", $count, $desc,
1412 $Opts{'sect_vars'} ?
1413 ('-' x
($Opts{'max_report_width'} - 18 - length($desc) - length($keyname))) . " [ $keyname ] -" :
1414 '-' x
($Opts{'max_report_width'} - 12 - length($desc))
1417 printTree
($treeref, \
@levelspecs, $Opts{'line_style'}, $Opts{'max_report_width'},
1418 $Opts{'debug'} & D_TREE
);
1420 #print STDERR "Total size Counts: ", total_size(\%Counts), "\n";
1421 #print STDERR "Total size Totals: ", total_size(\%Totals), "\n";
1423 $Totals{$keyname} = undef;
1424 delete $Totals{$keyname};
1425 delete $Counts{$keyname};
1432 Print out a standard percentiles report
1434 === Delivery Delays Percentiles ===============================================================
1435 0% 25% 50% 75% 90% 95% 98% 100%
1436 -----------------------------------------------------------------------------------------------
1437 Before qmgr 0.01 0.70 1.40 45483.70 72773.08 81869.54 87327.42 90966.00
1438 In qmgr 0.00 0.00 0.00 0.01 0.01 0.01 0.01 0.01
1439 Conn setup 0.00 0.00 0.00 0.85 1.36 1.53 1.63 1.70
1440 Transmission 0.03 0.47 0.92 1.61 2.02 2.16 2.24 2.30
1441 Total 0.05 1.18 2.30 45486.15 72776.46 81873.23 87331.29 90970.00
1442 ===============================================================================================
1444 === Postgrey Delays Percentiles ===========================================================
1445 0% 25% 50% 75% 90% 95% 98% 100%
1446 -------------------------------------------------------------------------------------------
1447 Postgrey 727.00 727.00 727.00 727.00 727.00 727.00 727.00 727.00
1448 ===========================================================================================
1451 data table: ref to array of arrays, first cell is label, subsequent cells are data
1455 string of space or comma separated integers, which are the percentiles
1456 calculated and output as table column data
1458 sub print_percentiles_report2
($$$) {
1459 my ($tableref, $title, $percentiles_str) = @_;
1461 return unless @$tableref;
1463 my $myfw2 = $fw2 - 1;
1464 my @percents = split /[ ,]/, $percentiles_str;
1466 # Calc y label width from the hash's keys. Each key is padded with the
1467 # string "#: ", # where # is a single-digit sort index.
1468 my $y_label_max_width = 0;
1470 $y_label_max_width = length($_->[0]) if (length($_->[0]) > $y_label_max_width);
1474 my $col_titles_str = sprintf "%-${y_label_max_width}s" . "%${myfw2}s%%" x
@percents , ' ', @percents;
1475 my $table_width = length($col_titles_str);
1478 my $table_header_str = sprintf "%s %s ", $sep1 x
3, $title;
1479 $table_header_str .= $sep1 x
($table_width - length($table_header_str));
1481 print "\n", $table_header_str;
1482 print "\n", $col_titles_str;
1483 print "\n", $sep2 x
$table_width;
1485 my (@p, @coldata, @xformed);
1486 foreach (@$tableref) {
1487 my ($title, $ref) = ($_->[0], $_->[1]);
1488 #xxx my @sorted = sort { $a <=> $b } @{$_->[1]};
1492 for my $bucket (sort { $a <=> $b } keys %$ref) {
1493 #print "Key: $title: Bucket: $bucket = $ref->{$bucket}\n";
1494 # pairs: bucket (i.e. key), tally
1495 push @byscore, $bucket, $ref->{$bucket};
1499 my @p = get_percentiles2
(@byscore, @percents);
1500 printf "\n%-${y_label_max_width}s" . "%${fw2}.2f" x
scalar (@p), $title, @p;
1504 foreach (@percents) {
1505 #printf "\n%-${y_label_max_width}s" . "%${fw2}.2f" x scalar (@p), substr($title,3), @p;
1506 printf "\n%3d%%", $title;
1507 foreach my $val (@{shift @xformed}) {
1516 printf "%${fw3}.2f%-2s", $val, $unit;
1521 print "\n", $sep1 x
$table_width, "\n";
1524 sub clear_level_specs
($ $) {
1525 my ($max_level,$lspecsref) = @_;
1526 #print "Zeroing $max_level rows of levelspecs\n";
1527 $max_level = 0 if (not defined $max_level);
1528 for my $x (0..$max_level) {
1529 $lspecsref->[$x]{topn
} = undef;
1530 $lspecsref->[$x]{threshold
} = undef;
1534 # topn = 0 means don't limit
1535 # threshold = 0 means no min threshold
1536 sub create_level_specs
($ $ $) {
1537 my ($optkey,$gdetail,$lspecref) = @_;
1539 return 0 if ($optkey eq "0");
1541 my $max_level = $gdetail; # default to global detail level
1542 my (@specsP1, @specsP2, @specsP3);
1544 #printf "create_level_specs: key: %s => \"%s\", max_level: %d\n", $optkey, $max_level;
1546 foreach my $sp (split /[\s,]+/, $optkey) {
1547 #print "create_level_specs: SP: \"$sp\"\n";
1548 # original level specifier
1549 if ($sp =~ /^\d+$/) {
1551 #print "create_level_specs: max_level set: $max_level\n";
1553 # original level specifier + topn at level 1
1554 elsif ($sp =~ /^(\d*)\.(\d+)$/) {
1555 if ($1) { $max_level = $1; }
1556 else { $max_level = $gdetail; } # top n specified, but no max level
1558 # force top N at level 1 (zero based)
1559 push @specsP1, { level
=> 0, topn
=> $2, threshold
=> 0 };
1562 elsif ($sp =~ /^::(\d+)$/) {
1563 push @specsP3, { level
=> undef, topn
=> 0, threshold
=> $1 };
1565 elsif ($sp =~ /^:(\d+):(\d+)?$/) {
1566 push @specsP2, { level
=> undef, topn
=> $1, threshold
=> defined $2 ? $2 : 0 };
1568 elsif ($sp =~ /^(\d+):(\d+)?:(\d+)?$/) {
1569 push @specsP1, { level
=> ($1 > 0 ? $1 - 1 : 0), topn
=> $2 ? $2 : 0, threshold
=> $3 ? $3 : 0 };
1572 print STDERR
"create_level_specs: unexpected levelspec ignored: \"$sp\"\n";
1576 #foreach my $sp (@specsP3, @specsP2, @specsP1) {
1577 # printf "Sorted specs: L%d, topn: %3d, threshold: %3d\n", $sp->{level}, $sp->{topn}, $sp->{threshold};
1581 foreach my $sp ( @specsP3, @specsP2, @specsP1) {
1582 ($min, $max) = (0, $max_level);
1584 if (defined $sp->{level
}) {
1585 $min = $max = $sp->{level
};
1587 for my $level ($min..$max) {
1588 #printf "create_level_specs: setting L%d, topn: %s, threshold: %s\n", $level, $sp->{topn}, $sp->{threshold};
1589 $lspecref->[$level]{topn
} = $sp->{topn
} if ($sp->{topn
});
1590 $lspecref->[$level]{threshold
} = $sp->{threshold
} if ($sp->{threshold
});
1597 sub print_level_specs
($ $) {
1598 my ($max_level,$lspecref) = @_;
1599 for my $level (0..$max_level) {
1600 printf "LevelSpec Row %d: %3d %3d\n", $level, $lspecref->[$level]{topn
}, $lspecref->[$level]{threshold
};
1608 package Logreporters
;
1611 import Logreporters
::Utils
;
1612 import Logreporters
::Config
;
1613 import Logreporters
::TreeData
qw(%Totals %Counts %Collecting printTree buildTree);
1614 import Logreporters
::Reports
;
1619 no warnings
"uninitialized";
1625 our $progname = fileparse
($0);
1627 # the list of supplemental reports available in the Detail section
1629 my @supplemental_reports = qw(
1630 autolearn score_percentiles score_frequencies sarules timings sa_timings startinfo
1633 # Default values for various options, used if no config file exists,
1634 # or some option is not set.
1636 # These are used to reset default values after an option has been
1637 # disabled (via undef'ing its value). This allows a report to be
1638 # disabled via config file or --nodetail, but reenabled via subsequent
1639 # command line option
1641 detail
=> 10, # report level detail
1642 max_report_width
=> 100, # maximum line width for report output
1643 line_style
=> undef, # lines > max_report_width, 0=truncate,1=wrap,2=full
1644 syslog_name
=> $progname_prefix, # amavis' syslog service name
1645 sect_vars
=> 0, # show section vars in detail report hdrs
1646 ipaddr_width
=> 15, # width for printing ip addresses
1647 first_recip_only
=> 0, # Show only the first recipient, or all
1649 autolearn
=> 1, # show Autolearn report
1650 bayes
=> 1, # show hit Bayesian buckets
1651 #p0f => 'all all', # p0f hits report
1652 sarules
=> '20 20', # show SpamAssassin rules hit
1653 score_frequencies
=> '-10 -5 0 5 10 20 30', # buckets shown in spam scores report
1654 score_percentiles
=> '0 50 90 95 98 100', # percentiles shown in spam scores report
1655 startinfo
=> 1, # show amavis startup info
1656 timings
=> 95, # show top N% of the timings report
1657 timings_percentiles
=> '0 5 25 50 75 95 100', # percentiles shown in timing report
1658 sa_timings
=> 95, # show top N% of the SA timings report
1659 sa_timings_percentiles
=> '0 5 25 50 75 95 100', # percentiles shown in SA timing report
1662 my $usage_str = <<"END_USAGE";
1663 Usage: $progname [ ARGUMENTS ] [logfile ...]
1665 ARGUMENTS can be one or more of options listed below. Later options override earlier ones.
1666 Any argument may be abbreviated to an unambiguous length. Input comes from named logfiles,
1669 --debug AREAS provide debug output for AREAS
1670 --help print usage information
1671 --version print program version
1673 --config_file FILE, -f FILE use alternate configuration file FILE
1674 --syslog_name PATTERN only consider log lines that match
1675 syslog service name PATTERN
1677 --detail LEVEL print LEVEL levels of detail
1679 --nodetail set all detail levels to 0
1680 --[no]summary display the summary section
1682 --ipaddr_width WIDTH use WIDTH chars for IP addresses in
1683 address/hostname pairs
1684 --line_style wrap|full|truncate disposition of lines > max_report_width
1686 --full same as --line_style=full
1687 --truncate same as --line_style=truncate
1688 --wrap same as --line_style=wrap
1689 --max_report_width WIDTH limit report width to WIDTH chars
1691 --limit L=V, -l L=V set level limiter L with value V
1692 --[no]sect_vars [do not] show config file var/cmd line
1693 option names in section titles
1695 --[no]autolearn show autolearn report
1696 --[no]by_ccat_summary include by contents category grouping in summary
1697 --[no]first_recip_only show first recipient only, or all recipients
1698 --nosarules disable SpamAssassin spam and ham rules hit reports
1699 --sarules "S,H" enable SpamAssassin spam and ham rules reports, showing
1700 --sarules "default" showing the top S spam and top H ham rules hit (range:
1701 0..., "all", or the keyword "default").
1702 --noscore_frequencies disable spam score frequency report
1703 --score_frequencies "B1 [B2 ...]" enable spam score frequency report, using buckets
1704 --score_frequencies "default" specified with B1 [B2 ...] (range: real numbers), or using their
1705 internal default values when the keyword "default" is given
1706 --noscore_percentiles disable spam score percentiles report
1707 --score_percentiles "P1 [P2 ...]" enable spam score percentiles report, using percentiles
1708 --score_percentiles "default" specified with P1 [P2 ...] (range: 0...100), or using their
1709 internal default values when the keyword "default" is given
1710 --[no]startinfo show latest amavis startup details, if available
1712 --nosa_timings disable the SA timings report (same as --sa_timings 0)
1713 --sa_timings PERCENT show top PERCENT percent of the SA timings report (range: 0...100)
1714 --sa_timings_percentiles "P1 [P2 ...]"
1715 set SA timings report percentiles to P1 [P2 ...] (range: 0...100)
1717 --notimings disable the timings report (same as --timings 0)
1718 --timings PERCENT show top PERCENT percent of the timings report (range: 0...100)
1719 --timings_percentiles "P1 [P2 ...]" set timings report percentiles to P1 [P2 ...] (range: 0...100)
1724 sub init_getopts_table
();
1725 sub init_defaults
();
1726 sub build_sect_table
();
1729 sub triway_opts
($$);
1731 sub printSpamScorePercentilesReport
;
1732 sub printSpamScoreFrequencyReport
;
1733 sub printAutolearnReport
;
1734 sub printSARulesReport
;
1735 sub printTimingsReport
($$$$);
1736 sub printStartupInfoReport
;
1738 sub prioritize_cmdline
(@);
1740 sub create_ignore_list
();
1741 sub check_ignore_list
($ \
@);
1743 # lines that match any RE in this list will be ignored.
1744 # see create_ignore_list();
1745 my @ignore_list_final = ();
1747 # The Sections table drives Summary and Detail reports. For each entry in the
1748 # table, if there is data avaialable, a line will be output in the Summary report.
1749 # Additionally, a sub-section will be output in the Detail report if both the
1750 # global --detail, and the section's limiter variable, are sufficiently high (a
1751 # non-existent section limiter variable is considered to be sufficiently high).
1755 # Initialize main running mode and basic opts
1756 init_run_mode
($config_file);
1758 # Configure the Getopts options table
1759 init_getopts_table
();
1761 # Place configuration file/environment variables onto command line
1764 # Initialize default values
1767 # Process command line arguments, 0=no_permute,no_pass_through
1770 # Build the Section table
1773 # Run through the list of Limiters, setting the limiters in %Opts.
1774 process_limiters
(@Sections);
1776 # Set collection for any enabled supplemental sections
1777 foreach (@supplemental_reports) {
1778 $Logreporters::TreeData
::Collecting
{$_} = (($Opts{'detail'} >= 5) && $Opts{$_}) ? 1 : 0;
1781 # Don't collect SpamScores when not necessary
1782 $Collecting{'spamscores'} = ($Opts{'detail'} >= 5 && ($Opts{'score_percentiles'} || $Opts{'score_frequencies'})) ? 1 : 0;
1784 if (! defined $Opts{'line_style'}) {
1785 # default line style to full if detail >= 11, or truncate otherwise
1786 $Opts{'line_style'} =
1787 ($Opts{'detail'} > 10) ? $line_styles{'full'} : $line_styles{'truncate'};
1790 # Create the list of REs used to match against log lines
1791 create_ignore_list
();
1793 my (%Timings, %TimingsSA, @TimingsTotals, @TimingsSATotals);
1794 my (%SaveLine, %StartInfo);
1795 my (%SpamScores, %spamtags, %p0ftags);
1797 # Priority: VIRUS BANNED UNCHECKED SPAM SPAMMY BADH OVERSIZED MTA CLEAN
1798 my %ccatmajor_to_sectkey = (
1799 'INFECTED' => 'malware',
1800 'BANNED' => 'bannedname',
1801 'UNCHECKED' => 'unchecked',
1802 'UNCHECKED-ENCRYPTED' => 'unchecked',
1804 'SPAMMY' => 'spammy',
1805 'BAD-HEADER' => 'badheader',
1806 'OVERSIZED' => 'oversized',
1807 'MTA-BLOCKED' => 'mta',
1809 'TEMPFAIL' => 'tempfail',
1813 my %ccatmajor_to_priority = (
1828 my %ccatmajor_to_spamham = (
1829 'INFECTED' => 'malware',
1830 'BANNED' => 'bannedname',
1831 'UNCHECKED' => 'unchecked',
1834 'BAD-HEADER' => 'ham',
1835 'OVERSIZED' => 'ham',
1836 'MTA-BLOCKED' => 'ham',
1838 'TEMPFAIL' => 'ham',
1842 my $logline_maxlen = 980;
1844 # Create the list of REs against which log lines are matched.
1845 # Lines that match any of the patterns in this list are ignored.
1847 # Note: This table is created at runtime, due to a Perl bug which
1848 # I reported as perl bug #56202:
1850 # http://rt.perl.org/rt3/Public/Bug/Display.html?id=56202
1853 sub create_ignore_list
() {
1854 push @ignore_list_final, qr/^lookup_ip_acl/;
1855 push @ignore_list_final, qr/^lookup_acl/;
1856 push @ignore_list_final, qr/^lookup_hash/;
1857 push @ignore_list_final, qr/^lookup_re/;
1858 push @ignore_list_final, qr/^lookup_ldap/;
1859 push @ignore_list_final, qr/^lookup_sql_field.* result=[YN]$/;
1860 push @ignore_list_final, qr/^lookup .* does not match$/;
1861 push @ignore_list_final, qr/^lookup [[(]/;
1862 push @ignore_list_final, qr/^lookup => /;
1863 push @ignore_list_final, qr/^lookup: /;
1864 push @ignore_list_final, qr/^save_info_preliminary/; # log level 4
1865 push @ignore_list_final, qr/^save_info_final/; # log level 4
1866 push @ignore_list_final, qr/^sql: /;
1867 push @ignore_list_final, qr/^sql_storage: retrying/;
1868 push @ignore_list_final, qr/^sql flush: /;
1869 push @ignore_list_final, qr/^sql print/;
1870 push @ignore_list_final, qr/^sql begin transaction/;
1871 push @ignore_list_final, qr/^sql rollback/;
1872 push @ignore_list_final, qr/^mail_via_sql: /;
1873 push @ignore_list_final, qr/^CALLING SA check$/;
1874 push @ignore_list_final, qr/^calling SA parse,/;
1875 push @ignore_list_final, qr/^timer set to \d+/;
1876 push @ignore_list_final, qr/^query_keys/;
1877 push @ignore_list_final, qr/^find_or_save_addr: /;
1878 push @ignore_list_final, qr/^header: /;
1879 push @ignore_list_final, qr/^DO_QUARANTINE, /;
1880 push @ignore_list_final, qr/^DEBUG_ONESHOT: /;
1881 push @ignore_list_final, qr/^TempDir::/;
1882 push @ignore_list_final, qr/^check_mail_begin_task: /;
1883 push @ignore_list_final, qr/^program: .*?(anomy|altermime|disclaimer).*? said: /; # log_level 2
1884 push @ignore_list_final, qr/^body (?:type|hash): /;
1885 push @ignore_list_final, qr/^\d+\.From: <.*>, \d+.Mail_From:/;
1886 push @ignore_list_final, qr/^The amavisd daemon is (?:apparently )?not running/;
1887 push @ignore_list_final, qr/^rw_loop/;
1888 push @ignore_list_final, qr/^[SL]MTP[><]/;
1889 push @ignore_list_final, qr/^[SL]MTP response for/;
1890 push @ignore_list_final, qr/^dsn:/i, # DSN or dsn
1891 push @ignore_list_final, qr/^enqueue: /;
1892 push @ignore_list_final, qr/^write_header: /;
1893 push @ignore_list_final, qr/^banned check: /;
1894 push @ignore_list_final, qr/^child_finish_hook/;
1895 push @ignore_list_final, qr/^inspect_dsn:/;
1896 push @ignore_list_final, qr/^client IP address unknown/;
1897 push @ignore_list_final, qr/^final_destiny/;
1898 push @ignore_list_final, qr/^one_response_for_all/;
1899 push @ignore_list_final, qr/^headers CLUSTERING/;
1900 push @ignore_list_final, qr/^notif=/;
1901 push @ignore_list_final, qr/^\(about to connect/;
1902 push @ignore_list_final, qr/^Original mail size/;
1903 push @ignore_list_final, qr/^TempDir removal/;
1904 push @ignore_list_final, qr/^Issued a new file name/;
1905 push @ignore_list_final, qr/^starting banned checks/;
1906 push @ignore_list_final, qr/^skip admin notification/;
1907 push @ignore_list_final, qr/^do_notify_and_quarantine - done/;
1908 push @ignore_list_final, qr/^do_[a-zA-Z]+.* done$/i;
1909 push @ignore_list_final, qr/^Remote host presents itself as:/;
1910 push @ignore_list_final, qr/^connect_to_ldap/;
1911 push @ignore_list_final, qr/^connect_to_sql: trying /;
1912 push @ignore_list_final, qr/^ldap begin_work/;
1913 push @ignore_list_final, qr/^Connecting to LDAP server/;
1914 push @ignore_list_final, qr/^loaded base policy bank/;
1915 push @ignore_list_final, qr/^\d+\.From:/;
1916 push @ignore_list_final, qr/^Syslog (retries|warnings)/;
1917 push @ignore_list_final, qr/^smtp connection cache/;
1918 push @ignore_list_final, qr/^smtp cmd> /;
1919 push @ignore_list_final, qr/^smtp session/;
1920 push @ignore_list_final, qr/^Ignoring stale PID file/;
1921 push @ignore_list_final, qr/^mime_decode_preamble/;
1922 push @ignore_list_final, qr/^doing banned check for/;
1923 push @ignore_list_final, qr/^open_on_specific_fd/;
1924 push @ignore_list_final, qr/^reparenting /;
1925 push @ignore_list_final, qr/^Issued a new pseudo part: /;
1926 push @ignore_list_final, qr/^run_command: /;
1927 push @ignore_list_final, qr/^result line from file/;
1928 push @ignore_list_final, qr/^Charging /;
1929 push @ignore_list_final, qr/^check_for_banned /;
1930 push @ignore_list_final, qr/^Extracting mime components$/;
1931 push @ignore_list_final, qr/^response to /;
1932 push @ignore_list_final, qr/^File-type of /;
1933 push @ignore_list_final, qr/^Skip admin notification, /;
1934 push @ignore_list_final, qr/^run_av: /;
1935 push @ignore_list_final, qr/^string_to_mime_entity /;
1936 push @ignore_list_final, qr/^ndn_needed=/;
1937 push @ignore_list_final, qr/^sending RCPT TO:/;
1938 push @ignore_list_final, qr/^decode_parts: /;
1939 push @ignore_list_final, qr/^decompose_part: /;
1940 push @ignore_list_final, qr/^setting body type: /;
1941 push @ignore_list_final, qr/^mime_decode_epilogue: /;
1942 push @ignore_list_final, qr/^string_to_mime_entity: /;
1943 push @ignore_list_final, qr/^at the END handler: /;
1944 push @ignore_list_final, qr/^Amavis::.* called$/;
1945 push @ignore_list_final, qr/^Amavis::.* close,/;
1946 push @ignore_list_final, qr/^dkim: /; # XXX provide stats
1947 push @ignore_list_final, qr/^collect banned table/;
1948 push @ignore_list_final, qr/^collect_results from/;
1949 push @ignore_list_final, qr/^blocking contents category is/;
1950 push @ignore_list_final, qr/^running file\(/;
1951 push @ignore_list_final, qr/^Found av scanner/;
1952 push @ignore_list_final, qr/^Found myself/;
1953 push @ignore_list_final, qr/^mail_via_smtp/;
1954 push @ignore_list_final, qr/^switch_to_client_time/;
1955 push @ignore_list_final, qr/^parse_message_id/;
1956 push @ignore_list_final, qr/^parse_received: /;
1957 push @ignore_list_final, qr/^parse_ip_address_from_received: /;
1958 push @ignore_list_final, qr/^fish_out_ip_from_received: /;
1959 push @ignore_list_final, qr/^Waiting for the process \S+ to terminate/;
1960 push @ignore_list_final, qr/^Valid PID file \(younger than sys uptime/;
1961 push @ignore_list_final, qr/^Sending SIG\S+ to amavisd/;
1962 push @ignore_list_final, qr/^Can't send SIG\S+ to process/;
1963 push @ignore_list_final, qr/^killing process/;
1964 push @ignore_list_final, qr/^no need to kill process/;
1965 push @ignore_list_final, qr/^process .* is still alive/;
1966 push @ignore_list_final, qr/^Daemon \[\d+\] terminated by SIG/;
1967 push @ignore_list_final, qr/^storage and lookups will use .* to SQL/;
1968 push @ignore_list_final, qr/^idle_proc, /;
1969 push @ignore_list_final, qr/^switch_to_my_time/;
1970 push @ignore_list_final, qr/^TempDir::strip: /;
1971 push @ignore_list_final, qr/^rmdir_recursively/;
1972 push @ignore_list_final, qr/^sending [SL]MTP response/;
1973 push @ignore_list_final, qr/^prolong_timer/;
1974 push @ignore_list_final, qr/^process_request:/;
1975 push @ignore_list_final, qr/^exiting process_request/;
1976 push @ignore_list_final, qr/^post_process_request_hook: /;
1977 push @ignore_list_final, qr/^SMTP session over/;
1978 push @ignore_list_final, qr/^updating snmp variables/;
1979 push @ignore_list_final, qr/^best_try_originator_ip/;
1980 push @ignore_list_final, qr/^mail checking ended: /; # log level 2
1981 push @ignore_list_final, qr/^The amavisd daemon is already running/;
1982 push @ignore_list_final, qr/^AUTH not needed/;
1983 push @ignore_list_final, qr/^load: \d+ %, total idle/;
1984 push @ignore_list_final, qr/^policy protocol: [^=]+=\S+(?:,\S+)*$/; # allow "policy protocol: INVALID ..." later
1985 push @ignore_list_final, qr/^penpals: /;
1986 push @ignore_list_final, qr/^Not calling virus scanners, no files to scan in/;
1987 push @ignore_list_final, qr/^local delivery: /;
1988 push @ignore_list_final, qr/^run_as_subprocess: child process \S*: Broken pipe/;
1989 push @ignore_list_final, qr/^initializing Mail::SpamAssassin/;
1990 push @ignore_list_final, qr/^Error reading mail header section/; # seems to occur gen. due to perl getline() bug
1991 push @ignore_list_final, qr/^flatten_and_tidy_dir/;
1992 push @ignore_list_final, qr/^do_7zip: member/;
1993 push @ignore_list_final, qr/^Expanding \S+ archive/;
1994 push @ignore_list_final, qr/^files_to_scan:/;
1995 push @ignore_list_final, qr/^Unzipping p\d+/;
1996 push @ignore_list_final, qr/^writing mail text to SQL/;
1997 push @ignore_list_final, qr/^strip_tempdir/;
1998 push @ignore_list_final, qr/^no parts, file/;
1999 push @ignore_list_final, qr/^warnsender_with_pass/;
2000 push @ignore_list_final, qr/^RETURNED FROM SA check/;
2001 push @ignore_list_final, qr/^mime_traverse: /;
2002 push @ignore_list_final, qr/^do_spam: /;
2003 push @ignore_list_final, qr/^prepare_tempdir: /;
2004 push @ignore_list_final, qr/^check_header: /;
2005 push @ignore_list_final, qr/^skip admin notification/;
2006 push @ignore_list_final, qr/^do_executable: not a/;
2007 push @ignore_list_final, qr/^Skip spam admin notification, no administrators$/;
2008 push @ignore_list_final, qr/^skip banned check for/;
2009 push @ignore_list_final, qr/^is_outgoing /;
2010 push @ignore_list_final, qr/^NO Disclaimer/;
2011 push @ignore_list_final, qr/^Using \(\S+\) on file/;
2012 push @ignore_list_final, qr/^no anti-spam code loaded/;
2013 push @ignore_list_final, qr/^entered child_init_hook/;
2014 push @ignore_list_final, qr/^body type/;
2015 push @ignore_list_final, qr/^establish_or_refresh/;
2016 push @ignore_list_final, qr/^get_body_digest/;
2017 push @ignore_list_final, qr/^ask_daemon_internal/;
2018 push @ignore_list_final, qr/^Turning AV infection into a spam report, name already accounted for/;
2019 push @ignore_list_final, qr/^Calling virus scanners/;
2020 push @ignore_list_final, qr/^timer stopped after /;
2021 push @ignore_list_final, qr/^virus_presence /;
2022 push @ignore_list_final, qr/^cache entry /;
2023 push @ignore_list_final, qr/^generate_mail_id /;
2024 push @ignore_list_final, qr/^Load low precedence policybank/;
2025 push @ignore_list_final, qr/^warm restart on /; # XXX could be placed instartup info
2026 push @ignore_list_final, qr/^Signalling a SIGHUP to a running daemon/;
2027 push @ignore_list_final, qr/^Deleting db files /;
2028 push @ignore_list_final, qr/^address modified \(/;
2029 push @ignore_list_final, qr/^Request: AM\.PDP /;
2030 push @ignore_list_final, qr/^DSPAM result: /;
2031 push @ignore_list_final, qr/^bind to \//;
2032 push @ignore_list_final, qr/^ZMQ enabled: /;
2034 push @ignore_list_final, qr/^Inserting header field: X-Amavis-Hold: /;
2035 push @ignore_list_final, qr/^Decoding of .* failed, leaving it unpacked: /;
2036 push @ignore_list_final, qr/^File::LibMagic::describe_filename failed on p\d+: /;
2038 # various forms of "Using ..."
2039 # more specific, interesting variants already captured: search "Using"
2040 push @ignore_list_final, qr/^Using \(.*\) on dir:/;
2041 push @ignore_list_final, qr/^Using [^:]+: \(built-in interface\)/;
2042 push @ignore_list_final, qr/^Using \(.*\): /;
2043 push @ignore_list_final, qr/: sleeping for /;
2044 push @ignore_list_final, qr/creating socket by /;
2047 push @ignore_list_final, qr/\bRUSAGE\b/;
2048 push @ignore_list_final, qr/: Sending .* to UNIX socket/;
2050 # Lines beginning with "sd_notify:" or "sd_notify (no socket):"
2051 # describe what is being sent to the systemd notification socket,
2053 push @ignore_list_final, qr/^sd_notify( \(no socket\))?:/;
2058 # - IN REs, always use /o flag or qr// at end of RE when RE uses unchanging interpolated vars
2059 # - In REs, email addresses may be empty "<>" - capture using *, not + ( eg. from=<[^>]*> )
2060 # - See additional notes below, search for "Note:".
2061 # - XXX indicates change, fix or more thought required
2063 # Main processing loop
2070 $Logreporters::Reports
::origline
= $_;
2072 if ($Opts{'standalone'}) {
2073 next unless s/^[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} (?:<[^>]+> )?\S+ $Opts{'syslog_name'}(?:\[\d+\])?: (?:\[ID \d+ \w+\.\w+\] )?//o;
2078 my $action = "blocked"; # default action is blocked if not present in log
2080 # For now, ignore the amavis startup timing lines. Need to do this
2081 # before stripping out the amavis pid to differentiate these from the
2082 # scan timing reports
2083 next if ($p1 =~ /^TIMING/);
2085 my $linelen = length $p1;
2086 # Strip amavis process id-instance id, or release id
2087 if (($pid,$p2) = ($p1 =~ /^\(([^)]+)\) (.*)$/ )) {
2091 # Handle continuation lines. Continuation lines should be in order per PID, meaning line1, line2, line3,
2092 # but never line3, line1, line2.
2094 # amavis log lines as chopped by sub write_log are exactly 980 characters long starting with '(' as in:
2095 # amavis[47061]: (47061-15) SPAM, etc ...
2096 # ^ <-----980------------->
2097 # but this can be changed in amavis via $logline_maxlen.
2098 # There may also be the alert markers (!) and (!!) preceeding any continuation ellipsis.
2101 # ... a continued line ...
2102 if ($p1 =~ s/^(\([!]{1,2}\))?\.\.\.//) {
2103 if (!exists($SaveLine{$pid})) {
2105 #printf "Unexpected continue line: \"%s\"\n", $p1;
2106 $SaveLine{$pid} = $alert || '';
2108 $SaveLine{$pid} .= $p1;
2109 next if $SaveLine{$pid} =~ s/\.\.\.$//; # next if line has more pieces
2112 # this line continues ...
2113 if ($p1 =~ /\.\.\.$/ and $linelen == $logline_maxlen) {
2115 $SaveLine{$pid} = $p1;
2119 if (exists($SaveLine{$pid})) {
2120 # printf "END OF SaveLine: %s\n", $SaveLine{$pid};
2121 $p1 = delete $SaveLine{$pid};
2124 #if (length($p1) > 10000) {
2125 # printf "Long log entry %d chars: \"%s\"\n", length($p1), $p1;
2130 # Place REs here that should ignore log lines otherwise caught below.
2131 # Some are located here historically, and need to be checked for candidates
2132 # to be relocated to ignore_list_final.
2133 ($p1 =~ /^do_ascii/)
2134 or ($p1 =~ /^Checking/)
2135 or ($p1 =~ /^header_edits_for_quar: /)
2136 or ($p1 =~ /^Not-Delivered/)
2137 or ($p1 =~ /^SpamControl/)
2139 or ($p1 =~ /^ESMTP/)
2140 or ($p1 =~ /^UTF8SMTP/)
2141 or ($p1 =~ /^(?:\(!+\))?(\S+ )?(?:FWD|SEND) from /) # log level 4
2142 or ($p1 =~ /^(?:\(!+\))?(\S+ )?(?:ESMTP|FWD|SEND) via /) # log level 4
2143 or ($p1 =~ /^tempdir being removed/)
2144 or ($p1 =~ /^do_notify_and_quar(?:antine)?: .*ccat/)
2145 or ($p1 =~ /^cached [a-zA-Z0-9]+ /)
2146 or ($p1 =~ /^loaded policy bank/)
2147 or ($p1 =~ /^p\.path/)
2148 or ($p1 =~ /^virus_scan: /)
2149 or ($p1 =~ /^Requesting (a |)process rundown after [0-9]+ tasks/)
2150 or ($p1 =~ /^Cached (virus|spam) check expired/)
2151 or ($p1 =~ /^pr(?:esent|ovid)ing full original message to scanners as/) # log level 2
2152 or ($p1 =~ /^Actual message size [0-9]+ B(,| greater than the) declared [0-9]+ B/)
2153 or ($p1 =~ /^disabling DSN/)
2154 or ($p1 =~ /^Virus ([^,]+ )?matches [^,]+, sender addr ignored/)
2155 or ($p1 =~ /^release /)
2156 or ($p1 =~ /^adding SA score \S+ to existing/)
2157 or ($p1 =~ /^Maia:/) # redundant
2158 or ($p1 =~ /^AM\.PDP /) # this appears to be always have two spaces
2159 # because in amavisd::preprocess_policy_query() when $ampdp is
2160 # set, it will pass an unset $attr_ref->{'mail_id'} to do_log(1
2161 or ($p1 =~ /^_(?:WARN|DIE):$/) # bug: empty _WARN|_DIE: http://marc.info/?l=amavis-user&m=121725098111422&w=2
2163 # non-begin anchored
2164 or ($p1 =~ /result: clean$/)
2165 or ($p1 =~ /DESTROY called$/)
2166 or ($p1 =~ /email\.txt no longer exists, can't re-use it/)
2167 or ($p1 =~ /SPAM\.TAG2/)
2168 or ($p1 =~ /BAD-HEADER\.TAG2/)
2169 or ($p1 =~ /: Connecting to socket/)
2170 or ($p1 =~ /broken pipe \(don't worry\), retrying/)
2171 or ($p1 =~ /(?:Sending|on dir:) (?:CONT)?SCAN /)
2174 my ($ip, $from, $to, $key,, $reason, $item,
2175 $decoder, $scanner, $stage, $sectkey);
2177 # Coerce older "INFECTED" quarantined lines into "Blocked INFECTED",
2178 # to be processed in the Passed/Blocked section.
2179 if ($p1 =~ /^INFECTED.*, quarantine/) {
2180 $p1 = 'Blocked ' . $p1;
2183 # SPAM entry occurs at kill level
2184 # SPAM-TAG entry occurs at log level 2, when spam header is inserted
2185 # log_level >= 2 || (log_level > 2 && syslog_priority=debug)
2186 my ($tagtype,$fromto,$isspam,$tags,$tests,$autolearn);
2188 # amavisd-new 2.7.0 changes SPAM-TAG to Spam-tag and its log_level to 3
2189 if (($tagtype,$fromto,$isspam,$tags,$tests,$autolearn) = ($p1 =~ /^((?i:SPAM(?:-TAG)?)), (.*), (Yes|No), score=[-+x\d.]+(.*) tests=\[([^\]]*)](?:, autolearn=(\w+))?/) or
2190 ($tagtype,$fromto,$isspam,$tags,$tests) = ($p1 =~ /^((?i:SPAM(?:-TAG)?)), (.*), (Yes|No), hits=[-+x\d.]+(.*) tests=(.*)(?:, quarantine )?/)) {
2192 #TD SPAM, <from@example.com> -> <to@sample.com>, Yes, score=17.709 tag=-10 tag2=6.31 kill=6.31 tests=[AWL=-0.678, BAYES_99=4], autolearn=spam, quarantine Cc4+GUJhgpqh (spam-quarantine)
2193 #TD SPAM, <from@example.com> -> <to@sample.net>, Yes, score=21.161 tag=x tag2=8.15 kill=8.15 tests=[BAYES_99=2.5, FORGED_RCVD_HELO=0.135], autolearn=no, quarantine m6lWPoTGJ2O (spam-quarantine)
2194 #TD SPAM, <from@example.com> -> <to@sample.net>, Yes, score=17.887 tag=-10 tag2=6.31 kill=6.31 tests=[BAYES_99=4], autolearn=spam, quarantine VFYjDOVTW4zd (spam-quarantine)
2195 #TD SPAM-TAG, <from@example.com> -> <to@sample.net>, No, score=-0.069 tagged_above=-10 required=6.31 tests=[BAYES_00=-2.599, FROM_ENDS_IN_NUMS=2.53]
2196 #TD SPAM-TAG, <from@example.com> -> <to@sample.net>, No, score=-1.294 required=8.15 tests=[BAYES_00=-2.599, FROM_LOCAL_HEX=1.305]
2198 #TD SPAM-TAG, <from@example.com> -> <to@sample.net>, Yes, hits=6.159 tagged_above=-999 required=3.4 tests=BAYES_99=3.5, FUZZY_CPILL=0.518, HTML_MESSAGE=0.001, URIBL_WS_SURBL=2.14
2199 #TD SPAM, <from@example.com> -> <to@sample.net>, Yes, hits=8.1 tag1=-999.0 tag2=7.0 kill=7.0 tests=MANGLED_TAKE, UPPERCASE_25_50, quarantine spam-14156-09 (maia-spam-quarantine)
2201 $Totals{'tagged'}++ if uc($tagtype) eq 'SPAM-TAG';
2204 my $type = $isspam =~ /^Y/ ? 'Spam' : 'Ham';
2206 # Note: A SPAM line may be followed by an almost identical SPAM-TAG line. To avoid double counting,
2207 # maintain a list of (abbreviated) SPAM tag lines keyed by pid. Since pid's are recycled,
2208 # maintain an approximation of uniqueness by combining several components from the log
2209 # line (we can't use the date information, as in logwatch, it is not present).
2210 # XXX: It is safe to delete an entry when the final Passed/Block line occurs
2212 #TD SPAM, <from@example.com> -> <to@sample.net>, Yes, score=34.939 tag=x tag2=6.31 kill=6.31 tests=[DATE_IN_FUTURE_03_06=1.961], autolearn=disabled
2213 #TD SPAM-TAG, <from@example.com> -> <to@sample.net>, Yes, score=34.939 required=6.31 tests=[DATE_IN_FUTURE_03_06=1.961]
2214 #TD SPAM, <from@example.com> -> tod@sample.net>, Yes, score=31.565 tag=x tag2=6.9 kill=6.9 tests=[AV:Sanesecurity.Phishing.Bank.2666.UNOFFICIAL=4.1, AV:Sanesecurity.Phishing.Bank.2666.UNOFFICIAL=4.1, BAYES_99=4, DCC_CHECK=4, DIGEST_MULTIPLE=0.001, FORGED_MUA_OUTLOOK=3.116, FORGED_OUTLOOK_HTML=0.001, FORGED_OUTLOOK_TAGS=0.001, HTML_MESSAGE=0.001, L_AV_SS_Phish=5, MIME_HTML_ONLY=1.457, NORMAL_HTTP_TO_IP=0.001, RAZOR2_CF_RANGE_51_100=2, RAZOR2_CF_RANGE_E4_51_100=1.5, RAZOR2_CF_RANGE_E8_51_100=1.5, RAZOR2_CHECK=3, RDNS_NONE=0.1, URIBL_PH_SURBL=1.787] autolearn=spam
2217 my $tagstr = $fromto . '/' . $isspam . '/' . $tests;
2218 if (uc($tagtype) eq 'SPAM-TAG' and exists $spamtags{$pid}) {
2219 next if ($spamtags{$pid} eq $tagstr);
2221 $spamtags{$pid} = $tagstr;
2223 #for (split /=[^,]+(?:, +|$)/, $tests)
2224 # amavis < 2.6.2 would double list AV names when using
2225 # @virus_name_to_spam_score_maps.
2226 my @unique_tests = unique_list
(split /, +/, $tests);
2227 for (@unique_tests) {
2228 # skip possible trailing junk ("quarantine, ...") when older non-bracked tests=xxx is used
2229 next if ! /[^=]+=[\-.\d]+/;
2230 my ($id,$val) = split /=/;
2231 if ($id =~ /^BAYES_\d+$/) {
2232 $Counts{'bayes'}{$id}++ if ($Collecting{'bayes'});
2234 if ($Opts{'sarules'}) {
2235 if ($id eq 'DKIM_POLICY_SIGNSOME') { $val = 0 }
2236 elsif ($id eq 'AWL') { $val = '-' }
2237 $Counts{'sarules'}{$type}{sprintf "%6s %s", $val, $id}++;
2241 #autolearn= is available only at ll>=3 or SPAM messages; so ham doesn't naturally occur here
2242 # SA 2.5/2.6 : ham/spam/no
2243 # SA 3.0+ : ham/spam/no/disabled failed/unavailable
2244 #$Counts{'autolearn'}{$type}{$autolearn}++ if ($Opts{'autolearn'});
2249 elsif ($p1 =~ /^(Passed|Blocked)(.*)/) {
2250 $action = lcfirst $1;
2251 ($p1 = $2) =~ s/^\s+//;
2253 $p1 =~ s/^,/CLEAN,/; # canonicalize older log entries
2254 #print "P1: \"$p1\"\n";
2256 # amavis 20030616p10-5
2257 #TD Passed, <from@example.com> -> <to@sample.net>, Message-ID: <652.44494541@example.com>, Hits: 4.377
2258 #TD Passed, <from@example.com> -> <to@sample.net>, Message-ID: <B5C@example.com>, Hits: -
2259 #TD Passed, <from@example.com> -> <to@sample.net>, quarantine IJHkgliCm2Ia, Message-ID: <20080307140552.16E127641E@example.com>, Hits: 0.633
2261 #TD Passed CLEAN, [10.0.0.1] [10.0.0.1] <from@example.com> -> <to@sample.net>, Message-ID: <2qxz191@example.com>, mail_id: w4DHD8, Hits: -2.599, size: 3045, queued_as: 2056, 2664 ms
2262 #TD Passed CLEAN, [10.0.0.1] [10.0.0.1] <from@example.com> -> <to@sample.net>, Message-ID: <2qxz191@example.com>, mail_id: w4DHD8, Hits: -2.541-3, size: 3045, queued_as: 2056, 2664 ms
2263 #TD Blocked SPAM, [10.0.0.1] [192.168.0.1] <bogus@example.com> -> <to@sample.net>, quarantine: spam-EzEbE9W, Message-ID: <117894@example.com>, mail_id: EzEbE9W, Hits: 6.364, size: 16493, 6292 ms
2264 #TD Blocked SPAM, LOCAL [10.0.0.1] [10.0.0.2] <bogus@example.com> -> <to@sample.net>, quarantine: spam-EzEbE9W, Message-ID: <110394@example.com>, mail_id: EzEbE9W, Hits: 6.364, size: 16493, 6292 ms
2265 #TD Blocked SPAM, [IPv6:2001:630:d0:f102:230:48ff:fe77:96e] [192.168.0.1] <joe@example.com> -> <user@sample.net>, quarantine: spam-EzEbE9W, Message-ID: <11780394@example.com>, mail_id: EzEbE9W, Hits: 6.364, size: 16493, 6292 ms
2266 #TD Passed SPAMMY, ORIGINATING/MYNETS LOCAL [10.0.0.1] [10.0.0.1] <from@example.com> -> <to1@sample.net>,<to2@sample.net>, quarantine: spam-EzEbE9W, Message-ID: <11780394@example.com>, mail_id: EzEbE9W, Hits: 6.364, size: 16493, 6292 ms
2267 #TD Blocked SPAM, B-BANK/C-BANK/B-BANK [10.0.0.1] [10.0.0.1] <from@sample.net> -> <to@example.com>, quarantine: spam-EzEbE9W, Message-ID: <11780394@example.com>, mail_id: EzEbE9W, Hits: 6.364, size: 16493, 6292 ms
2268 #TD Blocked SPAM, [10.0.0.1] [10.0.0.1] <from@example.com> -> <to@sample.net>, quarantine: spam-AV49p5, Message-ID: <1.007@sample.net>, mail_id: AV49p5, Hits: 7.487, size: 27174, 4406 ms
2269 #TD Passed SPAM, MYNETS <root@example.com> -> <root@example.com>, quarantine: spam-V3Wq, Message-ID: <220.1B@example.com>, mail_id: V3Wq, Hits: 7, size: 8838, queued_as: C63EC, 18 ms
2270 #TD Passed SPAM, <> -> <"fred).flintstone"@domain.tld>, Message-ID: <200801180104.CAA23669@aserver.sub.adomain.tld>, mail_id: 6AzQ1g0l5RgP, Hits: 9.061, size: 5555, queued_as: C1840506CB8, 8766 ms
2271 #TD Blocked INFECTED (HTML.Phishing.Bank-43), [198.168.0.1] [10.0.0.1] <bogus@example.com> -> <to@sample.net>, quarantine: virus-SCwJcs, Message-ID: <509@acm.org>, mail_id: SCwJcs, Hits: -, size: 4134, 3721 ms
2272 #TD Blocked INFECTED (Trojan.Downloader.Small-9993), LOCAL [10.0.0.2] [10.0.0.2] <bogus@example.net> -> <to@example.com>, quarantine: virus-SCwJcs, Message-ID: <9009@acm.org>, mail_id: SCwJcs, Hits: -, size: 4134, 3721 ms
2273 #TD Blocked BANNED (multipart/report | message/partial,.txt), [192.168.0.1] [10.0.0.2] <> -> <someuser@sample.net>, quarantine: virus-SCwJcs, Message-ID: <509@acm.org>, mail_id: SCwJcs, Hits: -, size: 4134, 3721 ms
2274 #TD Blocked BANNED (multipart/report | message/partial,.txt), LOCAL [192.168.0.1] [10.0.0.2] <> -> <someuser@sample.net>, quarantine: virus-SCwJcs, Message-ID: <509@acm.org>, mail_id: SCwJcs, Hits: -, size: 4134, 3721 ms
2275 #TD Blocked BANNED (multipart/mixed | application/octet-stream,.asc,=?iso-8859-1?Q?FTP=5FFile=5F (1)=File(1).reg), [192.168.0.0] [192.168.0.0] <from@example.com> -> <to@sample.us>, quarantine: virus-SCwJcs, Message-ID: <509@acm.org>, mail_id: SCwJcs, Hits: -, size: 4134, 3721 ms
2276 #TD Blocked BANNED (multipart/related | application/zip,.zip,card.zip | .exe,.exe-ms,Card.exe), [10.0.0.2] [10.0.0.2] <from@example.com> -> <to@sample.net>, quarantine: banned-9OXm4Q3ah, Message-ID: <08517$@from>, mail_id: 9OXm4Q3ah, Hits: -, size: 2366, 3803 ms
2277 #TD Passed BAD-HEADER, [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>, quarantine: virus-SCwJcs, Message-ID: <df@acm.org>, mail_id: SCwJcs, Hits: 2.54 size: 4134, 3721 ms
2278 #TD Passed BAD-HEADER, LOCAL [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>, quarantine: virus-SCwJcs, Message-ID: <df@acm.org>, mail_id: SCwJcs, Hits: 3.2 size: 4134, 3721 ms
2279 #TD Passed BAD-HEADER, MYNETS AM.PDP [127.0.0.1] [127.0.0.1] <bogus@example.com> -> <someuser@sample.net>, quarantine: virus-SCwJcs, Message-ID: <df@acm.org>, mail_id: SCwJcs, Hits: 1.2 size: 4134, 3721 ms
2280 #TD Passed BAD-HEADER, ORIGINATING/MYNETS LOCAL [10.0.0.1] [10.0.0.1] <from@sample.net> -> <to1@sample.net>,<to2@sample.net>,<to3@example.com>, quarantine: virus-SCwJcs, Message-ID: <df@acm.org>, mail_id: SCwJcs, Hits: -, size: 4134, 3721 ms
2281 #TD Passed BAD-HEADER, [10.0.0.1] [10.0.0.2] <from@example.com> -> <to@sample.net>, quarantine: badh-lxR, Message-ID: <7fm@example.com>, mail_id: lxR, Hits: -2.292, size: 422, queued_as: E3B, 981 ms
2282 #TD Passed UNCHECKED, MYNETS LOCAL [192.168.0.1] [192.168.0.1] <from@sample.net> -> <to@example.com> Message-ID: <002e01c759c7$5de437b0$0a02a8c0@somehost>, mail_id: 7vtR-7BAvHZV, Hits: -, queued_as: B5420C2E10, 6585 ms
2283 #TD Blocked MTA-BLOCKED, LOCAL [192.168.0.1] [192.168.0.2] <from@example.com> -> <to@sample.net>, Message-ID: <438548@example.com>, mail_id: tfgTCiyvFw, Hits: -2.54, size: 4895, 31758 ms
2284 #TD Blocked OVERSIZED, LOCAL [10.0.0.1] [10.0.0.1] <f@example.com> -> <t@sample.net>, Message-ID: <435@example.com>, mail_id: tfTivFw, Hits: -2.54, size: 444444895, 31758 ms
2285 #TD Blocked OTHER, LOCAL [10.0.0.1] [10.0.0.1] <f@example.com> -> <t@sample.net>, Message-ID: <435@example.com>, mail_id: tfTivFw, Hits: -2.54, size: 495, 31758 ms
2286 #TD Blocked TEMPFAIL, [10.0.0.2] [10.0.0.1] <user@example.com> -> <to@sample.net>, Message-ID: <200703302301.9f1899470@example.com>, mail_id: bgf52ZCNbPo, Hits: -2.586, 3908 ms
2289 #<>,<info@example.com>,Passed,Hits=-3.3,Message-ID=<200506440.1.sample.net>,Size=51458
2291 #Not-Delivered, <from@example.com> -> <to@localhost>, quarantine spam-ea32770-03, Message-ID: <BAA618FE2CB585@localhost>, Hits: 9.687
2293 # malwarepassed, malwareblocked
2295 # Virus found - quarantined|
2296 #amavisd-new-20030616
2297 # INFECTED (JS/IllWill-A), <from@[127.0.0.1]> -> <to@sample.net>, quarantine virus-20040811-207-0-03, Message-ID: <0440.5577-101@sample.net>, Hits: -
2298 # INFECTED (Exploit.HTML.IFrame, Worm.SomeFool.P), <from@sample.net> -> <to@example.com>,<to2@example.com>, quarantine qiO2ZG4K, Message-ID: <200608.5A5@mail.example.com>, Hits: -
2299 #XXX (?:(Passed|Blocked) )?INFECTED \(([^\)]+)\),[A-Z .]*(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -> [(<]([^(<]+)[)>]/o ))
2300 #XXX elsif (($action, $key, $ip, $from, $to) = ( $p1 =~ /^(?:Virus found - quarantined|(?:(Passed|Blocked) )?INFECTED) \(([^\)]+)\),[A-Z .]*(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -> [(<]([^(<]+)[(>]/o ))
2302 # the first IP is the envelope sender.
2303 if ($p1 !~ /^(CLEAN|SPAM(?:MY)?|INFECTED \(.*?\)|BANNED \(.*?\)|BAD-HEADER(?:-\d)?|UNCHECKED|UNCHECKED-ENCRYPTED|MTA-BLOCKED|OVERSIZED|OTHER|TEMPFAIL)(?: \{[^}]+})?, ([^[]+ )?(?:([^<]+) )?[<(](.*?)[>)] -> ([(<].*?[)>]), (?:.*Hits: ([-+.\d]+))(?:.* size: (\d+))?(?:.* autolearn=(\w+))?/) {
2304 inc_unmatched
('passblock');
2308 my ($ccatmajor, $pbanks, $ips, $from, $reciplist, $hits, $size, $autolearn) = ($1, $2, $3, $4, $5, $6, $7, $8);
2310 $Totals{'bytesscanned'} += $size if defined $size;
2312 #print "ccatmajor: \"$ccatmajor\", pbanks: \"$pbanks\"\n";
2313 if ($ccatmajor =~ /^(INFECTED|BANNED) \((.*)\)$/) {
2314 ($ccatmajor, $trigger) = ($1, $2);
2315 #print "\tccatmajor: \"$ccatmajor\", trigger: \"$trigger\"\n";
2318 $ccatmajor =~ s/(BAD-HEADER)-\d/$1/; # strip amavis 2.7's [:ccat|minor] BAD-HEADER sub-classification
2319 $sectkey = $ccatmajor_to_sectkey{$ccatmajor} . $action;
2320 $Totals{$sectkey}++;
2322 # Not checked by spamassassin, due to $sa_mail_body_size_limit or @bypass_spam_checks_maps
2324 # Don't increment sabypassed for INFECTED (SA intentionally not called)
2325 unless ($ccatmajor eq 'INFECTED') {
2326 # The following order is used, the first condition met decides the outcome:
2327 # 1. a virus is detected: mail is considered infected;
2328 # 2. contains banned name or type: mail is considered banned;
2329 # 3. spam level is above kill level for at least one recipient, or a sender is blacklisted: mail is considered spam;
2330 # 4. bad (invalid) headers: mail is considered as having a bad header.
2331 # Priority: VIRUS BANNED UNCHECKED SPAM SPAMMY BADH OVERSIZED MTA CLEAN
2332 $Totals{'sabypassed'}++;
2335 if ($Collecting{'spamscores'}) {
2337 if ($hits =~ /^(-?[.\d]+)([-+])([.\d]+)$/) {
2338 $hits = eval $1.$2.$3; # untaint $hits, to sum $1 and $3 values
2340 # SA not called for ccats INFECTED and BANNED (Hits: -).
2341 # UNCHECKED may have a score, so we can't distinguish Ham from Spam
2342 push @{$SpamScores{$ccatmajor_to_spamham{$ccatmajor}}}, $hits;
2346 # autolearn is available here only if enabled in amavis template
2347 if ($autolearn ne '' and $Opts{'autolearn'}) {
2348 #if ($autolearn ne '' and ($ccatmajor eq 'SPAM' or $ccatmajor eq 'CLEAN')) {
2349 # SA 2.5/2.6 : ham/spam/no
2350 # SA 3.0+ : ham/spam/no/disabled/failed/unavailable
2351 # printf "INC: autolearn: %s, %s: %d\n", $ccatmajor eq 'SPAM' ? 'Spam' : 'Ham', $autolearn, $Opts{'autolearn'};;
2352 # Priorities other than SPAM will be considered HAM for autolearn stats
2353 $Counts{'autolearn'}{$ccatmajor eq 'SPAM' ? 'Spam' : 'Ham'}{$autolearn}++;
2356 # p0f fingerprinting
2357 if (exists $p0ftags{$pid}) {
2358 my ($ip,$score,$os) = split(/\//, $p0ftags{$pid});
2359 $Counts{'p0f'}{ucfirst($ccatmajor_to_spamham{$ccatmajor})}{$os}{$ip}++;
2360 #print "Deleting p0ftag: $pid\n";
2361 delete $p0ftags{$pid};
2364 next unless ($Collecting{$sectkey});
2365 # cleanpassed never gets here...
2367 # prefer xforward IP if it exists
2368 # $ip_a => %a original SMTP session client IP address (empty if unknown, e.g. no XFORWARD)
2369 # $ip_e => %e best guess of the originator IP address collected from the Received trace
2370 my ($ip_a, $ip_e) = split(/ /, $ips, 2);
2372 $ip = $ip_a ? $ip_a : $ip_e;
2374 #print "ip: \"$ip\", ip_a: \"$ip_a\", ip_e: \"$ip_e\", from: \"$from\", reciplist: \"$reciplist\"; hits: \"$hits\"\n";
2375 $ip = '*unknown IP' if ($ip eq '');
2376 $from = '<>' if ($from eq '');
2378 # Show first recipient only, or all
2379 my @recips = split /,/, $reciplist;
2380 @recips = map { /^<(.+)>$/ } @recips;
2381 # show only first recipient
2382 $to = lc ($Opts{'first_recip_only'} ? $recips[0] : "@recips");
2384 if ($ccatmajor eq 'INFECTED') { # $ccatmajor: INFECTED malwarepassed, malwareblocked
2385 $Counts{$sectkey}{$trigger}{$to}{$ip}{$from}++;
2387 elsif ($ccatmajor eq 'BANNED') { # $ccatmajor: BANNED bannednamepassed, bannednameblocked
2388 $Counts{$sectkey}{$to}{$trigger}{$ip}{$from}++;
2390 # $ccatmajor: CLEAN | SPAM{MY} | BAD-HEADER | UNCHECKED | MTA-BLOCKED | OVERSIZED | OTHER | TEMPFAIL
2391 # cleanpassed, cleanblocked, spampassed, spamblocked, badheaderpassed, badheaderblocked
2392 # uncheckedpassed, uncheckblocked, mtapassed, mtablocked, oversizedpassed, oversizedblocked
2393 # otherpassed, otherblocked, tempfailpassed, tempfailblocked
2394 $Counts{$sectkey}{$to}{$ip}{$from}++;
2398 #XXX elsif (($action, $item, $ip, $from, $to) = ( $p1 =~ /^(?:(Blocked|Passed) )?BANNED (?:name\/type )?\((.+)\),[^[]*(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -> [(<]([^(<]+)[(>]/o))
2399 #XXXX elsif (($action, $ip, $from, $to) = ( $p1 =~ /^(?:(Passed|Blocked) )?UNCHECKED,[^[]*(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -> [(<]([^>)]*)[)>]/o ))
2400 #XXX elsif (($action, $ip, $from, $to) = ( $p1 =~ /^(?:(Passed|Blocked) )?TEMPFAIL,[^[]*(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -> [(<]([^>)]*)[)>]/o ))
2401 #XXX elsif (($action, $ip, $from, $to) = ( $p1 =~ /^(?:(Blocked|Passed) )?BAD-HEADER,[^[]*(?: \[($re_IP)\])?(?: \[$re_IP\])* [(<]([^>)]*)[)>](?: -> [(<]([^>)]+)[)>])[^:]*/o ))
2403 #BAD-HEADER, <> -> <info@example.com>, Message-ID: <200506440.1.sample.net>, Hits=-3.3 tag1=3.0 tag2=7.5 kill=7.5, tests=ALL_TRUSTED=-3.3, [10.0.0.1]
2404 } # end Passed or Blocked
2407 elsif ($p1 =~ /^FAKE SENDER, ([^:]+): ($[^,]+), (.*)$/o) {
2408 #TD FAKE SENDER, SPAM: 192.168.0.1, bogus@example.com
2409 $Totals{'fakesender'}++; next unless ($Collecting{'fakesender'});
2410 $Counts{'fakesender'}{$1}{$2}{$3}++;
2413 elsif ($p1 =~ /^p\d+ \d+(?:\/\d
+)* Content-Type
: ([^,]+)(?:, size
: [^,]+, name
: (.*))?/) {
2414 my ($ts, $name) = ($1, $2);
2415 #TD p006 1 Content-Type: multipart/mixed
2416 #TD p008 1/1 Content-Type: multipart/signed
2417 #TD p001 1/1/1 Content-Type: text/plain, size: 460 B, name:
2418 #TD p002 1/1/2 Content-Type: application/pgp-signature, size: 189 B, name:
2419 #TD p002 1/2 Content-Type: application/octet-stream, size: 3045836 B, name: abc.pdf
2420 next unless ($Collecting{'contenttype'});
2421 my ($type, $subtype) = $ts !~ '""' ? split /\//, $ts : ('unspecified', 'unspecified');
2423 $name = '' if !defined $name or $name =~ /^\s*$/;
2424 $Counts{'contenttype'}{$type}{$subtype}{$name}++;
2427 # LMTP/SMTP connection
2428 # NOTE: no longer used. size data now being obtained from Passed/Block line, as size info may not be available here
2429 #elsif (my ($size) = ($p1 =~ /^[LS]MTP:(?:\[$re_IP\])?:\d+ [^:]+: [<(](?:.*?)[>)] -> \S+ (?:SIZE=(\d+))?.*?Received: / )) {
2430 elsif ($p1 =~ /^[LS]MTP:/) {
2431 #TD LMTP::10024 /var/spool/amavis/tmp/amavis-20070119T144757-09086: <from@example.com> -> <to@sample.net> SIZE=1000 Received: from mail.sample.net ([127.0.0.1]) by localhost (mail.sample.net [127.0.0.1]) (amavisd-new, port 10024) with LMTP for <to@sample.net>; Fri, 19 Jan 2007 15:41:45 -0800 (PST)
2432 #TD SMTP:[127.0.0.1]:10024 /var/spool/amavis/tmp/amavis-20070119T144757-09086: <from@example.com> -> <to@sample.net>,<recip@sample.net> SIZE=2500000 Received: from mail.sample.net ([127.0.0.1]) by localhost (mail.sample.net [127.0.0.1]) (amavisd-new, port 10024) with LMTP for <to@sample.net>; Fri, 19 Jan 2007 15:41:45 -0800 (PST)
2433 #TD SMTP::10024 /var/lib/amavis/tmp/amavis-27-26927: <from@example.com> -> <to@example.net> Received: from localhost ([127.0.0.1]) by localhost (example.com [127.0.0.1]) (amavisd-new, port 10024) with SMTP for <to@example.net>; Sat, 7 Jun 2008 23:09:34 +0200 (CEST)
2434 #$Totals{'bytesscanned'} += $size if defined $size;
2437 #(\S+) ([^[(]+)(.*)$
2438 elsif ($p1 =~ /^OS_fingerprint: (\S+) ([-\d.]+) (\S+)(?: ([^[(]+|\[[^]]+\]))?/o) {
2439 #TD OS_fingerprint: 213.193.24.113 29.789 Linux 2.6 (newer, 1) (up: 1812 hrs), (distance 14, link: ethernet/modem)
2440 #TD OS_fingerprint: 10.47.2.155 -1.312 MYNETWORKS
2441 # Note: safe to delete entry when the final Passed/Block line occurs
2442 if ($Collecting{'p0f'}) {
2443 my ($genre,$vers) = ($3,$4);
2444 #print "p0f:\t$3\t\t$vers\n";
2445 if ($genre eq 'Windows') {
2447 $vers = $1 if $vers =~ /^(\S+) /;
2448 $genre .= ' ' . $vers;
2450 elsif ($genre eq 'UNKNOWN') {
2453 $p0ftags{$pid} = join('/', $1,$2,$genre);
2454 #print "Added PID: $pid, $p0ftags{$pid}\n";
2458 elsif ( ($reason) = ( $p1 =~ /^BAD HEADER from [^:]+: (.+)$/) or
2459 ($reason) = ( $p1 =~ /check_header: \d, (.+)$/)) {
2460 # When log_level > 1, provide additional header or MIME violations
2462 # amavisd < 2.4.0, log_level >= 1
2463 #TD BAD HEADER from <bogus@example.com>: Improper use of control character (char 0D hex) in message header 'Received': Received: example.com[10.0.0.1\r]
2464 #TD BAD HEADER from <bogus@example.com>: Non-encoded 8-bit data (char F7 hex) in message header 'Subject': Subject: \367\345\370\361 \344\351\351\362\345\365\n
2465 #TD BAD HEADER from <bogus@example.com>: MIME error: error: part did not end with expected boundary
2466 #TD BAD HEADER from (bulk ) <bogus@bounces@lists.example.com>: Non-encoded 8-bit data (char E6 hex) in message header 'Subject': Subject: spam\\346ham\\n
2467 #TD BAD HEADER from (list) <bogus@bounces@lists.example.com>: MIME error: error: part did not end with expected boundary
2468 # amavisd >= 2.4.3, log_level >= 2
2469 #TD check_header: 2, Non-encoded 8-bit data (char AE hex): Subject: RegionsNet\\256 Online Banking\\n
2470 #TD check_header: 2, Non-encoded 8-bit data (char E1 hex): From: "any user" <from\\341k@example.com>\\n
2471 #TD check_header: 3, Improper use of control character (char 0D hex): Content-type: text/html; charset=i...
2472 #TD check_header: 8, Duplicate header field: "Reply-To"
2473 #TD check_header: 8, Duplicate header field: "Subject"
2474 #TD check_header: 4, Improper folded header field made up entirely of whitespace (char 09 hex): X-Loop-Detect: 3\\n\\t\\n
2475 #TD check_header: 4, Improper folded header field made up entirely of whitespace: Received: ...8 ; Thu, 10 Jan 2008 03:41:35 +0100\\n\\t \\n
2479 if ($reason =~ /^(.*?) \((char \S+ hex)\)(.*)$/) {
2481 my ($char,$sub) = ($2,$3);
2483 $sub =~ s/^in message header '[^:]+': //;
2485 $subreason = "$char: $sub";
2487 elsif ($reason =~ /^(Improper folded header field made up entirely of whitespace):? (.*)/) {
2491 elsif ($reason =~ /^(Duplicate header field): "(.+)"$/) {
2495 elsif ($reason =~ /^(MIME error): (?:error: )?(.+)$/) {
2500 $Totals{'badheadersupp'}++; next unless ($Collecting{'badheadersupp'});
2501 $Counts{'badheadersupp'}{$reason}{$subreason}++;
2504 elsif ($p1 =~ /^truncating a message passed to SA at/) {
2505 #TD truncating a message passed to SA at 431018 bytes, orig 1875912
2506 $Totals{'truncatedmsg'}++;
2509 elsif ($p1 =~ /: spam level exceeds quarantine cutoff level/ or
2510 $p1 =~ /: cutoff, blacklisted/) {
2511 #TD do_notify_and_quarantine: spam level exceeds quarantine cutoff level 20
2512 #TD do_notify_and_quarantine: cutoff, blacklisted
2513 $Totals{'spamdiscarded'}++;
2516 elsif ( $p1 =~ /^spam_scan: (.*)$/) {
2517 #if ($1 =~ /^not wasting time on SA, message longer than/ ) {
2518 #TD spam_scan: not wasting time on SA, message longer than 409600 bytes: 1326+4115601
2519 # this causes duplicate counts, and the subsequent Passed/Blocked log line
2520 # will have "Hits: -," whereby sabypassed is incremented.
2521 #$Totals{'sabypassed'}++;
2523 # ignore other spam_scan lines
2527 elsif ( ($reason) = ( $p1 =~ /^WARN: MIME::Parser error: (.*)$/ )) {
2528 # WARN: MIME::Parser error: unexpected end of header
2529 $Totals{'mimeerror'}++; next unless ($Collecting{'mimeerror'});
2530 $Counts{'mimeerror'}{$reason}++;
2533 elsif ($p1 =~ /^WARN: address modified \((\w+)\): <(.*?)> -> <(.*)>$/) {
2534 #TD WARN: address modified (sender): <root> -> <root@>
2535 #TD WARN: address modified (recip): <root> -> <root@>
2536 #TD WARN: address modified (recip): <postmaster> -> <postmaster@>
2537 #TD WARN: address modified (recip): <"test@example.com"@> -> <"teszt@example.com">
2538 #TD WARN: address modified (sender): <fr\344om@sample.net> -> <"fr\344om"@sample.net>
2539 $Totals{'warningaddressmodified'}++; next unless ($Collecting{'warningaddressmodified'});
2540 $Counts{'warningaddressmodified'}{$1 eq 'sender' ? "Sender address" : "Recipient address"}{"$2 -> $3"}++;
2544 elsif ($p1 =~ /^NOTICE: (.*)$/) {
2546 #TD NOTICE: reconnecting in response to: err=2006, HY000, DBD::mysql::st execute failed: MySQL server has gone away at (eval 71) line 166, <GEN168> line 4.
2547 next if ($1 =~ /^Disconnected from SQL server/); # redundant
2548 next if ($1 =~ /^do_search: trying again: LDAP_OPERATIONS_ERROR/);
2549 next if ($1 =~ /^reconnecting in response to: /);
2552 if ($1 =~ /^Not sending DSN, spam level ([\d.]+ )?exceeds DSN cutoff level/) {
2553 #TD NOTICE: Not sending DSN, spam level exceeds DSN cutoff level for all recips, mail intentionally dropped
2554 $Totals{'dsnsuppressed'}++;
2555 $Counts{'dsnsuppressed'}{'DSN cutoff exceeded'}++;
2557 elsif ($1 =~ /^Not sending DSN to believed-to-be-faked sender/) {
2558 #TD NOTICE: Not sending DSN to believed-to-be-faked sender <user@example.com>, mail containing VIRUS intentionally dropped
2559 $Totals{'dsnsuppressed'}++;
2560 $Counts{'dsnsuppressed'}{'Sender likely faked'}++;
2562 elsif ($1 =~ /^DSN contains [^;]+; bounce is not bounc[ai]ble, mail intentionally dropped/) {
2563 $Totals{'dsnsuppressed'}++;
2564 $Counts{'dsnsuppressed'}{'Not bounceable'}++;
2566 elsif ($1 =~ /^UNABLE TO SEND DSN to /) {
2567 #TD NOTICE: UNABLE TO SEND DSN to <user@example.com>: 554 5.7.1 Failed, id=19838-01, from MTA([127.0.0.1]:10025): 554 5.7.1 <user@example.com>: Recipient address rejected: Access denied
2568 $Totals{'dsnsuppressed'}++;
2569 $Counts{'dsnsuppressed'}{'Unable to send'}++;
2572 elsif ($1 =~ /^Skipping (?:bad|extra) output from file\(1\)/) {
2573 #TD NOTICE: Skipping extra output from file(1): blah
2574 #TD NOTICE: Skipping bad output from file(1) at [1, p002], got: blah
2575 $Totals{'fileoutputskipped'}++;
2577 elsif (($p1) = ($1 =~ /^Virus scanning skipped: (.*)$/)) {
2578 #TD NOTICE: Virus scanning skipped: Maximum number of files (1500) exceeded at (eval 57) line 1283, <GEN212> line 1501.
2579 $Totals{'virusscanskipped'}++; next unless ($Collecting{'virusscanskipped'});
2580 $Counts{'virusscanskipped'}{strip_trace
($p1)}++;
2583 inc_unmatched
('NOTICE');
2589 elsif ($p1 =~ /^INFO: (.*)$/) {
2590 next if ($1 =~ /^unfolded \d+ illegal all-whitespace continuation line/);
2591 next if ($1 =~ /^removed bare CR/);
2593 if ($1 =~ /^truncat(ed|ing)/) {
2594 #TD INFO: truncating long header field (len=2639): X-Spam-Report: =?iso-8859-1?Q?=0A=0A*__1=2E7_SUBJECT=5FENCODED=5FTWICE_Subject=3A_MIME_e?= =?iso-885...
2595 #TD INFO: truncated 1 header line(s) longer than 998 characters
2596 $Totals{'truncatedheader'}++;
2597 } elsif ( $1 =~ /^no existing header field 'Subject', inserting it/) {
2598 $Totals{'nosubject'}++;
2600 elsif (my ($savers1, $savers2, $item) = ( $1 =~ /^(?:SA version: ([^,]+), ([^,]+), )?no optional modules: (.+)$/)) {
2601 #TD INFO: SA version: 3.1.8, 3.001008, no optional modules: DBD::mysql Mail::SpamAssassin::Plugin::DKIM Mail::SpamAssassin::Plugin::URIDetail Error
2602 next unless ($Opts{'startinfo'});
2603 if ($savers1 ne '') {
2604 $StartInfo{'sa_version'} = "$savers1 ($savers2)";
2606 foreach my $code (split / /, $item) {
2607 $StartInfo{'Code'}{'Not loaded'}{$code} = "";
2610 elsif (my ($name) = ( $1 =~ /^(unknown banned table name \S+), .+$/)) {
2611 #TD INFO: unknown banned table name 1, recip=r@example.com
2612 $Totals{'warning'}++; next unless ($Collecting{'warning'});
2613 $Counts{'warning'}{ucfirst $name}++;
2616 inc_unmatched
('INFO');
2621 elsif ( ($action,$reason,$from,$to) = ($p1 =~ /^DSN: NOTIFICATION: Action:([^,]+), ([^,]+), <(.*?)> -> <(.*?)>/)) {
2622 #TD DSN: NOTIFICATION: Action:failed, LOCAL 554 Banned, <from@example.net> -> <to@example.com>
2623 #TD DSN: NOTIFICATION: Action:delayed, LOCAL 454 Banned, <from@example.com> -> <to@example.net>
2625 $Totals{'dsnnotification'}++; next unless ($Collecting{'dsnnotification'});
2626 $Counts{'dsnnotification'}{$action}{$reason}{"$from -> $to"}++;
2629 elsif (($item, $from, $to) = ( $p1 =~ /^Quarantined message release(?: \([^)]+\))?: ([^ ]+) <(.*?)> -> (.+)$/) or
2630 ($item, $from, $to) = ( $p1 =~ /^Quarantine release ([^ ]+): overriding recips <([^>]*)> by (.+)$/)) {
2631 #TD Quarantine release arQcr95dNHaW: overriding recips <TO@EXAMPLE.COM> by <to@example.com>
2632 #TD Quarantined message release: hiyPJOsD2m9Z <from@sample.net> -> <to@example.com>
2633 #TD Quarantined message release: hiyPJOsD2m9Z <> -> <to@recipient.maildir>,<anyone@example.com>
2635 #TD Quarantined message release (miscategorized): Iu6+0u1voOA <from@example.com> -> <to@example.net>
2636 $Totals{'released'}++; next unless ($Collecting{'released'});
2637 $from = '<>' if ($from eq '');
2639 $Counts{'released'}{"\L$from"}{$to}{$item}++;
2641 elsif ($p1 =~ /^Quarantine release ([^:]+): missing X-Quarantine-ID$/) {
2642 #TD Quarantine release 7ejEBC7MThSc: missing X-Quarantine-ID
2643 $Totals{'warningnoquarantineid'}++; next unless ($Collecting{'warningnoquarantineid'});
2644 $Counts{'warningnoquarantineid'}{$1}++;
2647 elsif ( ($stage,$reason) = ($p1 =~ /^Negative SMTP resp\S* +to ([^:]+): *(.*)$/)) {
2648 #TD Negative SMTP response to data-dot (<u@example.com>): 550 5.7.1 Header Spam Rule 4
2649 $Totals{'smtpresponse'}++; next unless ($Collecting{'smtpresponse'});
2650 $Counts{'smtpresponse'}{'Negative response'}{$stage}{$reason}++;
2652 elsif ( ($stage,$reason) = ($p1 =~ /^smtp resp to ([^:]+): *(.*)$/)) {
2653 #TD smtp resp to NOOP (idle 4799.4 s): 421 4.4.2 nops.overtops.org Error: timeout exceeded
2654 #TD smtp resp to MAIL (pip): 250 2.1.0 Ok
2655 $Totals{'smtpresponse'}++; next unless ($Collecting{'smtpresponse'});
2656 $stage =~ s/ [\d.]+ s//;
2657 $Counts{'smtpresponse'}{'Response'}{$stage}{$reason}++;
2660 elsif ( ($item) = ($p1 =~ /^response to RCPT TO for <([^>]*)>: "501 Bad address syntax"/)) {
2661 #TD response to RCPT TO for <""@example.com>: "501 Bad address syntax"
2662 $Totals{'badaddress'}++; next unless ($Collecting{'badaddress'});
2663 $Counts{'badaddress'}{$item}++;
2666 # do_unip: archive extraction
2667 elsif ($p1 =~ s/^do_unzip: \S+, //) {
2668 $Totals{'archiveextract'}++; next unless ($Collecting{'archiveextract'});
2670 if ( $p1 =~ s/^\d+ members are encrypted, //) {
2671 #TD do_unzip: p003, 4 members are encrypted, none extracted, archive retained
2672 $Counts{'archiveextract'}{'Encrypted'}{$p1}++;
2674 } elsif ( $p1 =~ /^zero length members, archive retained/) {
2675 #TD do_unzip: p002, zero length members, archive retained
2676 $Counts{'archiveextract'}{'Empty member'}{''}++;
2678 } elsif ($p1 =~ s/^unsupported compr\. method: //) {
2679 #TD do_unzip: p003, unsupported compr. method: 99
2680 $Counts{'archiveextract'}{'Unsupported compression'}{$p1}++;
2683 $Counts{'archiveextract'}{'*unknown'}{$p1}++;
2687 # do_cabextract: archive extraction
2688 elsif ($p1 =~ s/^do_cabextract: //) {
2689 #TD do_cabextract: can't parse toc line: File size | Date Time | Name
2690 #TD do_cabextract: can't parse toc line: All done, no errors.
2691 $Totals{'archiveextract'}++; next unless ($Collecting{'archiveextract'});
2693 if ($p1 =~ /^([^:]+):\s*(.*)/) {
2694 $Counts{'archiveextract'}{"\u$1"}{$2}++;
2696 $Counts{'archiveextract'}{$p1}{''}++;
2700 elsif ($p1 =~ /^(?:\(!\) *)?SA TIMED OUT,/) {
2701 $Totals{'satimeout'}++;
2704 elsif ($p1 =~ /^mangling (.*)$/) {
2706 if ($p1 =~ /^by (.+?) failed: (.+?), mail will pass unmodified$/) {
2707 #TD mangling by altermine failed: SomeText, mail will pass unmodified
2708 $Totals{'defangerror'}++; next unless ($Collecting{'defangerror'});
2709 $Counts{'defangerror'}{$1}{$2}++;
2711 # other mangle message skipped
2713 #TD mangling YES: 1 (orig: 1), discl_allowed=0, <from@example.com> -> <to@sample.net>
2714 #TD mangling by built-in defanger: 1, <user@example.com>
2718 elsif ($p1 =~ /^DEFANGING MAIL: (.+)$/) {
2720 #TD DEFANGING MAIL: WARNING: possible mail bomb, NOT CHECKED FOR VIRUSES:\n Exceeded storage quota 5961070 bytes by d...
2721 #TD DEFANGING MAIL: WARNING: bad headers - Improper use of control character (char 0D hex): To: <to@example.com\\r>,\\n\\t<to@example.com>
2722 # could use instead...
2723 #do_log(1,"mangling by %s (%s) done, new size: %d, orig %d bytes", $actual_mail_mangle, $mail_mangle, $repl_size, $msginfo->msg_size);
2724 $Totals{'defanged'}++; next unless ($Collecting{'defanged'});
2725 $Counts{'defanged'}{$1}++;
2728 elsif ($p1 =~ /^PenPalsSavedFromKill [-.\d]+,/) {
2729 #TD PenPalsSavedFromKill 8.269-3.160, <ulyanov@steelpro.com.ua> -> <recipient1@recipientdomain.com>
2730 $Totals{'penpalsaved'}++;
2733 # I don't know how many variants of time outs there are... I suppose we'll fix as we go
2734 elsif (($p1 =~ /^\(!+\)([^ ]*) is taking longer than \d+ s and will be killed/) or
2735 ($p1 =~ /^\(!+\)(.*) av-scanner FAILED: timed out/) or
2736 ($p1 =~ /^(?:\(!+\))?(.*): timed out/))
2738 #TD (!)/usr/local/bin/uvscan is taking longer than 10 s and will be killed
2739 #TD (!!)NAI McAfee AntiVirus (uvscan) av-scanner FAILED: timed out
2740 #TD ClamAV-clamd: timed out, retrying (1)
2741 #TD (!)Sophie: timed out, retrying (2)
2743 $Totals{'avtimeout'}++; next unless ($Collecting{'avtimeout'});
2744 $Counts{'avtimeout'}{$1}++;
2746 elsif (($p2) = ($p1 =~ /SMTP shutdown: (.*)$/)) { # log level -1
2747 #TD SMTP shutdown: Error writing a SMTP response to the socket: Broken pipe at (eval 49) line 836, <GEN232> line 51.
2748 #TD SMTP shutdown: tempdir is to be PRESERVED: /var/amavis/tmp/amavis-20070704T095350-13145
2750 if ($p2 =~ /^tempdir is to be PRESERVED: (.*)\/([^\
/]+)$/) {
2751 $Totals{'tmppreserved'}++;
2752 $Counts{'tmppreserved'}{$1}{$2}++ if ($Collecting{'tmppreserved'});
2753 $p2 = "Preserved tempdir in $1";
2755 $Totals{'warningsmtpshutdown'}++; next unless ($Collecting{'warningsmtpshutdown'});
2756 $Counts{'warningsmtpshutdown'}{ucfirst($p2)}++;
2759 elsif (($p1 =~ /PRESERVING EVIDENCE in (.*)\/([^\
/]+)$/) or
2760 ($p1 =~ /tempdir is to be PRESERVED: (.*)\/([^\
/]+)$/)) {
2761 #TD (!)TempDir removal: tempdir is to be PRESERVED: /var/amavis/tmp/amavis-20080110T173606-05767
2763 #TD PRESERVING EVIDENCE in /var/amavis/tmp/amavis-20070704T111558-14883
2764 $Totals{'tmppreserved'}++; next unless ($Collecting{'tmppreserved'});
2765 $Counts{'tmppreserved'}{$1}{$2}++;
2768 elsif ($p1 =~ /^Open relay\? Nonlocal recips but not originating/) {
2769 $Totals{'warningsecurity'}++;
2770 $Counts{'warningsecurity'}{$p1}++ if ($Collecting{'warningsecurity'});
2773 # keep before general warnings below, so sadiag gets first crack at log
2774 # lines beginning with "(!) ...".
2775 elsif ($p1 =~ /^(?:\(!+\))?\!?SA (warn|info|error): (.*)$/) {
2776 #TD SA warn: FuzzyOcr: Cannot find executable for gocr
2777 my ($level,$msg) = ($1,$2);
2779 # XXX later, maybe break out stats on FuzzyOcr
2780 # skip "image too small" for now
2781 if ($msg =~ /^FuzzyOcr: Skipping .+, image too small$/) {
2782 #TD SA warn: FuzzyOcr: Skipping ocrad, image too small
2783 #TD SA warn: FuzzyOcr: Skipping ocrad-decolorize, image too small
2784 #$Counts{'sadiags'}{'fuzzyocr'}{'image too small'}++;
2787 elsif ($msg =~ /dns: \[\.\.\.\]/) {
2788 #TD SA info: dns: [...] ;; ADDITIONAL SECTION (1 record)
2791 # canonicalize some PIDs and IDs
2792 elsif ($msg =~ s/^pyzor: \[\d+\] error/pyzor: [<PID>] error/) {
2793 #TD SA info: pyzor: [11550] error: TERMINATED, signal 15 (000f)
2795 elsif ($msg =~ /dns: no likely matching queries for id \d+/) {
2796 $msg =~ s/\d+/<ID>/;
2798 elsif ($msg =~ /dns: no callback for id \d+/) {
2799 $msg =~ s/\d+.*$/<ID>.../;
2802 # report other SA warn's
2803 $Totals{'sadiags'}++;
2804 next unless ($Collecting{'sadiags'});
2805 $Counts{'sadiags'}{ucfirst($level)}{$msg}++;
2808 # catchall for most other warnings
2809 elsif (($p1 =~ /^\(!+\)/) or
2810 ($p1 =~ /^TROUBLE/) or
2811 ($p1 =~ /Can't (?:connect to UNIX|send to) socket/) or
2812 ($p1 =~ /: Empty result from /) or
2813 ($p1 =~ /: Error reading from socket: Connection reset by peer/) or
2814 ($p1 =~ /open\(.*\): Permission denied/) or
2815 ($p1 =~ /^_?WARN: /) or
2816 ($p1 =~ /Can't send SIG \d+ to process \[\d+\]: Operation not permitted/) or
2817 ($p1 =~ /(policy protocol: INVALID(?: AM\.PDP)? ATTRIBUTE LINE: .*)$/) or
2818 ($p1 =~ /(DKIM signature verification disabled, corresponding features not available. If not intentional.*)$/)
2821 #TD (!)loading policy bank "AM.PDP-SOCK": unknown field "0"
2822 #TD (!!)policy_server FAILED: SQL quarantine code not enabled at (eval 37) line 306, <GEN6> line 4.
2823 #TD (!!)policy_server FAILED: Can't open file /var/spool/amavis/quarantine/spam-CFJYXmeS+FLy: Permission denied at (eval 37) line 330, <GEN28> line 5.
2824 #TD ClamAV-clamd: Empty result from /var/run/clamav/clamd, retrying (1)
2825 #TDdcc open(/var/dcc/map): Permission denied
2826 #TD TROUBLE in check_mail: FAILED: Died at /usr/sbin/amavisd-maia line 2872, <GEN4> line 22.
2827 #TD TROUBLE in check_mail: spam_scan FAILED: DBD::mysql::st execute failed: MySQL server has gone away at /usr/sbin/amavisd-maia line 3786, <GEN4> line 3036.
2828 #TD TROUBLE in process_request: DBD::mysql::st execute failed: MySQL server has gone away at (eval 35) line 258, <GEN18> line 3.
2829 #TD TROUBLE in process_request: DBD::mysql::st execute failed: Lost connection to MySQL server during query at (eval 35) line 258, <GEN3> line 3.
2830 #TD TROUBLE in process_request: Can't call method "disconnect" on an undefined value at /usr/sbin/amavisd-maia line 2895, <GEN4> line 22.
2831 #TD TROUBLE: recipient not done: <to@example.com> smtp response ...
2832 #TD (!!)TROUBLE in process_request: Can't create file /var/amavis/tmp/amavis-98/email.txt: File exists at /usr/local/sbin/amavisd line 4774, <GEN12> line 4.
2833 #TD TROUBLE: lookup table is an unknown object: object ...
2834 #TD (!) policy protocol: INVALID ATTRIBUTE LINE: /var/spool/courier/tmp/114528/D967099\n
2835 #TD (!) policy protocol: INVALID AM.PDP ATTRIBUTE LINE: /var/spool/courier/tmp/114528/D967099\n
2836 #TD _WARN: bayes: cannot open bayes databases /var/spool/amavis/.spamassassin/bayes_* R/W: lock failed: Interrupted system call\n
2838 $p1 =~ s/^\(!+\)s*//;
2840 if ($p1 =~ /^WARN: (Using cpio instead of pax .*)$/) {
2841 #TD (!)WARN: Using cpio instead of pax can be a security risk; please add: $pax='pax'; to amavisd.conf and check that the pax(1) utility is available on the system!
2842 $Totals{'warningsecurity'}++;
2843 $Counts{'warningsecurity'}{$1}++ if ($Collecting{'warningsecurity'});
2847 $p1 =~ s/, retrying\s+\(\d+\)$//;
2850 # canonicalize variations of the same message
2851 $p1 =~ s/^run_av \(([^,]+), built-in i\/f\)/$1/;
2852 $p1 =~ s/ av-scanner FAILED: CODE\(0x[^)]+\)/:/;
2853 $p1 =~ s/^(.+: Too many retries to talk to \S+) .*/$1/;
2855 if (($p1 =~ /(\S+): Can't (?:connect|send) to (?:UNIX )?(.*)$/) or
2856 ($p1 =~ /(\S+): (Too many retries to talk to .*)$/))
2859 #TD (!)ClamAV-clamd: Can't connect to UNIX socket /var/run/clamav/clamd.socket: No such file or directory, retrying (2)
2860 #TD (!)ClamAV-clamd: Can't connect to UNIX socket /var/run/clamav/clamd: Connection refused, retrying (2)
2861 #TD ClamAV-clamd: Can't connect to UNIX socket /var/run/clamav/clamd: Connection refused, retrying (1)
2862 #TD ClamAV-clamd: Can't send to socket /var/run/clamav/clamd: Transport endpoint is not connected, retrying (1)
2863 #TD Sophie: Can't send to socket /var/run/sophie: Transport endpoint is not connected, retrying (1)
2864 #TD (!)run_av (Sophie, built-in i/f): Too many retries to talk to /var/run/sophie (timed out) at (eval 55) line 310, <GEN16> line 16.
2865 #TD (!)run_av (ClamAV-clamd, built-in i/f): Too many retries to talk to /var/run/clamav/clamd.socket (Can't connect to UNIX socket /var/run/clamav/clamd.socket: No such file or directory) at (eval 52) line 310.
2866 #TD (!!)ClamAV-clamd av-scanner FAILED: CODE(0x804fa08) Too many retries to talk to /var/run/clamav/clamd.socket (Can't connect to UNIX socket /var/run/clamav/clamd.socket: No such file or directory) at (eval 52) line 310. at (eval 52) line 511.
2867 #TD (!!)Sophie av-scanner FAILED: CODE(0x814fd24) Too many retries to talk to /var/run/sophie (timed out) at (eval 55) line 310, <GEN16> line 16. at (eval 55) line 511, <GEN16> line 16.
2869 $Totals{'avconnectfailure'}++;
2870 $Counts{'avconnectfailure'}{$1}{ucfirst($2)}++ if ($Collecting{'avconnectfailure'});
2874 # simplify or canonicalize variations of the same message
2875 $p1 =~ s/^TROUBLE(:| in) //;
2876 $p1 =~ s/^_?WARN: //;
2877 $p1 =~ s/Can't create file \S+: (.+)$/Can't create file: $1/;
2878 $p1 =~ s/Can't send SIG \d+ to process \[\d+\]/Can't send SIG to process/;
2880 $Totals{'warning'}++; next unless ($Collecting{'warning'});
2881 $Counts{'warning'}{$p1}++;
2884 # Begin forced warnings: Keep this code below warning catchall
2885 elsif ($p1 =~ /^lookup_sql: /) {
2886 #TD lookup_sql: 2006, MySQL server has gone away
2887 $Totals{'warningsql'}++; next unless ($Collecting{'warningsql'});
2888 $Counts{'warningsql'}{'SQL died'}++;
2890 } elsif (($reason,$item) = ($p1 =~ /^connect_to_sql: ([^']+) '\S+': (.*?)(?: \(\d+\))?$/) or
2891 ($item,$reason) = ($p1 =~ /^lookup_sql_field\((.*)\) \(WARN: (no such field in the SQL table)\)/)) {
2892 #TD connect_to_sql: unable to connect to DSN 'DBI:mysql:maia:sqlhost1.example.com': Lost connection to MySQL server during query
2893 #TD connect_to_sql: unable to connect to DSN 'DBI:mysql:maia:sqlhost2.example.com': Can't connect to MySQL server on 'sqlhost2.example.com' (111)
2894 #TD lookup_sql_field(id) (WARN: no such field in the SQL table), "from@example.com" result=undef
2895 $Totals{'warningsql'}++; next unless ($Collecting{'warningsql'});
2896 $Counts{'warningsql'}{ucfirst("$reason: $item")}++;
2898 # End forced warnings
2901 elsif ( ($p2) = ($p1 =~ /^(?:\(!\)\s*)?PANIC, (.*)$/)) {
2902 #TD PANIC, PANIC, SA produced a clone process of [19122], TERMINATING CLONE [19123]
2904 $Totals{'panic'}++; next unless ($Collecting{'panic'});
2905 $Counts{'panic'}{$p2}++;
2910 elsif ( $p1 =~ /^Requesting process rundown after fatal error$/) {
2911 #TD Requesting process rundown after fatal error
2912 $Totals{'fatal'}++; next unless ($Collecting{'fatal'});
2913 $Counts{'fatal'}{$p1}++;
2916 } elsif (($reason) = ($p1 =~ /^(missing message body; fatal error)/) or
2917 ($reason) = ($p1 =~ /^(try to start dccifd)/)) {
2918 $Totals{'dccerror'}++; next unless ($Collecting{'dccerror'});
2919 $Counts{'dccerror'}{ucfirst($reason)}++;
2921 elsif ($p1 =~ /^continue not asking DCC \d+ seconds after failure/) {
2922 $Totals{'dccerror'}++; next unless ($Collecting{'dccerror'});
2923 $Counts{'dccerror'}{'Continue not asking DCC after failure'}++;
2925 elsif ($p1 =~ /^no DCC answer from (\S+) after \d+ ms$/) {
2926 $Totals{'dccerror'}++; next unless ($Collecting{'dccerror'});
2927 $Counts{'dccerror'}{"No answer from $1"}++;
2930 elsif ( ($reason, $from, $to) = ($p1 =~ /^skip local delivery\((\d+)\): <(.*?)> -> <(.*?)>$/)) {
2931 $Totals{'localdeliveryskipped'}++; next unless ($Collecting{'localdeliveryskipped'});
2932 $from = '<>' if ($from eq '');
2933 $reason = $reason == 1 ? "No localpart" : $reason == 2 ? "Local alias is null" : "Other";
2934 $Counts{'localdeliveryskipped'}{$reason}{$from}{$to}++;
2937 # hard and soft whitelisted/blacklisted
2938 elsif ($p1 =~ /^wbl: (.*)$/) {
2939 # ignore wbl entries, can't think of good way to reliably summarize.
2940 # and 'black or whitelisted by all' makes using by-white or -black list
2941 # groupings impossible
2946 # TD wbl: black or whitelisted by all recips
2947 next if ($p1 =~ /^black or whitelisted/); # not clear how to report this, so skip
2948 next if ($p1 =~ /^checking sender/); # ll 4
2949 next if ($p1 =~ /^(LDAP) query keys/); # ll 5
2950 next if ($p1 =~ /^(LDAP) recip/); # ll 5
2951 next if ($p1 =~ /^recip <[^>]*> (?:black|white)listed sender/); # ll 5
2953 # lookup order: SQL, LDAP, static
2954 if ($p1 =~ s/^\(SQL\) recip <[^>]*>//) {
2955 next if ($p1 =~ /^, \S+ matches$/); # ll 5
2956 next if ($p1 =~ /^, rid=/); # ll 4
2957 next if ($p1 =~ /^ is neutral to sender/); # ll 5
2958 next if ($p1 =~ /^ (?:white|black)listed sender </); # ll 5
2960 #wbl: (SQL) recip <%s> whitelisted sender <%s>, '. unexpected wb field value
2963 # wbl: (SQL) soft-(white|black)listed (%s) sender <%s> => <%s> (rid=%s)', $val, $sender, $recip, $user_id);
2964 # multiple senders: message sender, then "from", etc.
2966 # wbl: soft-(white|black)listed (%s) sender <%s> => <%s>,
2968 #TD wbl: whitelisted sender <sender@example.com>
2969 #TD wbl: soft-whitelisted (-3) sender <from@example.com> => <to@sample.net>, recip_key="."
2970 #TD wbl: whitelisted by user@example.com, but not by all, sender <bounces@example.net>, <user@example.org>
2971 # wbl: (whitelisted|blacklisted|black or whitelisted by all recips|(white|black)listed by xxx,yyy,... but not by all) sender %s
2973 if ($p1 =~ /^(?:\(SQL\) )?(?:(soft)-)?((?:white|black)listed)(?: \([^)]+\))? sender <([^>]*)>/) {
2974 my ($type,$list,$sender) = ($1,$2,$3);
2975 $Totals{$list}++; next unless ($Collecting{$list});
2976 $type = $type ? 'Soft' : 'Hard' ;
2977 my ($localpart, $domainpart) = split (/@/, lc $sender);
2978 ($localpart, $domainpart) = ($sender, '*unspecified') if ($domainpart eq '');
2979 $Counts{$list}{$type}{$domainpart}{$localpart}++;
2982 inc_unmatched
('wbl');
2988 # XXX: WHITELISTED or BLACKLISTED should be caught in SPAM tag above
2989 elsif (($p1 =~ /^white_black_list: whitelisted sender/) or
2990 ($p1 =~ /.* WHITELISTED/) ) {
2991 $Totals{'whitelisted'}++;
2993 } elsif (($p1 =~ /^white_black_list: blacklisted sender/) or
2994 ( $p1 =~ /.* BLACKLISTED/) ) {
2995 $Totals{'blacklisted'}++;
2997 } elsif ($p1 =~ /^Turning AV infection into a spam report: score=([^,]+), (.+)$/) {
2998 #TD Turning AV infection into a spam report: score=4.1, AV:Sanesecurity.ScamL.375.UNOFFICIAL=4.1
2999 #TD Turning AV infection into a spam report: score=3.4, AV:Sanesecurity.Phishing.Cur.180.UNOFFICIAL=3.1,AV:Sanesecurity.Phishing.Cur.180.UNOFFICIAL=3.4
3000 #BAT.Backdoor.Poisonivy.E178-SecuriteInfo.com
3002 next unless ($Collecting{'malwaretospam'});
3003 #my $score_max = $1;
3004 my @list = split (/,/, $2);
3005 @list = unique_list
(\
@list);
3007 my ($name,$score) = split (/=/,$_);
3009 my $type = $name =~ s/\.UNOFFICIAL$// ? 'Unofficial' : 'Official';
3010 # strip trailing numeric variant (...Phishing.Cur.863)
3011 my $variant = $name =~ s/([.-]\d+)$// ? $1 : '*invariant';
3012 $Counts{'malwaretospam'}{$type}{$name}{$variant}{$score}++
3015 # The virus_scan line reports only the one virus name when more than one scanner detects a virus.
3016 # Use instead the ask_av and run_av lines (see below)
3018 #} elsif ( my ($malware, $scanners) = ($p1 =~ /virus_scan: \(([^)]+)\), detected by \d+ scanners: (.*)$/ )) {
3019 #TD virus_scan: (HTML.Phishing.Bank-43), detected by 1 scanners: ClamAV-clamd
3020 #TD virus_scan: (Worm.SomeFool.D, Worm.SomeFool.D), detected by 1 scanners: ClamAV-clamd
3021 #TD virus_scan: (Trojan.Downloader.Small-9993), detected by 2 scanners: ClamAV-clamd, NAI McAfee AntiVirus (uvscan)
3022 # foreach (split /, /, $scanners) {
3023 # #$Totals{'malwarebyscanner'}++; # No summary output: redundant w/malwarepassed,malwareblocked}
3024 # $Counts{'malwarebyscanner'}{"$_"}{$malware}++;
3027 } elsif ($p1 =~ /^(?:ask_av|run_av) (.*)$/) {
3028 next unless ($Collecting{'malwarebyscanner'});
3030 if (my ($scanner, $name) = ($1 =~ /^\((.+)\):(?: [^:]+)? INFECTED: ([^,]+)/)) {
3031 #TD ask_av (ClamAV-clamd): /var/amavis/tmp/amavis-20070830T070403-13776/parts INFECTED: Email.Malware.Sanesecurity.07082700
3032 #TD run_av (NAI McAfee AntiVirus (uvscan)): INFECTED: W32/Zhelatin.gen!eml, W32/Zhelatin.gen!eml
3033 my $type = $name =~ s/\.UNOFFICIAL$// ? 'Unofficial' : 'Official';
3035 if ($name =~ s/([.-]\d+)$//) { # strip trailing numeric variant (...Phishing.Cur.863)
3038 $Counts{'malwarebyscanner'}{$scanner}{$type}{$name}{$variant}++;
3040 # currently ignoring other ask_av or run_av lines
3043 # Extra Modules loaded at runtime
3044 #TD extra modules loaded after daemonizing/chrooting: Mail/SPF/Query.pm
3045 elsif (($item) = ( $p1 =~ /^extra modules loaded(?: after daemonizing(?:\/chrooting
)?)?: (.+)$/)) {
3046 #TD extra modules loaded: PerlIO.pm, PerlIO/scalar.pm
3047 foreach my $code (split /, /, $item) {
3048 #TD extra modules loaded: unicore/lib/gc_sc/Digit.pl, unicore/lib/gc_sc/SpacePer.pl
3049 # avoid useless reporting of pseudo-modules which can't be pre-loaded once
3050 unless ($code =~ m
#^unicore/lib/#) {
3051 $Totals{'extramodules'}++;
3052 $Counts{'extramodules'}{$code}++ if ($Collecting{'extramodules'});
3057 } elsif (my ($total,$report) = ( $p1 =~ /^(?:size: \d+, )?TIMING \[total (\d+) ms(?:, [^]]+)?\] - (.+)$/)) {
3058 next if ($report =~ /^got data/); # skip amavis release timing
3059 #TD TIMING [total 5808 ms] - SMTP greeting: 5 (0%)0, SMTP LHLO: 1 (0%)0, SMTP pre-MAIL: 2 (0%)0, SMTP pre-DATA-flush: 5 (0%)0, SMTP DATA: 34 (1%)1, check_init: 1 (0%)1
3060 # older format, maia mailguard
3061 #TD TIMING [total 3795 ms] - SMTP EHLO: 1 (0%), SMTP pre-MAIL: 0 (0%), maia_read_system_config: 1 (0%), maia_get_mysql_size_limit: 0 (0%), SA check: 3556 (94%), rundown: 0 (0%)
3063 # .... size: 3815, TIMING [total 1901 ms, cpu 657 ms] - ...
3066 # Timing line is incomplete - let's report it
3067 if ($p1 !~ /\d+ \(\d+%\)\d+$/ and $p1 !~ /\d+ \(\d+%\)$/) {
3068 inc_unmatched
('timing');
3072 if ($Opts{'timings'}) {
3073 my @pairs = split(/[,:] /, $report);
3074 while (my ($key,$value) = @pairs) {
3076 my ($ms) = ($value =~ /^([\d.]+) /);
3077 # maintain a per-test list of timings
3078 push @{$Timings{$key}}, $ms;
3079 shift @pairs; shift @pairs;
3081 push @TimingsTotals, $total;
3084 } elsif ((($total,$report) = ( $p1 =~ /^TIMING-SA total (\d+) ms - (.+)$/ )) or
3085 (($total,$report) = ( $p1 =~ /^TIMING-SA \[total (\d+) ms, cpu \d+ ms\] - (.+)$/ ))) {
3086 #TIMING-SA [total 3219 ms, cpu 432 ms] - parse: 6 (0.2%), ext
3087 #TD TIMING-SA total 5478 ms - parse: 1.69 (0.0%), extract_message_metadata: 16 (0.3%), get_uri_detail_list: 2 (0.0%), tests_pri_-1000: 25 (0.4%), tests_pri_-950: 0.67 (0.0%), tests_pri_-900: 0.83 (0.0%), tests_pri_-400: 19 (0.3%), check_bayes: 17 (0.3%), tests_pri_0: 5323 (97.2%), check_spf: 12 (0.2%), poll_dns_idle: 0.81 (0.0%), check_dkim_signature: 1.50 (0.0%), check_razo r2: 5022 (91.7%), check_dcc: 192 (3.5%), check_pyzor: 0.02 (0.0%), tests_pri_500: 9 (0.2%), tests_pri_1000: 24 (0.4%), total_awl: 23 (0.4%), check_awl: 10 (0.2%), update_awl: 8 (0.1%), learn: 36 (0.7%), get_report: 1.77 (0.0%)
3089 # Timing line is incomplete - let's report it
3090 if ($p1 !~ /[\d.]+ \([\d.]+%\)[\d.]+$/ and $p1 !~ /[\d.]+ \([\d.]+%\)$/) {
3091 inc_unmatched
('timing-sa');
3094 if ($Opts{'sa_timings'}) {
3095 my @pairs = split(/[,:] /, $report);
3096 while (my ($key,$value) = @pairs) {
3098 my ($ms) = ($value =~ /^([\d.]+) /);
3099 # maintain a per-SA test list of timings
3100 push @{$TimingsSA{$key}}, $ms;
3101 shift @pairs; shift @pairs;
3103 push @TimingsSATotals, $total;
3106 # Bounce killer: 2.6+
3107 } elsif ($p1 =~ /^bounce (.*)$/) {
3108 #TD bounce killed, <user@example.com> -> <to@example.net>, from: user@example.com, message-id: <CA8E335-CC-2EFB@example.com>, return-path: <user@example.com>
3109 #TD bounce rescued by domain, <user@example.com> -> <to@example.net>, from: user@example.com, message-id: <CA8E335-CC-2EFB@example.com>, return-path: <user@example.com>
3110 #TD bounce rescued by originating, <user@example.com> -> <to@example.net>, from: user@example.com, message-id: <CA8E335-CC-2EFB@example.com>, return-path: <user@example.com>
3111 #TD bounce rescued by: pen pals disabled, <user@example.com> -> <to@example.net>, from: user@example.com, message-id: <CA8E335-CC-2EFB@example.com>, return-path: <user@example.com>
3114 if ($p2 =~ /^killed, <(.+?)> -> /) {
3115 $Totals{'bouncekilled'}++;
3116 $Counts{'bouncekilled'}{$1 eq '' ? '<>' : $1}++ if ($Collecting{'bouncekilled'});
3118 elsif ($p2 =~ /^rescued by ([^,]+), <(.+?)> -> /) {
3119 # note: ignores "rescued by: pen pals disabled"
3120 $Totals{'bouncerescued'}++;
3121 $Counts{'bouncerescued'}{'By ' . $1}{$2 eq '' ? '<>' : $2}++ if ($Collecting{'bouncerescued'});
3123 elsif ($p2 =~ /^unverifiable, <(.+?)> -> /) {
3124 # note: ignores "rescued by: pen pals disabled"
3125 $Totals{'bounceunverifiable'}++;
3126 $Counts{'bounceunverifiable'}{$1 eq '' ? '<>' : $1}++ if ($Collecting{'bounceunverifiable'});
3128 #TD bounce unverifiable, <postmaster@nurturegood.com> -> <dave@davewolloch.com>
3129 #TD bounce unverifiable, <> -> <Dave@davewolloch.com>
3133 elsif (my ($suffix, $info) = ( $p1 =~ /^Internal decoder for (\.\S*)\s*(?:\(([^)]*)\))?$/)) {
3134 #TD Internal decoder for .gz (backup, not used)
3135 #TD Internal decoder for .zip
3136 next unless ($Opts{'startinfo'});
3137 $StartInfo{'Decoders'}{'Internal'}{$suffix} = $info;
3140 elsif (($suffix, $decoder) = ( $p1 =~ /^No decoder for\s+(\.\S*)\s*(?:tried:\s+(.*))?$/)) {
3141 #TD No decoder for .tnef tried: tnef
3143 #TD No decoder for .doc
3144 next unless ($Opts{'startinfo'});
3145 $StartInfo{'Decoders'}{'None'}{$suffix} = "tried: " . ($decoder ? $decoder : "unknown");
3148 elsif (($suffix, $decoder) = ( $p1 =~ /^Found decoder for\s+(\.\S*)\s+at\s+(.*)$/)) {
3149 #TD Found decoder for .bz2 at /usr/bin/bzip2 -d
3150 #TD Found decoder for .bz2 at /usr/bin/7za (backup, not used)
3151 next unless ($Opts{'startinfo'});
3152 $StartInfo{'Decoders'}{'External'}{$suffix} = exists $StartInfo{'Decoders'}{'External'}{$suffix} ?
3153 join '; ', $StartInfo{'Decoders'}{'External'}{$suffix}, $decoder : $decoder;
3157 elsif (my ($tier, $scanner, $location) = ( $p1 =~ /^Found (primary|secondary) av scanner (.+) at (.+)$/)) {
3158 #TD Found primary av scanner NAI McAfee AntiVirus (uvscan) at /usr/local/bin/uvscan
3159 #TD Found secondary av scanner ClamAV-clamscan at /usr/local/bin/clamscan
3160 next unless ($Opts{'startinfo'});
3161 $StartInfo{'AVScanner'}{"\u$tier"}{$scanner} = $location;
3163 } elsif (($tier, $scanner, $location) = ( $p1 =~ /^No (primary|secondary) av scanner: (.+)$/)) {
3164 #TD No primary av scanner: CyberSoft VFind
3165 next unless ($Opts{'startinfo'});
3166 $StartInfo{'AVScanner'}{"\u$tier (not found)"}{$scanner} = '';
3168 } elsif ( (($tier, $scanner) = ( $p1 =~ /^Using internal av scanner code for \(([^)]+)\) (.+)$/)) or
3169 (($tier, $scanner) = ( $p1 =~ /^Using (.*) internal av scanner code for (.+)$/))) {
3170 #TD Using internal av scanner code for (primary) ClamAV-clamd
3171 #TD Using primary internal av scanner code for ClamAV-clamd
3172 next unless ($Opts{'startinfo'});
3173 $StartInfo{'AVScanner'}{"\u$tier internal"}{$scanner} = '';
3175 # (Un)Loaded code, protocols, etc.
3176 } elsif (my ($code, $loaded) = ( $p1 =~ /^(\S+)\s+(?:proto? |base |protocol )?\s*(?:code)?\s+((?:NOT )?loaded)$/)) {
3177 next unless ($Opts{'startinfo'});
3178 $StartInfo{'Code'}{"\u\L$loaded"}{$code} = "";
3180 } elsif (my ($module, $vers) = ( $p1 =~ /^Module (\S+)\s+(.+)$/)) {
3181 #TD Module Amavis::Conf 2.086
3182 next unless ($Opts{'startinfo'});
3183 $StartInfo{'Code'}{'Loaded'}{$module} = $vers;
3185 } elsif (($module, my $families) = ( $p1 =~ /^socket module (\S+),\s+(.+)$/)) {
3186 #TD socket module IO::Socket::IP, protocol families available: INET, INET6
3187 next unless ($Opts{'startinfo'});
3188 $StartInfo{'Code'}{'Loaded'}{$module} = $families;
3190 } elsif (($code, $location) = ( $p1 =~ /^Found \$(\S+)\s+at\s+(.+)$/)) {
3191 #TD Found $file at /usr/bin/file
3192 #TD Found $uncompress at /usr/bin/gzip -d
3193 next unless ($Opts{'startinfo'});
3194 $StartInfo{'Code'}{'Loaded'}{$code} = $location;
3196 } elsif (($code, $location) = ( $p1 =~ /^No \$(\S+),\s+not using it/)) {
3197 #TD No $dspam, not using it
3198 next unless ($Opts{'startinfo'});
3199 $StartInfo{'Code'}{'Not loaded'}{$code} = $location;
3201 } elsif (($code, $location) = ( $p1 =~ /^No ext program for\s+([^,]+), (tried: .+)/)) {
3202 #TD No ext program for .kmz, tried: 7za, 7z
3203 #TD No ext program for .F, tried: unfreeze, freeze -d, melt, fcat
3204 next unless ($Opts{'startinfo'});
3205 $StartInfo{'Code'}{'Not found'}{$code} = $location;
3208 } elsif ( $p1 =~ /^starting\.\s+(.+) at \S+ (?:amavisd-new-|Maia Mailguard )([^,]+),/) {
3209 #TD starting. /usr/local/sbin/amavisd at mailhost.example.com amavisd-new-2.5.0 (20070423), Unicode aware, LANG="C"
3210 #TD starting. /usr/sbin/amavisd-maia at vwsw02.eon.no Maia Mailguard 1.0.2, Unicode aware, LANG=en_US.UTF-8
3211 next unless ($Opts{'startinfo'});
3212 %StartInfo = () if !exists $StartInfo{'Logging'};
3213 $StartInfo{'ampath'} = $1;
3214 $StartInfo{'amversion'} = $2;
3216 } elsif ( $p1 =~ /^config files read: (.*)$/) {
3217 #TD config files read: /etc/amavisd.conf, /etc/amavisd-overrides.conf
3218 next unless ($Opts{'startinfo'});
3219 $StartInfo{'Configs'} = "$1";
3221 } elsif ($p1 =~ /^Creating db in ([^;]+); [^,]+, (.*)$/) {
3222 #TD Creating db in /var/spool/amavis/db/; BerkeleyDB 0.31, libdb 4.4
3223 next unless ($Opts{'startinfo'});
3224 $StartInfo{'db'} = "$1\t($2)";
3226 } elsif ($p1 =~ /^BerkeleyDB-based Amavis::Cache not available, using memory-based local cache$/) {
3227 #TD BerkeleyDB-based Amavis::Cache not available, using memory-based local cache
3228 next unless ($Opts{'startinfo'});
3229 $StartInfo{'db'} = "BerkeleyDB\t(memory-based cache: Amavis::Cache unavailable)";
3231 } elsif (my ($log) = ($p1 =~ /^logging initialized, log (level \d+, (?:STDERR|syslog: \S+))/)) {
3232 next unless ($Opts{'startinfo'});
3233 %StartInfo = (); # first amavis log entry, clear out previous start info
3234 $StartInfo{'Logging'} = $log;
3236 } elsif (( $p1 =~ /^(:?perl=[^,]*, )?user=([^,]*), EUID: (\d+) [(](\d+)[)];\s+group=([^,]*), EGID: ([\d ]+)[(]([\d ]+)[)]/)) {
3238 #next unless ($Opts{'startinfo'});
3239 #$StartInfo{'IDs'}{'user'} = $1;
3240 #$StartInfo{'IDs'}{'euid'} = $2;
3241 #$StartInfo{'IDs'}{'uid'} = $3;
3242 #$StartInfo{'IDs'}{'group'} = $4;
3243 #$StartInfo{'IDs'}{'egid'} = $5;
3244 #$StartInfo{'IDs'}{'gid'} = $6;
3245 } elsif ($p1 =~ /^after_chroot_init: EUID: (\d+) [(](\d+)[)]; +EGID: ([\d ]+)[(]([\d ]+)[)]/) {
3246 #TD after_chroot_init: EUID: 999 (999); EGID: 54322 54322 54322 (54322 54322 54322)
3249 } elsif ($p1 =~ /^SpamAssassin debug facilities: (.*)$/) {
3250 next unless ($Opts{'startinfo'});
3251 $StartInfo{'sa_debug'} = $1;
3254 } elsif ($p1 =~ /^SpamAssassin loaded plugins: (.*)$/) {
3255 #TD SpamAssassin loaded plugins: AWL, AutoLearnThreshold, Bayes, BodyEval, Check, DCC, DKIM, DNSEval, HTMLEval, HTTPSMismatch, Hashcash, HeaderEval, ImageInfo, MIMEEval, MIMEHeader, Pyzor, Razor2, RelayEval, ReplaceTags, SPF, SpamCop, URIDNSBL, URIDetail, URIEval, VBounce, WLBLEval, WhiteListSubject
3256 next unless ($Opts{'startinfo'});
3257 map { $StartInfo{'SAPlugins'}{'Loaded'}{$_} = '' } split(/, /, $1);
3259 } elsif (($p2) = ( $p1 =~ /^Net::Server: (.*)$/ )) {
3260 next unless ($Opts{'startinfo'});
3261 if ($p2 =~ /^.*starting! pid\((\d+)\)/) {
3262 #TD Net::Server: 2007/05/02-11:05:24 Amavis (type Net::Server::PreForkSimple) starting! pid(4405)
3263 $StartInfo{'Server'}{'pid'} = $1;
3264 } elsif ($p2 =~ /^Binding to UNIX socket file (.*) using/) {
3265 #TD Net::Server: Binding to UNIX socket file /var/spool/amavis/amavisd.sock using SOCK_STREAM
3266 $StartInfo{'Server'}{'socket'} = $1;
3267 } elsif ($p2 =~ /^Binding to TCP port (\d+) on host (.*)$/) {
3268 #TD Net::Server: Binding to TCP port 10024 on host 127.0.0.1
3269 $StartInfo{'Server'}{'ip'} = "$2:$1";
3270 } elsif ($p2 =~ /^Setting ([ug]id) to "([^"]+)"$/) {
3271 $StartInfo{'Server'}{$1} = $2;
3272 #TD Net::Server: Setting gid to "91 91"
3273 #TD Net::Server: Setting uid to "91"
3278 # higher debug level or rare messages skipped last
3279 elsif (! check_ignore_list
($p1, @ignore_list_final)) {
3280 inc_unmatched
('final');
3284 ########################################
3285 # Final tabulations, and report printing
3288 # spamblocked includes spamdiscarded; adjust here
3289 $Totals{'spamblocked'} -= $Totals{'spamdiscarded'};
3292 #Totals: Blocked/Passed totals
3293 $Totals{'totalblocked'} += $Totals{$_} foreach (
3309 $Totals{'totalpassed'} += $Totals{$_} foreach (
3324 # Priority: VIRUS BANNED UNCHECKED SPAM SPAMMY BADH OVERSIZED MTA CLEAN
3327 $Totals{'totalmalware'} += $Totals{$_} foreach (
3328 qw(malwarepassed malwareblocked));
3330 $Totals{'totalbanned'} += $Totals{$_} foreach (
3331 qw(bannednamepassed bannednameblocked));
3333 $Totals{'totalunchecked'} += $Totals{$_} foreach (
3334 qw(uncheckedpassed uncheckedblocked));
3336 $Totals{'totalspammy'} += $Totals{$_} foreach (
3337 qw(spammypassed spammyblocked));
3339 $Totals{'totalbadheader'} += $Totals{$_} foreach (
3340 qw(badheaderpassed badheaderblocked));
3342 $Totals{'totaloversized'} += $Totals{$_} foreach (
3343 qw(oversizedpassed oversizedblocked));
3345 $Totals{'totalmta'} += $Totals{$_} foreach (
3346 qw(mtapassed mtablocked));
3348 $Totals{'totalclean'} += $Totals{$_} foreach (
3349 qw(cleanpassed cleanblocked));
3351 $Totals{'totalother'} += $Totals{$_} foreach (
3352 qw(tempfailpassed tempfailblocked otherpassed otherblocked));
3354 $Totals{'totalspam'} += $Totals{$_} foreach (
3355 qw(spampassed spamblocked spamdiscarded totalspammy));
3357 # everything lower priority than SPAMMY is considered HAM
3358 $Totals{'totalham'} += $Totals{$_} foreach (
3359 qw(totalbadheader totaloversized totalmta totalclean));
3361 $Totals{'totalmsgs'} += $Totals{$_} foreach (
3362 qw(totalmalware totalbanned totalunchecked totalspam totalham totalother));
3364 # Print the summary report if any key has non-zero data.
3365 # Note: must explicitely check for any non-zero data,
3366 # as Totals always has some keys extant.
3368 if ($Opts{'summary'}) {
3369 for (keys %Totals) {
3371 print_summary_report
(@Sections);
3377 # Print the detailed report, if detail is sufficiently high
3379 if ($Opts{'detail'} >= 5) {
3380 print_detail_report
(@Sections);
3381 printAutolearnReport
;
3382 printSpamScorePercentilesReport
;
3383 printSpamScoreFrequencyReport
;
3385 printTimingsReport
("Scan Timing Percentiles", \
%Timings, \
@TimingsTotals, $Opts{'timings'});
3386 printTimingsReport
("SA Timing Percentiles", \
%TimingsSA, \
@TimingsSATotals, 0-$Opts{'sa_timings'});
3387 printStartupInfoReport
if ($Opts{'detail'} >= 10);
3392 #print Dumper(\%p0ftags);
3393 #print Dumper($Counts{'p0f'});
3396 # Finally, print any unmatched lines
3398 print_unmatched_report
();
3400 # Evaluates a given line against the list of ignore patterns.
3402 sub check_ignore_list
($ \
@) {
3403 my ($line, $listref) = @_;
3405 foreach (@$listref) {
3406 return 1 if $line =~ /$_/;
3413 # Spam score percentiles report
3416 ==================================================================================
3417 Spam Score Percentiles 0% 50% 90% 95% 98% 100%
3418 ----------------------------------------------------------------------------------
3419 Score Spam (100) 6.650 21.906 34.225 36.664 38.196 42.218
3420 Score Ham (1276) -17.979 -2.599 0.428 2.261 3.472 6.298
3421 ==================================================================================
3423 sub printSpamScorePercentilesReport
{
3424 return unless ($Opts{'score_percentiles'} and keys %SpamScores);
3426 #printf "Scores $_ (%d): @{$SpamScores{$_}}\n", scalar @{$SpamScores{$_}} foreach keys %SpamScores;
3428 my @percents = split /[\s,]+/, $Opts{'score_percentiles'};
3429 my $myfw2 = $fw2 - 1;
3431 print "\n", $sep1 x
$fw1, $sep1 x
$fw2 x
@percents;
3432 printf "\n%-${fw1}s" . "%${myfw2}s%%" x
@percents , "Spam Score Percentiles", @percents;
3433 print "\n", $sep2 x
$fw1, $sep2 x
$fw2 x
@percents;
3435 foreach my $ccat (keys %SpamScores) {
3436 @sorted = sort { $a <=> $b } @{$SpamScores{$ccat}};
3437 @p = get_percentiles
(@sorted, @percents);
3438 printf "\n%-${fw1}s" . "%${fw2}.3f" x
scalar (@p), "Score \u$ccat (" . scalar (@sorted) . ')', @p;
3441 print "\n", $sep1 x
$fw1, $sep1 x
$fw2 x
@percents, "\n";
3444 # Spam score frequency report
3447 ======================================================================================================
3448 Spam Score Frequency <= -10 <= -5 <= 0 <= 5 <= 10 <= 20 <= 30 > 30
3449 ------------------------------------------------------------------------------------------------------
3450 Hits (1376) 29 168 921 170 29 33 1 25
3451 Percent of Hits 2.11% 12.21% 66.93% 12.35% 2.11% 2.40% 0.07% 1.82%
3452 ======================================================================================================
3454 sub printSpamScoreFrequencyReport
{
3455 return unless ($Opts{'score_frequencies'} and keys %SpamScores);
3458 push @scores, @{$SpamScores{$_}} foreach (keys %SpamScores);
3459 my $nscores = scalar @scores;
3461 my @sorted = sort { $a <=> $b } @scores;
3462 my @buckets = sort { $a <=> $b } split /[\s,]+/, $Opts{'score_frequencies'};
3463 push @buckets, $buckets[-1] + 1;
3464 #print "Scores: @sorted\n";
3466 my @p = get_frequencies
(@sorted, @buckets);
3468 my @ranges = ( 0 ) x
@buckets;
3469 my $last = @buckets - 1;
3470 $ranges[0] = sprintf "%${fw2}s", " <= $buckets[0]";
3471 $ranges[-1] = sprintf "%${fw2}s", " > $buckets[-2]";
3472 for my $i (1 .. @buckets - 2) {
3473 $ranges[$i] = sprintf "%${fw2}s", " <= $buckets[$i]";
3476 print "\n", $sep1 x
$fw1, $sep1 x
$fw2 x
@buckets;
3477 printf "\n%-${fw1}s" . "%-${fw2}s" x
@buckets , "Spam Score Frequency", @ranges;
3478 print "\n", $sep2 x
$fw1, $sep2 x
$fw2 x
@buckets;
3479 printf "\n%-${fw1}s" . "%${fw2}d" x
scalar (@p), "Hits ($nscores)", @p;
3480 my $myfw2 = $fw2 - 1;
3481 printf "\n%-${fw1}s" . "%${myfw2}.2f%%" x
scalar (@p), "Percent of Hits", map {($_ / $nscores) * 100.0; } @p;
3482 print "\n", $sep1 x
$fw1, $sep1 x
$fw2 x
@buckets, "\n";
3485 # SpamAssassin rules report
3488 ===========================================================================
3489 SpamAssassin Rule Hits: Spam
3490 ---------------------------------------------------------------------------
3491 Rank Hits % Msgs % Spam % Ham Score Rule
3492 ---- ---- ------ ------ ----- ----- ----
3493 1 44 81.48% 93.62% 0.00% 1.961 URIBL_BLACK
3494 2 44 81.48% 93.62% 14.29% 0.001 HTML_MESSAGE
3495 3 42 77.78% 89.36% 0.00% 2.857 URIBL_JP_SURBL
3496 4 38 70.37% 80.85% 14.29% 2.896 RCVD_IN_XBL
3497 5 37 68.52% 78.72% 0.00% 2.188 RCVD_IN_BL_SPAMCOP_NET
3499 ===========================================================================
3501 ===========================================================================
3502 SpamAssassin Rule Hits: Ham
3503 ---------------------------------------------------------------------------
3504 Rank Hits % Msgs % Spam % Ham Score Rule
3505 ---- ---- ------ ------ ----- ----- ----
3506 1 5 9.26% 2.13% 71.43% 0.001 STOX_REPLY_TYPE
3507 2 4 7.41% 0.00% 57.14% -0.001 SPF_PASS
3508 3 4 7.41% 6.38% 57.14% - AWL
3509 4 1 1.85% 0.00% 14.29% 0.303 TVD_RCVD_SINGLE
3510 5 1 1.85% 25.53% 14.29% 0.1 RDNS_DYNAMIC
3512 ===========================================================================
3514 sub printSARulesReport
{
3515 return unless (keys %{$Counts{'sarules'}});
3519 sub getSAHitsReport
($ $) {
3520 my ($type, $topn) = @_;
3524 return if ($topn eq '0'); # topn can be numeric, or the string "all"
3526 for (sort { $Counts{'sarules'}{$type}{$b} <=> $Counts{'sarules'}{$type}{$a} } keys %{$Counts{'sarules'}{$type}}) {
3528 # only show top n lines; all when topn is "all"
3529 if ($topn ne 'all' and $i > $topn) {
3530 push @report, "...\n";
3533 my $n = $Counts{'sarules'}{$type}{$_};
3534 my $nham = $Counts{'sarules'}{'Ham'}{$_};
3535 my $nspam = $Counts{'sarules'}{'Spam'}{$_};
3536 # rank, count, % msgs, % spam, % ham
3537 push @report, sprintf "%4d %8d %6.2f%% %6.2f%% %6.2f%% %s\n",
3540 $Totals{'totalmsgs'} == 0 ? 0 : 100.0 * $n / $Totals{'totalmsgs'},
3541 $Totals{'totalspam'} == 0 ? 0 : 100.0 * $nspam / $Totals{'totalspam'},
3542 $Totals{'totalham'} == 0 ? 0 : 100.0 * $nham / $Totals{'totalham'},
3544 my $len = length($report[-1]) - 1;
3545 $maxlen = $len if ($len > $maxlen);
3548 if (scalar @report) {
3549 print "\n", $sep1 x
$maxlen, "\n";
3550 print "SpamAssassin Rule Hits: $type\n";
3551 print $sep2 x
$maxlen, "\n";
3552 print "Rank Hits % Msgs % Spam % Ham Score Rule\n";
3553 print "---- ---- ------ ------ ----- ----- ----\n";
3555 print $sep1 x
$maxlen, "\n";
3559 my ($def_limit_spam, $def_limit_ham) = split /[\s,]+/, $Defaults{'sarules'};
3560 my ($limit_spam, $limit_ham) = split /[\s,]+/, $Opts{'sarules'};
3561 $limit_spam = $def_limit_spam if $limit_spam eq '';
3562 $limit_ham = $def_limit_ham if $limit_ham eq '';
3564 getSAHitsReport
('Spam', $limit_spam);
3565 getSAHitsReport
('Ham', $limit_ham);
3568 # Autolearn report, only available if enabled in amavis $log_templ template
3571 ======================================================================
3572 Autolearn Msgs Spam Ham % Msgs % Spam % Ham
3573 ----------------------------------------------------------------------
3574 Spam 36 36 0 66.67% 76.60% 0.00%
3575 Ham 2 0 2 3.70% 0.00% 28.57%
3576 No 7 4 3 12.96% 8.51% 42.86%
3577 Disabled 6 6 0 11.11% 12.77% 0.00%
3578 Failed 2 1 1 3.70% 2.13% 14.29%
3579 ----------------------------------------------------------------------
3580 Totals 53 47 6 98.15% 100.00% 85.71%
3581 ======================================================================
3583 sub printAutolearnReport
{
3584 #print "printAutolearnReport:\n" if ($Opts{'debug'});
3585 return unless (keys %{$Counts{'autolearn'}});
3588 our ($nhamtotal, $nspamtotal);
3590 sub getAutolearnReport
($) {
3594 # SA 2.5/2.6 : ham/spam/no
3595 # SA 3.0+ : ham/spam/no/disabled/failed/unavailable
3596 for (qw(spam ham no disabled failed unavailable)) {
3598 next unless (exists $Counts{'autolearn'}{'Spam'}{$_} or exists $Counts{'autolearn'}{'Ham'}{$_});
3599 #print "printAutolearnReport: type: $_\n" if ($Opts{'debug'});
3601 my $nham = exists $Counts{'autolearn'}{'Ham'}{$_} ? $Counts{'autolearn'}{'Ham'}{$_} : 0;
3602 my $nspam = exists $Counts{'autolearn'}{'Spam'}{$_} ? $Counts{'autolearn'}{'Spam'}{$_} : 0;
3603 my $nboth = $nham + $nspam;
3604 $nhamtotal += $nham; $nspamtotal += $nspam;
3605 # type, nspam, nham, % msgs, % spam, % ham
3606 push @report, sprintf "%-13s %9d %9d %9d %6.2f%% %6.2f%% %6.2f%%\n",
3611 $Totals{'totalmsgs'} == 0 ? 0 : 100.0 * $nboth / $Totals{'totalmsgs'},
3612 $Totals{'totalspam'} == 0 ? 0 : 100.0 * $nspam / $Totals{'totalspam'},
3613 $Totals{'totalham'} == 0 ? 0 : 100.0 * $nham / $Totals{'totalham'};
3615 my $len = length($report[-1]) - 1;
3616 $maxlen = $len if ($len > $maxlen);
3621 my @report_spam = getAutolearnReport
('Spam');
3623 if (scalar @report_spam) {
3624 print "\n", $sep1 x
$maxlen, "\n";
3625 print "Autolearn Msgs Spam Ham % Msgs % Spam % Ham\n";
3626 print $sep2 x
$maxlen, "\n";
3628 print $sep2 x
$maxlen, "\n";
3630 printf "%-13s %9d %9d %9d %6.2f%% %6.2f%% %6.2f%%\n",
3632 $nspamtotal + $nhamtotal,
3635 $Totals{'totalmsgs'} == 0 ? 0 : 100.0 * ($nspamtotal + $nhamtotal) / $Totals{'totalmsgs'},
3636 $Totals{'totalspam'} == 0 ? 0 : 100.0 * $nspamtotal / $Totals{'totalspam'},
3637 $Totals{'totalham'} == 0 ? 0 : 100.0 * $nhamtotal / $Totals{'totalham'};
3638 print $sep1 x
$maxlen, "\n";
3643 # Timings percentiles report, used for amavis message scanning and spamassassin timings
3645 ========================================================================================================================
3646 Scan Timing Percentiles % Time Total (ms) 0% 5% 25% 50% 75% 95% 100%
3647 ------------------------------------------------------------------------------------------------------------------------
3648 AV-scan-2 (3) 69.23% 7209.00 2392.00 2393.50 2399.50 2407.00 2408.50 2409.70 2410.00
3649 SA check (2) 19.74% 2056.00 942.00 950.60 985.00 1028.00 1071.00 1105.40 1114.00
3650 SMTP DATA (3) 5.49% 572.00 189.00 189.20 190.00 191.00 191.50 191.90 192.00
3651 AV-scan-1 (3) 0.82% 85.00 11.00 12.60 19.00 27.00 37.00 45.00 47.00
3653 ------------------------------------------------------------------------------------------------------------------------
3654 Total 10413.00 2771.00 2867.10 3251.50 3732.00 3821.00 3892.20 3910.00
3655 ========================================================================================================================
3657 ========================================================================================================================
3658 SA Timing Percentiles % Time Total (ms) 0% 5% 25% 50% 75% 95% 100%
3659 ------------------------------------------------------------------------------------------------------------------------
3660 tests_pri_0 (1) 97.17% 5323.00 5323.00 5323.00 5323.00 5323.00 5323.00 5323.00 5323.00
3661 check_razor2 (1) 91.68% 5022.00 5022.00 5022.00 5022.00 5022.00 5022.00 5022.00 5022.00
3662 check_dcc (1) 3.50% 192.00 192.00 192.00 192.00 192.00 192.00 192.00 192.00
3663 learn (1) 0.66% 36.00 36.00 36.00 36.00 36.00 36.00 36.00 36.00
3664 tests_pri_-1000 (1) 0.46% 25.00 25.00 25.00 25.00 25.00 25.00 25.00 25.00
3666 ------------------------------------------------------------------------------------------------------------------------
3667 Total 5478.00 5478.00 5478.00 5478.00 5478.00 5478.00 5478.00 5478.00
3668 ========================================================================================================================
3670 sub printTimingsReport
($$$$) {
3671 my ($title, $timingsref, $totalsref, $cutoff) = @_;
3672 my @tkeys = keys %$timingsref;
3673 return unless scalar @tkeys;
3675 my (@p, @sorted, %perkey_totals, @col_subtotals);
3676 my ($pcnt,$max_pcnt,$max_rows,$time_total_actual,$time_total_hypo,$subtotal_pcnt);
3677 my @percents = split /[\s,]+/, $Opts{'timings_percentiles'};
3678 my $header_footer = $sep1 x
50 . ($sep1 x
10) x
@percents;
3679 my $header_end = $sep2 x
50 . ($sep2 x
10) x
@percents;
3680 my $title_width = '-28';
3682 print "\n$header_footer\n";
3683 printf "%${title_width}s %6s %13s" ." %8s%%" x
@percents , $title, "% Time", "Total (ms)", @percents;
3684 print "\n$header_end\n";
3686 # Sum the total time for each timing key
3687 foreach my $key (@tkeys) {
3688 foreach my $timeval (@{$$timingsref{$key}}) {
3689 $perkey_totals{$key} += $timeval;
3693 # Sum total time spent scanning
3694 map {$time_total_actual += $_} @$totalsref;
3696 # cutoff value used to limit the number of rows of output
3697 # positive cutoff is a percentage of cummulative time
3698 # negative cutoff limits number of rows
3700 $max_pcnt = $cutoff != 100 ? $cutoff : 150; # 150% avoids roundoff errors
3703 $max_rows = -$cutoff;
3706 # sort each timing key's values, required to compute the list of percentiles
3707 for (sort { $perkey_totals{$b} <=> $perkey_totals{$a} } @tkeys) {
3708 last if (($max_rows and $rows >= $max_rows) or ($max_pcnt and $subtotal_pcnt >= $max_pcnt));
3710 $pcnt = ($perkey_totals{$_} / $time_total_actual) * 100,
3711 @sorted = sort { $a <=> $b } @{$$timingsref{$_}};
3712 @p = get_percentiles
(@sorted, @percents);
3714 $subtotal_pcnt += $pcnt;
3715 printf "%${title_width}s %6.2f%% %13.2f" . " %9.2f" x
scalar (@p) . "\n",
3716 $_ . ' (' . scalar(@{$$timingsref{$_}}) . ')', # key ( number of elements )
3717 $pcnt, # percent of total time
3718 #$perkey_totals{$_} / 1000, # total time for this test
3719 $perkey_totals{$_}, # total time for this test
3720 #map {$_ / 1000} @p; # list of percentiles
3721 @p; # list of percentiles
3724 print "...\n" if ($rows != scalar @tkeys);
3726 print "$header_end\n";
3727 # actual total time as reported by amavis
3728 @sorted = sort { $a <=> $b } @$totalsref;
3729 @p = get_percentiles
(@sorted, @percents);
3730 printf "%${title_width}s %13.2f" . " %9.2f" x
scalar (@p) . "\n",
3732 #$time_total_actual / 1000,
3734 #map {$_ / 1000} @p;
3737 print "$header_footer\n";
3740 # Most recent startup info report
3742 sub printStartupInfoReport
{
3744 return unless (keys %StartInfo);
3746 sub print2col
($ $) {
3747 my ($label,$val) = @_;
3748 printf "%-50s %s\n", $label, $val;
3751 print "\nAmavis Startup\n";
3753 print2col
(" Amavis", $StartInfo{'ampath'}) if (exists $StartInfo{'ampath'});
3754 print2col
(" Version", $StartInfo{'amversion'}) if (exists $StartInfo{'amversion'});
3755 print2col
(" PID", $StartInfo{'Server'}{'pid'}) if (exists $StartInfo{'Server'}{'pid'});
3756 print2col
(" Socket", $StartInfo{'Server'}{'socket'}) if (exists $StartInfo{'Server'}{'socket'});
3757 print2col
(" TCP port", $StartInfo{'Server'}{'ip'}) if (exists $StartInfo{'Server'}{'ip'});
3758 print2col
(" UID", $StartInfo{'Server'}{'uid'}) if (exists $StartInfo{'Server'}{'uid'});
3759 print2col
(" GID", $StartInfo{'Server'}{'gid'}) if (exists $StartInfo{'Server'}{'gid'});
3760 print2col
(" Logging", $StartInfo{'Logging'}) if (exists $StartInfo{'Logging'});
3761 print2col
(" Configuration Files", $StartInfo{'Configs'}) if (exists $StartInfo{'Configs'});
3762 print2col
(" SpamAssassin", $StartInfo{'sa_version'}) if (exists $StartInfo{'sa_version'});
3763 print2col
(" SpamAssassin Debug Facilities", $StartInfo{'sa_debug'}) if (exists $StartInfo{'sa_debug'});
3764 print2col
(" Database", $StartInfo{'db'}) if (exists $StartInfo{'db'});
3765 #if (keys %{$StartInfo{'IDs'}}) {
3766 # print " Process startup user/group:\n";
3767 # print " User: $StartInfo{'IDs'}{'user'}, EUID: $StartInfo{'IDs'}{'euid'}, UID: $StartInfo{'IDs'}{'uid'}\n";
3768 # print " Group: $StartInfo{'IDs'}{'group'}, EGID: $StartInfo{'IDs'}{'egid'}, GID: $StartInfo{'IDs'}{'gid'}\n";
3771 sub print_modules
($ $) {
3772 my ($key, $label) = @_;
3774 foreach (sort keys %{$StartInfo{$key}}) {
3776 foreach my $module (sort keys %{$StartInfo{$key}{$_}}) {
3777 if ($StartInfo{$key}{$_}{$module}) {
3778 print2col
(" " . $module, $StartInfo{$key}{$_}{$module});
3781 print2col
(" " . $module, "");
3786 print_modules
('AVScanner', 'Antivirus scanners');
3787 print_modules
('Code', 'Code, modules and external programs');
3788 print_modules
('Decoders', 'Decoders');
3789 print_modules
('SAPlugins', 'SpamAssassin plugins');
3792 # Initialize the Getopts option list. Requires the Section table to
3795 sub init_getopts_table
() {
3796 print "init_getopts_table: enter\n" if $Opts{'debug'} & D_ARGS
;
3798 init_getopts_table_common
(@supplemental_reports);
3800 add_option
('first_recip_only!');
3801 add_option
('show_first_recip_only=i', sub { $Opts{'first_recip_only'} = $_[1]; 1;});
3802 add_option
('startinfo!');
3803 add_option
('show_startinfo=i', sub { $Opts{'startinfo'} = $_[1]; 1; });
3804 add_option
('by_ccat_summary!');
3805 add_option
('show_by_ccat_summary=i', sub { $Opts{'by_ccat_summary'} = $_[1]; 1; });
3806 add_option
('noscore_percentiles', \
&triway_opts
);
3807 add_option
('score_percentiles=s', \
&triway_opts
);
3808 add_option
('noscore_frequencies', \
&triway_opts
);
3809 add_option
('score_frequencies=s', \
&triway_opts
);
3810 add_option
('nosa_timings', sub { $Opts{'sa_timings'} = 0; 1; });
3811 add_option
('sa_timings=i');
3812 add_option
('sa_timings_percentiles=s');
3813 add_option
('notimings', sub { $Opts{'timings'} = 0; 1; });
3814 add_option
('timings=i');
3815 add_option
('timings_percentiles=s');
3816 add_option
('nosarules', \
&triway_opts
);
3817 add_option
('sarules=s', \
&triway_opts
);
3818 #add_option ('nop0f', \&triway_opts);
3819 #add_option ('p0f=s', \&triway_opts);
3820 add_option
('autolearn!');
3821 add_option
('show_autolearn=i', sub { $Opts{'autolearn'} = $_[1]; 1; });
3824 # Builds the entire @Section table used for data collection
3826 # Each Section entry has as many as six fields:
3828 # 1. Section array reference
3829 # 2. Key to %Counts, %Totals accumulator hashes, and %Collecting hash
3830 # 3. Output in Detail report? (must also a %Counts accumulator)
3831 # 4. Numeric output format specifier for Summary report
3832 # 5. Section title for Summary and Detail reports
3833 # 6. A hash to a divisor used to calculate the percentage of a total for that key
3835 # Use begin_section_group/end_section_group to create groupings around sections.
3837 # Sections can be freely reordered if desired, but maintain proper group nesting.
3839 sub build_sect_table
() {
3840 print "build_sect_table: enter\n" if $Opts{'debug'} & D_SECT
;
3843 # References to these are used in the Sections table below; we'll predeclare them.
3844 $Totals{'totalmsgs'} = 0;
3846 # Place configuration and critical errors first
3848 # SECTIONREF, NAME, DETAIL, FMT, TITLE, DIVISOR
3849 begin_section_group
($S, 'warnings');
3850 add_section
($S, 'fatal', 1, 'd', '*Fatal');
3851 add_section
($S, 'panic', 1, 'd', '*Panic');
3852 add_section
($S, 'warningsecurity', 1, 'd', '*Warning: Security risk');
3853 add_section
($S, 'avtimeout', 1, 'd', '*Warning: Virus scanner timeout');
3854 add_section
($S, 'avconnectfailure', 1, 'd', '*Warning: Virus scanner connection failure');
3855 add_section
($S, 'warningsmtpshutdown', 1, 'd', '*Warning: SMTP shutdown');
3856 add_section
($S, 'warningsql', 1, 'd', '*Warning: SQL problem');
3857 add_section
($S, 'warningaddressmodified', 1, 'd', '*Warning: Email address modified');
3858 add_section
($S, 'warningnoquarantineid', 1, 'd', '*Warning: Message missing X-Quarantine-ID header');
3859 add_section
($S, 'warning', 1, 'd', 'Miscellaneous warnings');
3860 end_section_group
($S, 'warnings');
3862 begin_section_group
($S, 'scanned', "\n");
3863 add_section
($S, 'totalmsgs', 0, 'd', [ 'Total messages scanned', '-' ], \
$Totals{'totalmsgs'});
3864 add_section
($S, 'bytesscanned', 0, 'Z', 'Total bytes scanned'); # Z means print scaled as in 1k, 1m, etc.
3865 end_section_group
($S, 'scanned', $sep1);
3868 # Priority: VIRUS BANNED UNCHECKED SPAM SPAMMY BADH OVERSIZED MTA CLEAN
3869 begin_section_group
($S, 'passblock', "\n");
3870 begin_section_group
($S, 'blocked', "\n");
3871 add_section
($S, 'totalblocked', 0, 'd', [ 'Blocked', '-' ], \
$Totals{'totalmsgs'});
3872 add_section
($S, 'malwareblocked', 1, 'd', ' Malware blocked', \
$Totals{'totalmsgs'});
3873 add_section
($S, 'bannednameblocked', 1, 'd', ' Banned name blocked', \
$Totals{'totalmsgs'});
3874 add_section
($S, 'uncheckedblocked', 1, 'd', ' Unchecked blocked', \
$Totals{'totalmsgs'});
3875 add_section
($S, 'spamblocked', 1, 'd', ' Spam blocked', \
$Totals{'totalmsgs'});
3876 add_section
($S, 'spamdiscarded', 0, 'd', ' Spam discarded (no quarantine)', \
$Totals{'totalmsgs'});
3877 add_section
($S, 'spammyblocked', 1, 'd', ' Spammy blocked', \
$Totals{'totalmsgs'});
3878 add_section
($S, 'badheaderblocked', 1, 'd', ' Bad header blocked', \
$Totals{'totalmsgs'});
3879 add_section
($S, 'oversizedblocked', 1, 'd', ' Oversized blocked', \
$Totals{'totalmsgs'});
3880 add_section
($S, 'mtablocked', 1, 'd', ' MTA blocked', \
$Totals{'totalmsgs'});
3881 add_section
($S, 'cleanblocked', 1, 'd', ' Clean blocked', \
$Totals{'totalmsgs'});
3882 add_section
($S, 'tempfailblocked', 1, 'd', ' Tempfail blocked', \
$Totals{'totalmsgs'});
3883 add_section
($S, 'otherblocked', 1, 'd', ' Other blocked', \
$Totals{'totalmsgs'});
3884 end_section_group
($S, 'blocked');
3886 begin_section_group
($S, 'passed', "\n");
3887 add_section
($S, 'totalpassed', 0, 'd', [ 'Passed', '-' ], \
$Totals{'totalmsgs'});
3888 add_section
($S, 'malwarepassed', 1, 'd', ' Malware passed', \
$Totals{'totalmsgs'});
3889 add_section
($S, 'bannednamepassed', 1, 'd', ' Banned name passed', \
$Totals{'totalmsgs'});
3890 add_section
($S, 'uncheckedpassed', 1, 'd', ' Unchecked passed', \
$Totals{'totalmsgs'});
3891 add_section
($S, 'spampassed', 1, 'd', ' Spam passed', \
$Totals{'totalmsgs'});
3892 add_section
($S, 'spammypassed', 1, 'd', ' Spammy passed', \
$Totals{'totalmsgs'});
3893 add_section
($S, 'badheaderpassed', 1, 'd', ' Bad header passed', \
$Totals{'totalmsgs'});
3894 add_section
($S, 'oversizedpassed', 1, 'd', ' Oversized passed', \
$Totals{'totalmsgs'});
3895 add_section
($S, 'mtapassed', 1, 'd', ' MTA passed', \
$Totals{'totalmsgs'});
3896 add_section
($S, 'cleanpassed', 1, 'd', ' Clean passed', \
$Totals{'totalmsgs'});
3897 add_section
($S, 'tempfailpassed', 1, 'd', ' Tempfail passed', \
$Totals{'totalmsgs'});
3898 add_section
($S, 'otherpassed', 1, 'd', ' Other passed', \
$Totals{'totalmsgs'});
3899 end_section_group
($S, 'passed');
3900 end_section_group
($S, 'passblock', $sep1);
3902 if ($Opts{'by_ccat_summary'}) {
3903 # begin level 1 group
3904 begin_section_group
($S, 'by_ccat', "\n");
3906 # begin level 2 groupings
3907 begin_section_group
($S, 'malware', "\n"); # level 2
3908 add_section
($S, 'totalmalware', 0, 'd', [ 'Malware', '-' ], \
$Totals{'totalmsgs'});
3909 add_section
($S, 'malwarepassed', 0, 'd', ' Malware passed', \
$Totals{'totalmsgs'});
3910 add_section
($S, 'malwareblocked', 0, 'd', ' Malware blocked', \
$Totals{'totalmsgs'});
3911 end_section_group
($S, 'malware');
3913 begin_section_group
($S, 'banned', "\n");
3914 add_section
($S, 'totalbanned', 0, 'd', [ 'Banned', '-' ], \
$Totals{'totalmsgs'});
3915 add_section
($S, 'bannednamepassed', 0, 'd', ' Banned file passed', \
$Totals{'totalmsgs'});
3916 add_section
($S, 'bannednameblocked', 0, 'd', ' Banned file blocked', \
$Totals{'totalmsgs'});
3917 end_section_group
($S, 'banned');
3919 begin_section_group
($S, 'unchecked', "\n");
3920 add_section
($S, 'totalunchecked', 0, 'd', [ 'Unchecked', '-' ], \
$Totals{'totalmsgs'});
3921 add_section
($S, 'uncheckedpassed', 0, 'd', ' Unchecked passed', \
$Totals{'totalmsgs'});
3922 add_section
($S, 'uncheckedblocked', 0, 'd', ' Unchecked blocked', \
$Totals{'totalmsgs'});
3923 end_section_group
($S, 'unchecked');
3925 begin_section_group
($S, 'spam', "\n");
3926 add_section
($S, 'totalspam', 0, 'd', [ 'Spam', '-' ], \
$Totals{'totalmsgs'});
3927 add_section
($S, 'spammypassed', 0, 'd', ' Spammy passed', \
$Totals{'totalmsgs'});
3928 add_section
($S, 'spammyblocked', 0, 'd', ' Spammy blocked', \
$Totals{'totalmsgs'});
3929 add_section
($S, 'spampassed', 0, 'd', ' Spam passed', \
$Totals{'totalmsgs'});
3930 add_section
($S, 'spamblocked', 0, 'd', ' Spam blocked', \
$Totals{'totalmsgs'});
3931 add_section
($S, 'spamdiscarded', 0, 'd', ' Spam discarded (no quarantine)', \
$Totals{'totalmsgs'});
3932 end_section_group
($S, 'spam');
3934 begin_section_group
($S, 'ham', "\n");
3935 add_section
($S, 'totalham', 0, 'd', [ 'Ham', '-' ], \
$Totals{'totalmsgs'});
3936 add_section
($S, 'badheaderpassed', 0, 'd', ' Bad header passed', \
$Totals{'totalmsgs'});
3937 add_section
($S, 'badheaderblocked', 0, 'd', ' Bad header blocked', \
$Totals{'totalmsgs'});
3938 add_section
($S, 'oversizedpassed', 0, 'd', ' Oversized passed', \
$Totals{'totalmsgs'});
3939 add_section
($S, 'oversizedblocked', 0, 'd', ' Oversized blocked', \
$Totals{'totalmsgs'});
3940 add_section
($S, 'mtapassed', 0, 'd', ' MTA passed', \
$Totals{'totalmsgs'});
3941 add_section
($S, 'mtablocked', 0, 'd', ' MTA blocked', \
$Totals{'totalmsgs'});
3942 add_section
($S, 'cleanpassed', 0, 'd', ' Clean passed', \
$Totals{'totalmsgs'});
3943 add_section
($S, 'cleanblocked', 0, 'd', ' Clean blocked', \
$Totals{'totalmsgs'});
3944 end_section_group
($S, 'ham');
3946 begin_section_group
($S, 'other', "\n");
3947 add_section
($S, 'totalother', 0, 'd', [ 'Other', '-' ], \
$Totals{'totalmsgs'});
3948 add_section
($S, 'tempfailpassed', 0, 'd', ' Tempfail passed', \
$Totals{'totalmsgs'});
3949 add_section
($S, 'tempfailblocked', 0, 'd', ' Tempfail blocked', \
$Totals{'totalmsgs'});
3950 add_section
($S, 'otherpassed', 0, 'd', ' Other passed', \
$Totals{'totalmsgs'});
3951 add_section
($S, 'otherblocked', 0, 'd', ' Other blocked', \
$Totals{'totalmsgs'});
3952 end_section_group
($S, 'other');
3953 # end level 2 groupings
3956 end_section_group
($S, 'by_ccat', $sep1);
3959 begin_section_group
($S, 'misc', "\n");
3960 add_section
($S, 'virusscanskipped', 1, 'd', 'Virus scan skipped');
3961 add_section
($S, 'sabypassed', 0, 'd', 'SpamAssassin bypassed');
3962 add_section
($S, 'satimeout', 0, 'd', 'SpamAssassin timeout');
3963 add_section
($S, 'released', 1, 'd', 'Released from quarantine');
3964 add_section
($S, 'defanged', 1, 'd', 'Defanged');
3965 add_section
($S, 'truncatedheader', 0, 'd', 'Truncated headers > 998 characters');
3966 add_section
($S, 'truncatedmsg', 0, 'd', 'Truncated message passed to SpamAssassin');
3967 add_section
($S, 'tagged', 0, 'd', 'Spam tagged');
3968 add_section
($S, 'smtpresponse', 1, 'd', 'SMTP response');
3969 add_section
($S, 'badaddress', 1, 'd', 'Bad address syntax');
3970 add_section
($S, 'fakesender', 1, 'd', 'Fake sender');
3971 add_section
($S, 'archiveextract', 1, 'd', 'Archive extraction problem');
3972 add_section
($S, 'dsnsuppressed', 1, 'd', 'DSN suppressed');
3973 add_section
($S, 'dsnnotification', 1, 'd', 'DSN notification (debug supplemental)');
3974 add_section
($S, 'bouncekilled', 1, 'd', 'Bounce killed');
3975 add_section
($S, 'bouncerescued', 1, 'd', 'Bounce rescued');
3976 add_section
($S, 'bounceunverifiable', 1, 'd', 'Bounce unverifiable');
3977 add_section
($S, 'nosubject', 0, 'd', 'Subject header inserted');
3978 add_section
($S, 'whitelisted', 1, 'd', 'Whitelisted');
3979 add_section
($S, 'blacklisted', 1, 'd', 'Blacklisted');
3980 add_section
($S, 'penpalsaved', 1, 'd', 'Penpals saved from kill');
3981 add_section
($S, 'tmppreserved', 1, 'd', 'Preserved temporary directory');
3982 add_section
($S, 'dccerror', 1, 'd', 'DCC error');
3983 add_section
($S, 'mimeerror', 1, 'd', 'MIME error');
3984 add_section
($S, 'defangerror', 1, 'd', 'Defang error');
3985 add_section
($S, 'badheadersupp', 1, 'd', 'Bad header (debug supplemental)');
3986 add_section
($S, 'fileoutputskipped', 0, 'd', 'File(1) output skipped');
3987 add_section
($S, 'localdeliveryskipped', 1, 'd', 'Local delivery skipped');
3988 add_section
($S, 'extramodules', 1, 'd', 'Extra code modules loaded at runtime');
3989 add_section
($S, 'malwarebyscanner', 1, 'd', 'Malware by scanner');
3990 add_section
($S, 'malwaretospam', 1, 'd', 'Malware to spam conversion');
3991 add_section
($S, 'contenttype', 1, 'd', 'Content types');
3992 add_section
($S, 'bayes', 1, 'd', 'Bayes probability');
3993 add_section
($S, 'p0f', 1, 'd', 'p0f fingerprint');
3994 add_section
($S, 'sadiags', 1, 'd', 'SpamAssassin diagnostics');
3995 end_section_group
($S, 'misc');
3997 print "build_sect_table: exit\n" if $Opts{'debug'} & D_SECT
;
4000 # XXX create array of defaults for detail <5, 5-9, >10
4001 sub init_defaults
() {
4002 map { $Opts{$_} = $Defaults{$_} unless exists $Opts{$_} } keys %Defaults;
4003 if (! $Opts{'standalone'}) {
4004 # LOGWATCH these take affect if no env present (eg. nothing in conf file)
4005 # 0 to 4 nostartinfo, notimings, nosarules, score_frequencies=0, score_percentiles=0, noautolearn
4006 # 5 to 9 nostartinfo, timings=95, sarules = 20 20, score_frequencies=defaults, score_percentiles=defaults, autolearn
4007 # 10 + startinfo, timings=100, sarules = all all score_frequencies=defaults, score_percentiles=defaults, autolearn
4009 if ($Opts{'detail'} < 5) { # detail 0 to 4, disable all supplimental reports
4010 $Opts{'autolearn'} = 0;
4012 $Opts{'timings'} = 0;
4013 $Opts{'sa_timings'} = 0;
4014 $Opts{'sarules'} = 0;
4015 $Opts{'startinfo'} = 0;
4016 $Opts{'score_frequencies'} = '';
4017 $Opts{'score_percentiles'} = '';
4019 elsif ($Opts{'detail'} < 10) { # detail 5 to 9, disable startinfo report
4020 $Opts{'startinfo'} = 0;
4022 else { # detail 10 and up, full reports
4023 #$Opts{'p0f'} = 'all all';
4024 $Opts{'timings'} = 100;
4025 $Opts{'sa_timings'} = 100;
4026 $Opts{'sarules'} = 'all all';
4031 # Return a usage string, built from:
4034 # a string built from each usable entry in the @Sections table.
4038 $ret = "@_\n" if ($_[0]);
4041 foreach my $sect (get_usable_sectvars
(@Sections, 0)) {
4042 $name = lc $sect->{NAME
};
4043 $desc = $sect->{TITLE
};
4044 $ret .= sprintf " --%-38s%s\n", "$name" . ' LEVEL', "$desc";
4050 sub strip_trace
($) {
4051 # at (eval 37) line 306, <GEN6> line 4.
4052 # at /usr/sbin/amavisd-maia line 2895, <GEN4> line 22.
4053 #$_[0] =~ s/ at \(.+\) line \d+(?:, \<GEN\d+\> line \d+)?\.$//;
4054 #$_[0] =~ s/ at (\S+) line \d+(?:, \<GEN\d+\> line \d+)?\.$/: $1/;
4055 while ($_[0] =~ s/ at (?:\(eval \d+\)|\S+) line \d+(?:, \<GEN\d+\> line \d+)?\.//) {
4058 #print "strip_trace: \"$_[0]\"\n";
4062 # Getopt helper, sets an option in Opts hash to one of three
4063 # values: its default, the specified value, or 0 if the option
4064 # was the "no" prefixed variant.
4066 sub triway_opts
($ $) {
4067 my ($opt,$val) = @_;
4069 print "triway_opts: OPT: $opt, VAL: $val\n" if $Opts{'debug'} & D_ARGS
;
4070 die "Option \"--${opt}\" requires an argument" if ($val =~ /^--/);
4072 if ($opt =~ s/^no//i) {
4074 } elsif ('default' =~ /^${val}$/i) {
4075 $Opts{$opt} = $Defaults{$opt};
4084 # vi: shiftwidth=3 tabstop=3 syntax=perl et