Changesets can be listed by changeset number.
The Git repository is here.
- Revision:
- 269
- Log:
Overdue upgrade to AWStats v6.9, the most recent version
available at the time of writing.
- Author:
- rool
- Date:
- Wed May 27 23:57:15 +0100 2009
- Size:
- 27802 Bytes
- Properties:
- Property svn:executable is set
1 | #!/usr/bin/perl |
2 | #------------------------------------------------------- |
3 | # Convert a mail log file to a common log file for analyzing with any log |
4 | # analyzer. |
5 | #------------------------------------------------------- |
6 | # Tool built from original work of Odd-Jarle Kristoffersen |
7 | # Note 1: QMail must log in syslog format for timestamps to work. |
8 | # Note 2: QMail logging is not 100% accurate. Some messages might |
9 | # not be logged correctly or completely. |
10 | # |
11 | # A mail received to 2 different receivers, report 2 records. |
12 | # A mail received to a forwarded account is reported as to the original receiver, not the "forwarded to". |
13 | # A mail locally sent to a local alias is reported as n mails to all addresses of alias. |
14 | #------------------------------------------------------- |
15 | use strict;no strict "refs"; |
16 | |
17 | |
18 | #------------------------------------------------------- |
19 | # Defines |
20 | #------------------------------------------------------- |
21 | use vars qw/ $REVISION $VERSION /; |
22 | $REVISION='$Revision: 1.34 $'; $REVISION =~ /\s(.*)\s/; $REVISION=$1; |
23 | $VERSION="1.2 (build $REVISION)"; |
24 | |
25 | use vars qw/ |
26 | $DIR $PROG $Extension |
27 | $Debug |
28 | %mail %qmaildelivery |
29 | $help |
30 | $mode $year $lastmon $Debug |
31 | $NBOFENTRYFOFLUSH |
32 | $MailType |
33 | %MonthNum |
34 | /; |
35 | $Debug=0; |
36 | $NBOFENTRYFOFLUSH=16384; # Nb or records for flush of %entry (Must be a power of 2) |
37 | $MailType=''; # Mail server family (postfix, sendmail, qmail) |
38 | %MonthNum = ( |
39 | 'Jan'=>1, |
40 | 'Feb'=>2, |
41 | 'Mar'=>3, |
42 | 'Apr'=>4, |
43 | 'May'=>5, |
44 | 'Jun'=>6, |
45 | 'Jul'=>7, |
46 | 'Aug'=>8, |
47 | 'Sep'=>9, |
48 | 'Oct'=>10, |
49 | 'Nov'=>11, |
50 | 'Dec'=>12 |
51 | ); |
52 | |
53 | |
54 | #------------------------------------------------------- |
55 | # Functions |
56 | #------------------------------------------------------- |
57 | |
58 | sub error { |
59 | print "Error: $_[0].\n"; |
60 | exit 1; |
61 | } |
62 | |
63 | sub debug { |
64 | my $level = $_[1] || 1; |
65 | if ($Debug >= $level) { |
66 | my $debugstring = $_[0]; |
67 | if ($ENV{"GATEWAY_INTERFACE"}) { $debugstring =~ s/^ /   /; $debugstring .= "<br />"; } |
68 | print localtime(time)." - DEBUG $level - $. - : $debugstring\n"; |
69 | } |
70 | 0; |
71 | } |
72 | |
73 | sub CleanVadminUser { $_=shift||''; |
74 | s/[#<|>\[\]]//g; # Remove unwanted characters first |
75 | s/^(.*?)-//gi; # Strip off unixuser- at beginning |
76 | return $_; |
77 | } |
78 | |
79 | sub CleanEmail { $_=shift||''; |
80 | s/[#<|>\[\]]//g; # Remove unwanted characters first |
81 | return $_; |
82 | } |
83 | |
84 | # Clean host addresses |
85 | # Input: "servername[123.123.123.123]", "servername [123.123.123.123]" |
86 | # "root@servername", "[123.123.123.123]" |
87 | # Return: servername or 123.123.123.123 if servername is 'unknown' |
88 | sub CleanHost { |
89 | $_=shift||''; |
90 | if (/^\[(.*)\]$/) { $_=$1; } # If [ip] we keep ip |
91 | if (/^unknown\s*\[/) { $_ =~ /\[(.*)\]/; $_=$1; } # If unknown [ip], we keep ip |
92 | else { $_ =~ s/\s*\[.*$//; } |
93 | $_ =~ s/^.*\@//; # If x@y, we keep y |
94 | return $_; |
95 | } |
96 | |
97 | # Return domain |
98 | # Input: host.domain.com, <user@domain.com>, <> |
99 | # |
100 | sub CleanDomain { $_=shift; |
101 | s/>.*$//; s/[<>]//g; |
102 | s/^.*@//; |
103 | if (! $_) { $_ = 'localhost'; } |
104 | return $_; |
105 | } |
106 | |
107 | # Return string without starting and ending space |
108 | # |
109 | sub trim { $_=shift; |
110 | s/^\s+//; s/\s+$//; |
111 | return $_; |
112 | } |
113 | |
114 | # Write a record |
115 | # |
116 | sub OutputRecord { |
117 | my $year=shift; |
118 | my $month=shift; # Jan,Feb,... or 1,2,3... |
119 | my $day=shift; |
120 | my $time=shift; |
121 | my $from=shift; |
122 | my $to=shift; |
123 | my $relay_s=shift; |
124 | my $relay_r=shift; |
125 | my $code=shift; |
126 | my $size=shift||0; |
127 | my $forwardto=shift; |
128 | my $extinfo=shift||'-'; |
129 | |
130 | # Clean day and month |
131 | $day=sprintf("%02d",$day); |
132 | $month=sprintf("%02d",$MonthNum{$month}||$month); |
133 | |
134 | # Clean from |
135 | $from=&CleanEmail($from); |
136 | $from||='<>'; |
137 | |
138 | # Clean to |
139 | if ($mode eq 'vadmin') { $to=&CleanVadminUser($to); } |
140 | else { $to=&CleanEmail($to); } |
141 | $to||='<>'; |
142 | |
143 | # Clean relay_s |
144 | $relay_s=&CleanHost($relay_s); |
145 | $relay_s||=&CleanDomain($from); |
146 | $relay_s=~s/\.$//; |
147 | if ($relay_s eq 'local' || $relay_s eq 'localhost.localdomain') { $relay_s='localhost'; } |
148 | |
149 | # Clean relay_r |
150 | $relay_r=&CleanHost($relay_r); |
151 | $relay_r||="-"; |
152 | $relay_r=~s/\.$//; |
153 | if ($relay_r eq 'local' || $relay_r eq 'localhost.localdomain') { $relay_r='localhost'; } |
154 | #if we don't have info for relay_s, we keep it unknown, awstats might then guess it |
155 | |
156 | # Write line |
157 | print "$year-$month-$day $time $from $to $relay_s $relay_r SMTP $extinfo $code $size\n"; |
158 | |
159 | # If there was a redirect |
160 | if ($forwardto) { |
161 | # Redirect to local address |
162 | # TODO |
163 | # Redirect to external address |
164 | # TODO |
165 | } |
166 | } |
167 | |
168 | |
169 | |
170 | #------------------------------------------------------- |
171 | # MAIN |
172 | #------------------------------------------------------- |
173 | |
174 | # Prepare QueryString |
175 | my %param=(); |
176 | for (0..@ARGV-1) { $param{$_}=$ARGV[$_]; } |
177 | foreach my $key (sort keys %param) { |
178 | if ($param{$key} =~ /(^|-|&)debug=([^&]+)/i) { $Debug=$2; shift; next; } |
179 | if ($param{$key} =~ /^(\d+)$/) { $year=$1; shift; next; } |
180 | if ($param{$key} =~ /^(standard|vadmin)$/i) { $mode=$1; shift; next; } |
181 | } |
182 | if ($mode ne 'standard' and $mode ne 'vadmin') { $help = 1; } |
183 | |
184 | ($DIR=$0) =~ s/([^\/\\]*)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1; |
185 | |
186 | my $starttime=time(); |
187 | my ($nowsec,$nowmin,$nowhour,$nowday,$nowmonth,$nowyear,$nowwday,$nowyday) = localtime($starttime); |
188 | $year||=($nowyear+1900); |
189 | |
190 | # Show usage help |
191 | if ($help) { |
192 | print "----- $PROG $VERSION -----\n"; |
193 | print <<HELPTEXT; |
194 | $PROG is mail log preprocessor that convert a mail log file (from |
195 | postfix, sendmail or qmail servers) into a human readable format. |
196 | The output format is also ready to be used by a log analyzer, like AWStats. |
197 | |
198 | Usage: |
199 | perl maillogconvert.pl [standard|vadmin] [year] < logfile > output |
200 | |
201 | The first parameter specifies what format the mail logfile is : |
202 | standard - logfile is standard postfix,sendmail,qmail or mdaemon log format |
203 | vadmin - logfile is qmail log format with vadmin multi-host support |
204 | |
205 | The second parameter specifies what year to timestamp logfile with, if current |
206 | year is not the correct one (ie. 2002). Always use 4 digits. If not specified, |
207 | current year is used. |
208 | |
209 | If no output is specified, it goes to the console (stdout). |
210 | |
211 | HELPTEXT |
212 | sleep 1; |
213 | exit; |
214 | } |
215 | |
216 | # |
217 | # Start Processing Input Logfile |
218 | # |
219 | $lastmon=0; |
220 | my $numrecord=0; |
221 | my $numrecordforflush=0; |
222 | while (<>) { |
223 | chomp $_; s/\r//; |
224 | $numrecord++; |
225 | $numrecordforflush++; |
226 | |
227 | my $mailid=0; |
228 | |
229 | if (/^__BREAKPOINT__/) { last; } # For debug only |
230 | |
231 | ### <CJK> ### |
232 | my ($mon)=m/(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s/; |
233 | if ($mon) { |
234 | $mon = $MonthNum{$mon}; |
235 | if ($mon==12 && $lastmon==1 ){$year--;} |
236 | if ($mon==1 && $lastmon==12){$year++;} |
237 | $lastmon=$mon; |
238 | } |
239 | ### </CJK> ### |
240 | |
241 | if (/^#/) { |
242 | debug("Comment record"); |
243 | next; |
244 | } |
245 | |
246 | # |
247 | # Get sender host for postfix |
248 | # |
249 | elsif (/: client=/) { |
250 | $MailType||='postfix'; |
251 | # Example: |
252 | # postfix: Jan 01 07:27:32 apollon.com postfix/smtpd[1684]: 2BC793B8A4: client=remt30.cluster1.abcde.net[209.225.8.40] |
253 | my ($id,$relay_s)=m/\w+\s+\d+\s+\d+:\d+:\d+\s+[\w\-\.\@]+\s+(?:sendmail|postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+client=(.*)/; |
254 | $mailid=$id; |
255 | $mail{$id}{'relay_s'}=$relay_s; |
256 | debug("For id=$id, found host sender on a 'client' line: $mail{$id}{'relay_s'}"); |
257 | } |
258 | |
259 | # |
260 | # See if we received postfix email reject error |
261 | # |
262 | elsif (/: reject/) { |
263 | $MailType||='postfix'; |
264 | # Example: |
265 | # postfix ?.? : Jan 01 12:00:00 halley postfix/smtpd[9245]: reject: RCPT from unknown[203.156.32.33]: 554 <userx@yahoo.com>: Recipient address rejected: Relay access denied; from=<sender@aol.com> to=<userx@yahoo.com> |
266 | # postfix 2.1+: Jan 01 12:00:00 localhost postfix/smtpd[11120]: NOQUEUE: reject: RCPT from unknown[62.205.124.145]: 450 Client host rejected: cannot find your hostname, [62.205.124.145]; from=<sender@msn.com> to=<usery@yahoo.com> proto=ESMTP helo=<xxx.com> |
267 | # postfix ?.? : Jan 01 12:00:00 apollon postfix/smtpd[26553]: 1954F3B8A4: reject: RCPT from unknown[80.245.33.2]: 450 <usery@yahoo.com>: User unknown in local recipient table; from=<sender@msn.com> to=<usery@yahoo.com> proto=ESMTP helo=<xxx.com> |
268 | my ($mon,$day,$time,$id,$code,$from,$to)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+(.*)\s+from=([^\s,]*)\s+to=([^\s,]*)/; |
269 | # postfix: Jan 01 14:10:16 juni postfix/smtpd[2568]: C34ED1432B: reject: RCPT from relay2.tp2rc.edu.tw[163.28.32.177]: 450 <linda@trieger.org>: User unknown in local recipient table; from=<> proto=ESMTP helo=<rmail.nccu.edu.tw> |
270 | if (! $mon) { ($mon,$day,$time,$id,$code,$from)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+(.*)\s+from=([^\s,]*)/; } |
271 | $mailid=($id eq 'reject' || $id eq 'NOQUEUE'?'999':$id); # id not provided in log, we take '999' |
272 | if ($mailid) { |
273 | # $code='reject: RCPT from unknown[203.156.32.33]: 554 <userx@yahoo.com>: Recipient address rejected: Relay access denied;' |
274 | # or 'reject: RCPT from unknown[62.205.124.145]: 450 Client host rejected: cannot find your hostname, [62.205.124.145]; from=<sender@msn.com> to=<usery@yahoo.com> proto=ESMTP helo=<xxx.com>' |
275 | # or 'reject: RCPT from unknown[80.245.33.2]: 450 <usery@yahoo.com>: User unknown in local recipient table;' |
276 | if ($code =~ /\s+(\d\d\d)\s+/) { $mail{$mailid}{'code'}=$1; } |
277 | else { $mail{$mailid}{'code'}=999; } # Unkown error |
278 | if (! $mail{$mailid}{'relay_s'} && $code =~ /from\s+([^\s]+)\s+/) { |
279 | $mail{$mailid}{'relay_s'}=&trim($1); |
280 | } |
281 | $mail{$mailid}{'from'}=&trim($from); |
282 | if ($to) { |
283 | $mail{$mailid}{'to'}=&trim($to); |
284 | } |
285 | elsif ($code =~ /<(.*)>/) { |
286 | $mail{$mailid}{'to'}=&trim($1); |
287 | } |
288 | $mail{$mailid}{'year'}=$year; ### <CJK>### |
289 | $mail{$mailid}{'mon'}=$mon; |
290 | $mail{$mailid}{'day'}=$day; |
291 | $mail{$mailid}{'time'}=$time; |
292 | if (! defined($mail{$mailid}{'size'})) { $mail{$mailid}{'size'}='?'; } |
293 | debug("For id=$mailid, found a postfix error incoming message: code=$mail{$mailid}{'code'} from=$mail{$mailid}{'from'} to=$mail{$mailid}{'to'} time=$mail{$mailid}{'time'}"); |
294 | } |
295 | } |
296 | # |
297 | # See if we received postfix email bounced error |
298 | # |
299 | elsif (/stat(us)?=bounced/) { |
300 | $MailType||='postfix'; |
301 | # Example: |
302 | # postfix: Sep 9 18:24:23 halley postfix/local[22003]: 12C6413EC9: to=<etavidian@partenor.com>, relay=local, delay=0, status=bounced (unknown user: "etavidian") |
303 | my ($mon,$day,$time,$id,$to,$relay_r)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+to=([^\s,]*)[\s,]+relay=([^\s,]*)/; |
304 | $mailid=($id eq 'reject'?'999':$id); # id not provided in log, we take '999' |
305 | if ($mailid) { |
306 | $mail{$mailid}{'code'}=999; # Unkown error (bounced) |
307 | $mail{$mailid}{'to'}=&trim($to); |
308 | $mail{$mailid}{'relay_r'}=&trim($relay_r); |
309 | $mail{$mailid}{'year'}=$year; ### <CJK>### |
310 | $mail{$mailid}{'mon'}=$mon; |
311 | $mail{$mailid}{'day'}=$day; |
312 | $mail{$mailid}{'time'}=$time; |
313 | if (! defined($mail{$mailid}{'size'})) { $mail{$mailid}{'size'}='?'; } |
314 | debug("For id=$mailid, found a postfix bounced incoming message: code=$mail{$mailid}{'code'} to=$mail{$mailid}{'to'} relay_r=$mail{$mailid}{'relay_r'}"); |
315 | } |
316 | } |
317 | # |
318 | # See if we received sendmail reject error |
319 | # |
320 | elsif (/, reject/) { |
321 | $MailType||='sendmail'; |
322 | # Example: |
323 | # sm-mta: Jul 27 04:06:05 androneda sm-mta[6641]: h6RB44tg006641: ruleset=check_mail, arg1=<7ms93d4ms@topprodsource.com>, relay=crelay1.easydns.com [216.220.57.222], reject=451 4.1.8 Domain of sender address 7ms93d4ms@topprodsource.com does not resolve |
324 | # sm-mta: Jul 27 06:21:24 androneda sm-mta[11461]: h6RDLNtg011461: ruleset=check_rcpt, arg1=<nobody@nova.dice.net>, relay=freedom.myhostdns.com [66.246.77.42], reject=550 5.7.1 <nobody@nova.dice.net>... Relaying denied |
325 | # sendmail: Sep 30 04:21:32 halley sendmail[3161]: g8U2LVi03161: ruleset=check_rcpt, arg1=<amber3624@netzero.net>, relay=moon.partenor.fr [10.0.0.254], reject=550 5.7.1 <amber3624@netzero.net>... Relaying denied |
326 | |
327 | # sendmail: Jan 10 07:37:48 smtp sendmail[32440]: ruleset=check_relay, arg1=[211.228.26.114], arg2=211.228.26.114, relay=[211.228.26.114], reject=554 5.7.1 Rejected 211.228.26.114 found in dnsbl.sorbs.net |
328 | # sendmail: Jan 10 07:37:08 smtp sendmail[32439]: ruleset=check_relay, arg1=235.Red-213-97-175.pooles.rima-tde.net, arg2=213.97.175.235, relay=235.Red-213-97-175.pooles.rima-tde.net [213.97.175.235], reject=550 5.7.1 Mail from 213.97.175.235 refused. Rejected for bad WHOIS info on IP of your SMTP server - see http://www.rfc-ignorant.org/ |
329 | # sendmail: Jan 10 17:15:42 smtp sendmail[12770]: ruleset=check_relay, arg1=[63.218.84.21], arg2=63.218.84.21, relay=[63.218.84.21], reject=553 5.3.0 Rejected - see http://spamhaus.org/ |
330 | my ($mon,$day,$time,$id,$ruleset,$arg,$relay_s,$code)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sendmail|sm-mta)\[\d+\][:\s]*(.*?):\sruleset=(\w+),\s+arg1=(.*),\s+relay=(.*),\s+(reject=.*)/; |
331 | # sendmail: Jan 10 18:00:34 smtp sendmail[5759]: i04Axx2c005759: Milter: data, reject=511 Virus found in email! |
332 | if (! $mon) { ($mon,$day,$time,$id,$ruleset,$code)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sendmail|sm-mta)\[\d+\]:\s+(.*?):\s\w+:\s(\w+),\s+(reject=.*)/; } |
333 | $mailid=(! $id && $mon?'999':$id); # id not provided in log, we take '999' |
334 | if ($mailid) { |
335 | if ($ruleset eq 'check_mail') { $mail{$mailid}{'from'}=$arg; } |
336 | if ($ruleset eq 'check_rcpt') { $mail{$mailid}{'to'}=$arg; } |
337 | if ($ruleset eq 'check_relay') { } |
338 | if ($ruleset eq 'data') { } |
339 | $mail{$mailid}{'relay_s'}=$relay_s; |
340 | # $code='reject=550 5.7.1 <amber3624@netzero.net>... Relaying denied' |
341 | if ($code =~ /=(\d\d\d)\s+/) { $mail{$mailid}{'code'}=$1; } |
342 | else { $mail{$mailid}{'code'}=999; } # Unkown error |
343 | $mail{$mailid}{'year'}=$year; ### <CJK>### |
344 | $mail{$mailid}{'mon'}=$mon; |
345 | $mail{$mailid}{'day'}=$day; |
346 | $mail{$mailid}{'time'}=$time; |
347 | if (! defined($mail{$mailid}{'size'})) { $mail{$mailid}{'size'}='?'; } |
348 | debug("For id=$mailid, found a sendmail error incoming message: code=$mail{$mailid}{'code'} from=$mail{$mailid}{'from'} to=$mail{$mailid}{'to'} relay_s=$mail{$mailid}{'relay_s'}"); |
349 | } |
350 | } |
351 | |
352 | # |
353 | # See if we send a sendmail (with ctladdr tag) email |
354 | # |
355 | elsif (/, ctladdr=/) { |
356 | $MailType||='sendmail'; |
357 | # |
358 | # Matched outgoing sendmail/postfix message |
359 | # |
360 | my ($mon,$day,$time,$id,$to,$fromorto)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sm-mta|sendmail(?:-out|)|postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[.*?\]:\s+([^:]*):\s+to=(.*?)[,\s]+ctladdr=([^\,\s]*)/; |
361 | $mailid=$id; |
362 | if (m/\s+relay=([^\s,]*)[\s,]/) { $mail{$id}{'relay_r'}=$1; } |
363 | elsif (m/\s+mailer=local/) { $mail{$id}{'relay_r'}='localhost'; } |
364 | if (/, stat\=Sent/) { $mail{$id}{'code'}=1; } |
365 | elsif (/, stat\=User\s+unknown/) { $mail{$id}{'code'}=550; } |
366 | elsif (/, stat\=Local\s+configuration/) { $mail{$id}{'code'}=451; } |
367 | elsif (/, stat\=Deferred:\s+(\d*)/) { $mail{$id}{'code'}=$1; } |
368 | else { $mail{$id}{'code'}=999; } |
369 | $mail{$mailid}{'year'}=$year; ### <CJK>### |
370 | $mail{$id}{'mon'}=$mon; |
371 | $mail{$id}{'day'}=$day; |
372 | $mail{$id}{'time'}=$time; |
373 | if (&trim($to)=~/^\|/) { |
374 | # In particular case of mails are sent to a pipe, the ctladdr contains the to |
375 | $mail{$id}{'to'}=&trim($fromorto); |
376 | } else { |
377 | # In most cases |
378 | $mail{$id}{'to'}=&trim($to); |
379 | $mail{$id}{'from'}=&trim($fromorto); |
380 | } |
381 | if (! defined($mail{$id}{'size'})) { $mail{$id}{'size'}='?'; } |
382 | debug("For id=$id, found a sendmail outgoing message: to=$mail{$id}{'to'} from=$mail{$id}{'from'} size=$mail{$id}{'size'} relay_r=".($mail{$id}{'relay_r'}||'')); |
383 | } |
384 | |
385 | # |
386 | # Matched incoming qmail message |
387 | # |
388 | elsif (/info msg .* from/) { |
389 | # Example: Sep 14 09:58:09 gandalf qmail: 1063526289.292776 info msg 270182: bytes 10712 from <john@john.do> qp 54945 uid 82 |
390 | $MailType||='qmail'; |
391 | #my ($id,$size,$from)=m/info msg \d+: bytes (\d+) from <(.*)>/; |
392 | my ($id,$size,$from)=m/info msg (\d+): bytes (\d+) from <(.*)>/; |
393 | $mailid=$id; |
394 | delete $mail{$mailid}; # If 'info msg' found, we start a new mail. This is to protect from wrong file |
395 | if (! $mail{$id}{'from'} || $mail{$id}{'from'} ne '<>') { $mail{$id}{'from'}=$from; } # TODO ??? |
396 | $mail{$id}{'size'}=$size; |
397 | if (m/\s+relay=([^\,]+)[\s\,]/ || m/\s+relay=([^\s\,]+)$/) { $mail{$id}{'relay_s'}=$1; } |
398 | debug("For id=$id, found a qmail 'info msg' message: from=$mail{$id}{'from'} size=$mail{$id}{'size'}"); |
399 | } |
400 | |
401 | # |
402 | # Matched incoming sendmail or postfix message |
403 | # |
404 | elsif (/: from=/) { |
405 | # sm-mta: Jul 28 06:55:13 androneda sm-mta[28877]: h6SDtCtg028877: from=<xxx@mysite.net>, size=2556, class=0, nrcpts=1, msgid=<w1$kqj-9-o2m45@0h2i38.4.m0.5u>, proto=ESMTP, daemon=MTA, relay=smtp.easydns.com [205.210.42.50] |
406 | # postfix: Jul 3 15:32:26 apollon postfix/qmgr[13860]: 08FB63B8A4: from=<nobody@ns3744.ovh.net>, size=3302, nrcpt=1 (queue active) |
407 | # postfix: Sep 24 14:45:15 wideboy postfix/qmgr[22331]: 7E0E6196: from=<xxx@hotmail.com>, size=1141 (queue active) |
408 | my ($id,$from,$size)=m/\w+\s+\d+\s+\d+:\d+:\d+\s+[\w\-\.\@]+\s+(?:sm-mta|sendmail(?:-in|)|postfix\/qmgr|postfix\/nqmgr)\[\d+\]:\s+(.*?):\s+from=(.*?),\s+size=(\d+)/; |
409 | $mailid=$id; |
410 | if (! $mail{$id}{'code'}) { $mail{$id}{'code'}=1; } # If not already defined, we define it |
411 | if (! $mail{$id}{'from'} || $mail{$id}{'from'} ne '<>') { $mail{$id}{'from'}=$from; } |
412 | $mail{$id}{'size'}=$size; |
413 | if (m/\s+relay=([^\,]+)[\s\,]/ || m/\s+relay=([^\s\,]+)$/) { $mail{$id}{'relay_s'}=$1; } |
414 | debug("For id=$id, found a sendmail/postfix incoming message: from=$mail{$id}{'from'} size=$mail{$id}{'size'} relay_s=".($mail{$id}{'relay_s'}||'')); |
415 | } |
416 | |
417 | # |
418 | # Matched exchange message |
419 | # |
420 | elsif (/^([^\t]+)\t([^\t]+)\t[^\t]+\t([^\t]+)\t([^\t]+)\t([^\t]+)\t[^\t]+\t([^\t]+)\t([^\t]+)\t([^\t]+)\t[^\t]+\t[^\t]+\t([^\t]+)\t[^\t]+\t[^\t]+\t[^\t]+\t[^\t]+\t[^\t]+\t([^\t]+)\t([^\t]+)/) { |
421 | # date hour GMT ip_s relay_s partner relay_r ip_r to code id size subject from |
422 | # Example: 2003-8-12 0:58:14 GMT 66.218.66.69 n14.grp.scd.yahoo.com - PACKRAT 192.168.1.2 christina@pirnie.org 1019 bh9e3f+5qvo@eGroups.com 0 0 4281 1 2003-8-12 0:58:14 GMT 0 Version: 6.0.3790.0 - [SRESafeHaven] Re: More Baby Stuff jtluvs2cq@wmconnect.com - |
423 | $MailType||='exchange'; |
424 | my $date=$1; |
425 | my $time=$2; |
426 | my $relay_s=$3; |
427 | my $partner=$4; |
428 | my $relay_r=$5; |
429 | my $to=$6; $to =~ s/\s/%20/g; |
430 | my $code=$7; |
431 | my $id=$8; |
432 | my $size=$9; |
433 | my $subject=&trim($10); |
434 | my $from=$11; $from =~ s/\s/%20/g; |
435 | $id=sprintf("%s_%s_%s",$id,$from,$to); |
436 | # Check if record is significant record |
437 | my $ok=0; |
438 | |
439 | # Code 1031=SMTP End Outbound Transfer |
440 | if ($code == 1031) { # This is for external bound mails |
441 | $ok=1; |
442 | my $savrelay_s=$relay_s; |
443 | $relay_s=$relay_r; $relay_r=$savrelay_s; |
444 | #$relay_s=$relay_r; |
445 | #$relay_r=$partner; |
446 | $code=1; |
447 | } |
448 | # Code 1028=SMTP Store Driver: Message Delivered Locally to Store |
449 | if ($code == 1028) { # This is for local bound mails |
450 | $code=1; |
451 | $ok=1; |
452 | } |
453 | # Code 1030=SMTP: Non-Delivered Report (NDR) Generated |
454 | if ($code == 1030) { # This is for errors. |
455 | $code=999; |
456 | $ok=1; |
457 | } |
458 | |
459 | if ($ok && !$mail{$id}{'code'} ) { |
460 | $mailid=$id; |
461 | if ($date =~ /(\d+)-(\d+)-(\d+)/) { |
462 | $mail{$id}{'year'}=sprintf("%02s",$1); |
463 | $mail{$id}{'mon'}=sprintf("%02s",$2); |
464 | $mail{$id}{'day'}=sprintf("%02s",$3); |
465 | } |
466 | if ($time =~ /^(\d+):(\d+):(\d+)/) { |
467 | $mail{$id}{'time'}=sprintf("%02s:%02s:%02s",$1,$2,$3); |
468 | } |
469 | if ( $from eq '<>' && $subject =~ /^Delivery\s+Status/) { |
470 | $from='postmaster@localhost'; |
471 | } |
472 | $mail{$id}{'from'}=$from; |
473 | $mail{$id}{'to'}=$to; |
474 | $mail{$id}{'code'}=$code; |
475 | $mail{$id}{'size'}=$size; |
476 | $mail{$id}{'relay_s'}=$relay_s; |
477 | $mail{$id}{'relay_r'}=$relay_r; |
478 | debug("For id=$id, found an exchange message: year=$mail{$id}{'year'} mon=$mail{$id}{'mon'} day=$mail{$id}{'day'} time=$mail{$id}{'time'} from=$mail{$id}{'from'} to=$mail{$id}{'to'} size=$mail{$id}{'size'} code=$mail{$id}{'code'} relay_s=$mail{$id}{'relay_s'} relay_r=$mail{$id}{'relay_r'}"); |
479 | } |
480 | } |
481 | |
482 | # |
483 | # Matched sendmail or postfix "to" message |
484 | # |
485 | elsif (/: to=.*stat(us)?=sent/i) { |
486 | # Example: |
487 | # postfix: Jan 01 07:27:38 apollon postfix/local[1689]: 2BC793B8A4: to=<jo@jo.com>, orig_to=<webmaster@toto.com>, relay=local, delay=6, status=sent ("|/usr/bin/procmail") |
488 | my ($mon,$day,$time,$id,$to)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sm-mta|sendmail(?:-out|)|postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[.*?\]:\s+(.*?):\s+to=(.*?),/; |
489 | $mailid=$id; |
490 | $mail{$id}{'code'}='1'; |
491 | if (m/\s+relay=([^\s,]*)[\s,]/) { $mail{$id}{'relay_r'}=$1; } |
492 | elsif (m/\s+mailer=local/) { $mail{$id}{'relay_r'}='localhost'; } |
493 | if (m/forwarded as/) { |
494 | # If 'forwarded as idnewmail' is found, we discard this mail to avoid counting it twice |
495 | debug("For id=$id, mail was forwarded to other id, we discard it"); |
496 | delete $mail{$id}; |
497 | } |
498 | ########################################### |
499 | elsif (m/\s*dsn=2.6.0\s*/) { |
500 | # if the DSN is not 2.0.0, we discard this mail to avoid counting it twice |
501 | # postfix: Aug 29 19:22:38 example postfix/smtp[1347]: D989FD6C302: to=<webmaster@example.com>, relay=127.0.0.1[127.0.0.1]:10024, delay=2.9, delays=0.31/0.01/0/2.6, dsn=2.6.0, status=sent (250 2.6.0 Ok, id=01182-01, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 995DCD6C315) |
502 | debug("For id=$id, mail DSN is not 2.0.0, we discard it"); |
503 | delete $mail{$id}; |
504 | } |
505 | ########################################### |
506 | else { |
507 | if (m/\s+orig_to=([^\s,]*)[\s,]/) { |
508 | # If we have a orig_to, we used it as receiver |
509 | $mail{$id}{'to'}=&trim($1); |
510 | $mail{$id}{'forwardedto'}=&trim($to); |
511 | } |
512 | else { |
513 | $mail{$id}{'to'}=&trim($to); |
514 | } |
515 | $mail{$mailid}{'year'}=$year; ### <CJK>### |
516 | $mail{$id}{'mon'}=$mon; |
517 | $mail{$id}{'day'}=$day; |
518 | $mail{$id}{'time'}=$time; |
519 | debug("For id=$id, found a sendmail/postfix record: mon=$mail{$id}{'mon'} day=$mail{$id}{'day'} time=$mail{$id}{'time'} to=$mail{$id}{'to'} relay_r=$mail{$id}{'relay_r'}"); |
520 | } |
521 | } |
522 | |
523 | # |
524 | # Matched qmail "to" record |
525 | # |
526 | elsif (/starting delivery/) { |
527 | # Example: Sep 14 09:58:09 gandalf qmail: 1063526289.574100 starting delivery 251: msg 270182 to local spamreport@john.do |
528 | # Example: 2003-09-27 11:22:07.039237500 starting delivery 3714: msg 163844 to local name_also_removed@maildomain.com |
529 | $MailType||='qmail'; |
530 | my ($yea,$mon,$day,$time,$delivery,$id,$relay_r,$to)=(); |
531 | ($mon,$day,$time,$delivery,$id,$relay_r,$to)=m/^(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+.*\s+\d+(?:\.\d+)?\s+starting delivery (\d+):\s+msg\s+(\d+)\s+to\s+(.*)?\s+(.*)$/; |
532 | if (! $id) { ($yea,$mon,$day,$time,$delivery,$id,$relay_r,$to)=m/^(\d+)-(\d+)-(\d+)\s+(\d+:\d+:\d+).*\s+starting delivery (\d+):\s+msg\s+(\d+)\s+to\s+(.*)?\s+(.*)$/; } |
533 | $mailid=$id; |
534 | if ($relay_r eq 'local') { $mail{$id}{'relay_r'}='localhost'; } |
535 | elsif (m/\s+relay=([^\s,]*)[\s,]/) { $mail{$id}{'relay_r'}=$1; } |
536 | elsif (m/\s+mailer=local/) { $mail{$id}{'relay_r'}='localhost'; } |
537 | $qmaildelivery{$delivery}=$id; # Save mail id for this delivery to be able to get error code |
538 | if ($yea) { $mail{$id}{'year'}=$yea; } |
539 | $mail{$id}{'mon'}=$mon; |
540 | $mail{$id}{'day'}=$day; |
541 | $mail{$id}{'time'}=$time; |
542 | $mail{$id}{'to'}{$delivery}=&trim($to); |
543 | debug("For id=$id, found a qmail 'start delivery' record: year=".($mail{$id}{'year'}||'')." mon=$mail{$id}{'mon'} day=$mail{$id}{'day'} time=$mail{$id}{'time'} to=$mail{$id}{'to'}{$delivery} relay_r=".($mail{$id}{'relay_r'}||'')." delivery=$delivery"); |
544 | } |
545 | |
546 | # |
547 | # Matched qmail status code record |
548 | # |
549 | elsif (/delivery (\d+): (\w+):/) { |
550 | # Example: Sep 14 09:58:09 gandalf qmail: 1063526289.744259 delivery 251: success: did_0+0+1/ |
551 | # Example: 2003-09-27 11:22:07.070367500 delivery 3714: success: did_1+0+0/ |
552 | $MailType||='qmail'; |
553 | my ($delivery,$code)=($1,$2); |
554 | my $id=$qmaildelivery{$delivery}; |
555 | $mailid=$id; |
556 | if ($code =~ /success/i) { $mail{$id}{'code'}{$delivery}=1; } |
557 | elsif ($code =~ /deferral/i) { $mail{$id}{'code'}{$delivery}=999; } |
558 | else { $mail{$id}{'code'}{$delivery}=999; } |
559 | debug("For id=$qmaildelivery{$delivery}, found a qmail 'delivery' record: delivery=$delivery code=$mail{$id}{'code'}{$delivery}"); |
560 | } |
561 | # |
562 | # Matched qmail end of mail record |
563 | # |
564 | elsif (/end msg (\d+)/ && scalar %{$mail{$1}{'to'}}) { # If records for mail id are finished and still mails with no delivery status |
565 | # Example: Sep 14 09:58:12 gandalf qmail: 1063526292.782444 end msg 270182 |
566 | $MailType||='qmail'; |
567 | my ($id)=($1); |
568 | $mailid=$id; |
569 | foreach my $delivery (keys %{$mail{$mailid}{'to'}}) { $mail{$id}{'code'}{$delivery}||=1; } |
570 | debug("For id=$id, found a qmail 'end msg' record. This replace 'delivery' record for delivery=".join(',',keys %{$mail{$id}{'code'}})); |
571 | } |
572 | # |
573 | # Matched MDaemon log file record |
574 | # |
575 | elsif (/^\"(\d\d\d\d)-(\d\d)-(\d\d) (\d\d:\d\d:\d\d)\",\"[^\"]*\",(\w+),\d+,\"([^\"]*)\",\"([^\"]*)\",\"([^\"]*)\",\"[^\"]*\",\"([^\"]*)\",\"([^\"]*)\",\"([^\"]*)\",([\.\d]+),(\d+),(\d+)/) { |
576 | # Example: "2003-11-06 00:00:42","2003-11-06 00:00:45",SMTPI,9443,"dillon_fm@aaaaa.net","cpeltier@domain.com","","","10.0.0.16","","",0,4563,1 |
577 | $MailType||='mdaemon'; |
578 | my ($id)=($numrecord); |
579 | if ($5 eq 'SMTPI' || $5 eq 'SMTPO') { |
580 | $mail{$id}{'year'}=$1; |
581 | $mail{$id}{'mon'}=$2; |
582 | $mail{$id}{'day'}=$3; |
583 | $mail{$id}{'time'}=$4; |
584 | $mail{$id}{'direction'}=($5 eq 'SMTPI'?'in':'out'); |
585 | $mail{$id}{'from'}=$6; |
586 | $mail{$id}{'to'}=$7||$8; |
587 | if ($5 eq 'SMTPI') { |
588 | $mail{$id}{'relay_s'}=$9; |
589 | $mail{$id}{'relay_r'}='-'; |
590 | } |
591 | if ($5 eq 'SMTPO') { |
592 | $mail{$id}{'relay_s'}=$9; |
593 | $mail{$id}{'relay_r'}='-'; |
594 | } |
595 | $mail{$id}{'code'}=1; |
596 | $mail{$id}{'size'}=$13; |
597 | $mail{$id}{'extinfo'}="?virus=$10&rbl=$11&heuristicspam=$12&ssl=$14"; |
598 | $mail{$id}{'extinfo'}=~s/\s/_/g; |
599 | $mailid=$id; |
600 | } |
601 | } |
602 | |
603 | |
604 | # |
605 | # Write record if all required data were found |
606 | # |
607 | if ($mailid) { |
608 | my $code; my $to; |
609 | my $delivery=0; |
610 | my $canoutput=0; |
611 | |
612 | debug("ID:$mailid RELAY_S:".($mail{$mailid}{'relay_s'}||'')." RELAY_R:".($mail{$mailid}{'relay_r'}||'')." FROM:".($mail{$mailid}{'from'}||'')." TO:".($mail{$mailid}{'to'}||'')." CODE:".($mail{$mailid}{'code'}||'')); |
613 | |
614 | # Check if we can output a mail line |
615 | if ($MailType eq 'qmail') { |
616 | if ($mail{$mailid}{'code'} && scalar %{$mail{$mailid}{'code'}}) { |
617 | # This is a hash variable |
618 | foreach my $key (keys %{$mail{$mailid}{'code'}}) { |
619 | $delivery=$key; |
620 | $code=$mail{$mailid}{'code'}{$key}; |
621 | $to=$mail{$mailid}{'to'}{$key}; |
622 | } |
623 | $canoutput=1; |
624 | } |
625 | } |
626 | elsif ($MailType eq 'mdaemon') { |
627 | $code=$mail{$mailid}{'code'}; |
628 | $to=$mail{$mailid}{'to'}; |
629 | $canoutput=1; |
630 | } |
631 | else { |
632 | $code=$mail{$mailid}{'code'}; |
633 | $to=$mail{$mailid}{'to'}; |
634 | if ($mail{$mailid}{'from'} && $mail{$mailid}{'to'}) { $canoutput=1; } |
635 | if ($mail{$mailid}{'from'} && $mail{$mailid}{'code'} > 1) { $canoutput=1; } |
636 | if ($mailid && $mail{$mailid}{'code'} > 1) { $canoutput=1; } |
637 | } |
638 | |
639 | # If we can |
640 | if ($canoutput) { |
641 | &OutputRecord($mail{$mailid}{'year'}?$mail{$mailid}{'year'}:$year,$mail{$mailid}{'mon'},$mail{$mailid}{'day'},$mail{$mailid}{'time'},$mail{$mailid}{'from'},$to,$mail{$mailid}{'relay_s'},$mail{$mailid}{'relay_r'},$code,$mail{$mailid}{'size'},$mail{$mailid}{'forwardto'},$mail{$mailid}{'extinfo'}); |
642 | # Delete mail with generic unknown id (This id can by used by another mail) |
643 | if ($mailid eq '999') { |
644 | debug(" Delete mail for id=$mailid",3); |
645 | delete $mail{$mailid}; |
646 | } |
647 | # Delete delivery instance for id if qmail (qmail can use same id for several mails with multiple delivery) |
648 | elsif ($MailType eq 'qmail') { |
649 | debug(" Delete delivery instances for mail id=$mailid and delivery id=$delivery",3); |
650 | if ($delivery) { |
651 | delete $mail{$mailid}{'to'}{$delivery}; |
652 | delete $mail{$mailid}{'code'}{$delivery}; |
653 | } |
654 | } |
655 | |
656 | # We flush %mail if too large |
657 | if (scalar keys %mail > $NBOFENTRYFOFLUSH) { |
658 | debug("We reach $NBOFENTRYFOFLUSH records in %mail, so we flush mail hash array"); |
659 | #foreach my $id (keys %mail) { |
660 | # debug(" Delete mail for id=$id",3); |
661 | # delete $mail{$id}; |
662 | #} |
663 | %mail=(); |
664 | %qmaildelivery=(); |
665 | } |
666 | |
667 | } |
668 | } |
669 | else { |
670 | debug("Not interesting row"); |
671 | } |
672 | |
673 | } |
674 | |
675 | #foreach my $key (keys %mail) { |
676 | # print ".$key.$mail{$key}{'to'}.\n"; |
677 | #} |
678 | |
679 | 0; |