Changesets can be listed by changeset number.
The Git repository is here.
- Revision:
- 297
- Log:
Updated to AWStats 7.0.
- Author:
- rool
- Date:
- Fri Mar 18 13:33:29 +0000 2011
- Size:
- 16466 Bytes
- Properties:
- Property svn:executable is set to *
1 | #!/usr/bin/perl |
2 | #----------------------------------------------------------------------------- |
3 | # Generates override files for the GEOIP databases for a given IP range |
4 | # This tool is part of AWStats log analyzer but can be use |
5 | # alone for any other log analyzer. |
6 | # See COPYING.TXT file about AWStats GNU General Public License. |
7 | #----------------------------------------------------------------------------- |
8 | # $Revision: 1.1 $ - $Author: eldy $ - $Date: 2010/04/17 17:57:58 $ |
9 | |
10 | use strict; no strict "refs"; |
11 | use Switch; |
12 | |
13 | #------------------------------------------------------------------------------ |
14 | # Defines |
15 | #------------------------------------------------------------------------------ |
16 | my $REVISION='$Revision: 1.1 $'; $REVISION =~ /\s(.*)\s/; $REVISION=$1; |
17 | my $VERSION="0.5 (build $REVISION)"; |
18 | |
19 | use vars qw/ |
20 | $DirData |
21 | /; |
22 | |
23 | # Variables |
24 | my %temp = {}; |
25 | my $SiteConfig = ""; |
26 | my $Output = ""; |
27 | my $IPStart = ""; |
28 | my $IPEnd = ""; |
29 | my $DBType = ""; |
30 | my $OutputDir = ""; |
31 | my $Debug = 0; |
32 | my $Overwrite = 0; |
33 | my $Fields = ""; |
34 | my $DIR=""; |
35 | my $PROG; |
36 | my $FileConfig; |
37 | my $DirData; |
38 | |
39 | my @Values = (); |
40 | |
41 | # each array entry consists of the commandline name and the pluginname |
42 | my %Types = ( |
43 | lc("GeoIP") => "geoip", |
44 | lc("GeoIPCity") => "geoip_city_maxmind", |
45 | lc("GeoIPCityLite") => "geoip_city_maxmind", |
46 | lc("GeoIPRegion") => "geoip_region_maxmind", |
47 | lc("GeoIPOrg") => "geoip_org_maxmind", |
48 | lc("GeoIPASN") =>"geoip_asn_maxmind" |
49 | ); |
50 | |
51 | #----------------------------------------------------------------------------- |
52 | # Functions |
53 | #----------------------------------------------------------------------------- |
54 | |
55 | #------------------------------------------------------------------------------ |
56 | # Function: Write an error message and exit |
57 | # Parameters: $message |
58 | # Input: None |
59 | # Output: None |
60 | # Return: None |
61 | #------------------------------------------------------------------------------ |
62 | sub error { |
63 | print "Error: $_[0].\n"; |
64 | exit 1; |
65 | } |
66 | |
67 | #------------------------------------------------------------------------------ |
68 | # Function: Write a debug message |
69 | # Parameters: $message |
70 | # Input: $Debug |
71 | # Output: None |
72 | # Return: None |
73 | #------------------------------------------------------------------------------ |
74 | sub debug { |
75 | my $level = $_[1] || 1; |
76 | if ($Debug >= $level) { |
77 | my $debugstring = $_[0]; |
78 | print "DEBUG $level - ".localtime(time())." : $debugstring\n"; |
79 | } |
80 | } |
81 | |
82 | #------------------------------------------------------------------------------ |
83 | # Function: Write a warning message |
84 | # Parameters: $message |
85 | # Input: $Debug |
86 | # Output: None |
87 | # Return: None |
88 | #------------------------------------------------------------------------------ |
89 | sub warning { |
90 | my $messagestring=shift; |
91 | if ($Debug) { debug("$messagestring",1); } |
92 | print "$messagestring\n"; |
93 | } |
94 | |
95 | #------------------------------------------------------------------------------ |
96 | # Function: CL - returns just the root name of the config file. I.e. if the |
97 | # site config name is "awstats.conf" this will return "awstats" |
98 | # Parameters: - |
99 | # Input: $SiteConfig |
100 | # Output: String with the root config name |
101 | # Return: - |
102 | #------------------------------------------------------------------------------ |
103 | sub Get_Config_Name{ |
104 | my $temp = shift; |
105 | my $idx = -1; |
106 | # check for slash |
107 | $idx = rindex($temp, "/"); |
108 | if ($idx > -1){ $temp = substr($temp, $idx+1);} |
109 | else{ |
110 | $idx = rindex($temp, "\\"); |
111 | if ($idx > -1){ $temp = substr($temp, $idx+1);} |
112 | } |
113 | # get the dot |
114 | $idx = rindex($temp, "."); |
115 | if ($idx > -1){ $temp = substr($temp, 0, $idx);} |
116 | return $temp; |
117 | } |
118 | |
119 | #------------------------------------------------------------------------------ |
120 | # Function: Read config file |
121 | # Parameters: None or configdir to scan |
122 | # Input: $DIR $PROG $SiteConfig |
123 | # Output: Global variables |
124 | # Return: - |
125 | #------------------------------------------------------------------------------ |
126 | sub Read_Config { |
127 | # Check config file in common possible directories : |
128 | # Windows : "$DIR" (same dir than awstats.pl) |
129 | # Standard, Mandrake and Debian package : "/etc/awstats" |
130 | # Other possible directories : "/usr/local/etc/awstats", "/etc" |
131 | # FHS standard, Suse package : "/etc/opt/awstats" |
132 | my $configdir=shift; |
133 | my @PossibleConfigDir=(); |
134 | my $FileSuffix; |
135 | |
136 | # if an output was specified, then skip this |
137 | if (!($Output eq '')){return;} |
138 | if ($configdir) { @PossibleConfigDir=("$configdir"); } |
139 | else { @PossibleConfigDir=("$DIR","/etc/awstats","/usr/local/etc/awstats","/etc","/etc/opt/awstats"); } |
140 | |
141 | # Open config file |
142 | $FileConfig=$FileSuffix=''; |
143 | foreach my $dir (@PossibleConfigDir) { |
144 | my $searchdir=$dir; |
145 | if ($searchdir && $searchdir !~ /[\\\/]$/) { $searchdir .= "/"; } |
146 | if (open(CONFIG,"${searchdir}awstats.$SiteConfig.conf")) { |
147 | $FileConfig="${searchdir}awstats.$SiteConfig.conf"; |
148 | $FileSuffix=".$SiteConfig"; |
149 | last; |
150 | } |
151 | if (open(CONFIG,"${searchdir}awstats.conf")) { |
152 | $FileConfig="${searchdir}awstats.conf"; |
153 | $FileSuffix=''; |
154 | last; |
155 | } |
156 | if (open(CONFIG,"$SiteConfig")) { |
157 | $FileConfig="$SiteConfig"; |
158 | $FileSuffix=''; |
159 | last; |
160 | } |
161 | } |
162 | if (! $FileConfig) { error("Couldn't open config file \"awstats.$SiteConfig.conf\" nor \"awstats.conf\" nor \"$SiteConfig.conf\" after searching in path \"".join(',',@PossibleConfigDir)."\": $!"); } |
163 | |
164 | # Analyze config file content and close it |
165 | &Parse_Config( *CONFIG , 1 , $FileConfig); |
166 | close CONFIG; |
167 | } |
168 | |
169 | #------------------------------------------------------------------------------ |
170 | # Function: Parse content of a config file |
171 | # Parameters: opened file handle, depth level, file name |
172 | # Input: - |
173 | # Output: Global variables |
174 | # Return: - |
175 | #------------------------------------------------------------------------------ |
176 | sub Parse_Config { |
177 | my ( $confighandle ) = $_[0]; |
178 | my $level = $_[1]; |
179 | my $configFile = $_[2]; |
180 | my $versionnum=0; |
181 | my $conflinenb=0; |
182 | |
183 | if ($level > 10) { error("$PROG can't read down more than 10 level of includes. Check that no 'included' config files include their parent config file (this cause infinite loop)."); } |
184 | |
185 | while (<$confighandle>) { |
186 | chomp $_; s/\r//; |
187 | $conflinenb++; |
188 | |
189 | # Extract version from first line |
190 | if (! $versionnum && $_ =~ /^# AWSTATS CONFIGURE FILE (\d+).(\d+)/i) { |
191 | $versionnum=($1*1000)+$2; |
192 | #if ($Debug) { debug(" Configure file version is $versionnum",1); } |
193 | next; |
194 | } |
195 | |
196 | if ($_ =~ /^\s*$/) { next; } |
197 | |
198 | # Check includes |
199 | if ($_ =~ /^Include "([^\"]+)"/ || $_ =~ /^#include "([^\"]+)"/) { # #include kept for backward compatibility |
200 | my $includeFile = $1; |
201 | if ($Debug) { debug("Found an include : $includeFile",2); } |
202 | if ( $includeFile !~ /^[\\\/]/ ) { |
203 | # Correct relative include files |
204 | if ($FileConfig =~ /^(.*[\\\/])[^\\\/]*$/) { $includeFile = "$1$includeFile"; } |
205 | } |
206 | if ($level > 1) { |
207 | warning("Warning: Perl versions before 5.6 cannot handle nested includes"); |
208 | next; |
209 | } |
210 | if ( open( CONFIG_INCLUDE, $includeFile ) ) { |
211 | &Parse_Config( *CONFIG_INCLUDE , $level+1, $includeFile); |
212 | close( CONFIG_INCLUDE ); |
213 | } |
214 | else { |
215 | error("Could not open include file: $includeFile" ); |
216 | } |
217 | next; |
218 | } |
219 | |
220 | # Remove comments |
221 | if ($_ =~ /^\s*#/) { next; } |
222 | $_ =~ s/\s#.*$//; |
223 | |
224 | # Extract param and value |
225 | my ($param,$value)=split(/=/,$_,2); |
226 | $param =~ s/^\s+//; $param =~ s/\s+$//; |
227 | |
228 | if ($param =~ /^DirData/){ |
229 | $DirData = $value; |
230 | #$DirData =~ s/"//g; |
231 | } |
232 | |
233 | # If not a param=value, try with next line |
234 | if (! $param) { warning("Warning: Syntax error line $conflinenb in file '$configFile'. Config line is ignored."); next; } |
235 | if (! defined $value) { warning("Warning: Syntax error line $conflinenb in file '$configFile'. Config line is ignored."); next; } |
236 | |
237 | if ($value) { |
238 | $value =~ s/^\s+//; $value =~ s/\s+$//; |
239 | $value =~ s/^\"//; $value =~ s/\";?$//; |
240 | # Replace __MONENV__ with value of environnement variable MONENV |
241 | # Must be able to replace __VAR_1____VAR_2__ |
242 | while ($value =~ /__([^\s_]+(?:_[^\s_]+)*)__/) { my $var=$1; $value =~ s/__${var}__/$ENV{$var}/g; } |
243 | } |
244 | |
245 | # Extra parameters |
246 | # if ($param =~ /^ExtraSectionName(\d+)/) { $ExtraName[$1]=$value; next; } |
247 | # |
248 | # # Plugins |
249 | # if ( $param =~ /^LoadPlugin/ ) { push @PluginsToLoad, $value; next; } |
250 | |
251 | # If parameters was not found previously, defined variable with name of param to value |
252 | $$param=$value; |
253 | } |
254 | |
255 | if ($Debug) { debug("Config file read was \"$configFile\" (level $level)"); } |
256 | } |
257 | |
258 | #------------------------------------------------------------------------------ |
259 | # Function: Attempts to load an existing override file |
260 | # Parameters: $SiteConfig $DirData |
261 | # Input: None |
262 | # Output: None |
263 | # Return: None |
264 | #------------------------------------------------------------------------------ |
265 | sub Load_File{ |
266 | my $conf = Get_Config_Name($SiteConfig); |
267 | my $file = $DirData; |
268 | $file =~ s/"//g; |
269 | if (!(rindex($file, "/") >= length($file)-1)){$file .= "/";} |
270 | $file .= $Types{lc($DBType)}.".$conf.txt"; |
271 | if (!($Output eq "")){$file = $Output;} |
272 | # see if file exists |
273 | if (!(-s $file)){debug("$file does not exist"); return;} |
274 | |
275 | # try loading |
276 | debug("Attempting to load data from $file"); |
277 | if (!open(DATA, $file)){error("Unable to open the data file: $file");} |
278 | while (<DATA>) { |
279 | chomp $_; s/\r//; |
280 | # skip comments |
281 | if ($_ =~ m/^#/){next;} |
282 | my $idx = index($_, ","); |
283 | if ($idx < 0) { debug("Invalid line: $_"); next; } |
284 | my $ip = substr($_, 0, $idx); |
285 | my $vals = substr($_, $idx); |
286 | $temp{$ip} = $vals; |
287 | } |
288 | close(DATA); |
289 | debug("Loaded ".scalar(%temp)." entries from the file"); |
290 | } |
291 | |
292 | #------------------------------------------------------------------------------ |
293 | # Function: Dumps the temp hash to the file |
294 | # Parameters: $SiteConfig $DirData |
295 | # Input: None |
296 | # Output: None |
297 | # Return: None |
298 | #------------------------------------------------------------------------------ |
299 | sub Write_File{ |
300 | my $conf = Get_Config_Name($SiteConfig); |
301 | my $file = $DirData; |
302 | $file =~ s/"//g; |
303 | if (!(rindex($file, "/") >= length($file)-1)){$file .= "/";} |
304 | $file .= $Types{lc($DBType)}.".$conf.txt"; |
305 | if (!($Output eq '')){$file = $Output;} |
306 | |
307 | # try loading |
308 | debug("Attempting to write data to $file"); |
309 | if (!open(DATA, ">$file")){error("Unable to open the data file: $file");} |
310 | my $counter = 0; |
311 | |
312 | # sort to make it easier to find ips |
313 | foreach my $key (sort keys %temp){ |
314 | if ($temp{$key}){ |
315 | print DATA "$key$temp{$key}\n"; |
316 | $counter++; |
317 | } |
318 | } |
319 | close(DATA); |
320 | debug("Wrote $counter entries to the data file"); |
321 | } |
322 | |
323 | #------------------------------------------------------------------------------ |
324 | # Function: Converts an IPv4 address to a decimal value |
325 | # Parameters: IP address in dotted notation |
326 | # Input: None |
327 | # Output: None |
328 | # Return: Integer |
329 | #------------------------------------------------------------------------------ |
330 | sub addr_to_num { unpack( N => pack( C4 => split( /\./, $_[0] ) ) ) } |
331 | |
332 | #------------------------------------------------------------------------------ |
333 | # Function: Converts an IPv4 address from decimal to it's dotted form |
334 | # Parameters: IP address as an integer |
335 | # Input: None |
336 | # Output: None |
337 | # Return: Dotted IP address |
338 | #------------------------------------------------------------------------------ |
339 | sub num_to_addr { join q{.}, unpack( C4 => pack( N => $_[0] ) ) } |
340 | |
341 | #----------------------------------------------------------------------------- |
342 | # MAIN |
343 | #----------------------------------------------------------------------------- |
344 | ($DIR=$0) =~ s/([^\/\\]*)$//; |
345 | ($PROG=$1) =~ s/\.([^\.]*)$//; |
346 | |
347 | my $QueryString=''; for (0..@ARGV-1) { $QueryString .= "$ARGV[$_]&"; } |
348 | |
349 | if ($QueryString =~ /(^|-|&)debug=(\d+)/i) { $Debug=$2; } |
350 | if ($QueryString =~ /(^|-|&)config=([^&]+)/i) { $SiteConfig="$2"; } |
351 | if ($QueryString =~ /(^|-|&)output=([^&]+)/i) { $Output="$2"; } |
352 | if ($QueryString =~ /(^|-|&)type=([^&]+)/i) { $DBType="$2"; } |
353 | if ($QueryString =~ /(^|-|&)start=([^&]+)/i) { $IPStart="$2"; } |
354 | if ($QueryString =~ /(^|-|&)end=([^&]+)/i) { $IPEnd="$2"; } |
355 | if ($QueryString =~ /(^|-|&)overwrite/i) { $Overwrite=1; } |
356 | |
357 | # Values |
358 | if ($QueryString =~ /(^|-|&)cc=([^&]+)/i) { $Values[1]="$2"; } |
359 | if ($QueryString =~ /(^|-|&)rc=([^&]+)/i) { $Values[2]="$2"; } |
360 | if ($QueryString =~ /(^|-|&)cn=([^&]+)/i) { $Values[3]="$2"; } |
361 | if ($QueryString =~ /(^|-|&)pc=([^&]+)/i) { $Values[4]="$2"; } |
362 | if ($QueryString =~ /(^|-|&)la=([^&]+)/i) { $Values[5]="$2"; } |
363 | if ($QueryString =~ /(^|-|&)lo=([^&]+)/i) { $Values[6]="$2"; } |
364 | if ($QueryString =~ /(^|-|&)mc=([^&]+)/i) { $Values[7]="$2"; } |
365 | if ($QueryString =~ /(^|-|&)ac=([^&]+)/i) { $Values[8]="$2"; } |
366 | if ($QueryString =~ /(^|-|&)is=([^&]+)/i) { $Values[9]="$2"; } |
367 | if ($QueryString =~ /(^|-|&)as=([^&]+)/i) { $Values[10]="$2"; } |
368 | |
369 | if ($OutputDir) { if ($OutputDir !~ /[\\\/]$/) { $OutputDir.="/"; } } |
370 | |
371 | if ((!$SiteConfig && !$Output) || !$DBType || !$IPStart) { |
372 | print "----- $PROG $VERSION (c) Chris Larsen -----\n"; |
373 | print "$PROG generates GeoIP Override files using data you provide.\n"; |
374 | print "Very useful for Intranet reporting or correcting an old database.\n"; |
375 | print "\n"; |
376 | print "Usage:\n"; |
377 | print "$PROG -type={type} <-config={site config} | -output={file_path}>\n"; |
378 | print " -start{IP} [data options] [script options]\n"; |
379 | print "\n"; |
380 | print " Required:\n"; |
381 | print " -type=val Type of database you want to override.\n"; |
382 | print " -config=val The full path to your AWStats config file\n"; |
383 | print " -output=val The full path to an output file\n"; |
384 | print " -start=dotted IP Starting IP address in 127.0.0.1 format\n"; |
385 | print "\n"; |
386 | print " Data Options: (surround in quotes if spaces)\n"; |
387 | print " -cc=xx Two character country code \n"; |
388 | print " -rc=xx Region code or name\n"; |
389 | print " -cn=xx City name\n"; |
390 | print " -pc=xx Postal code\n"; |
391 | print " -la=xx Latitude\n"; |
392 | print " -lo=xx Longitude\n"; |
393 | print " -mc=xx Metro code (US only)\n"; |
394 | print " -ac=xx Area code (US only)\n"; |
395 | print " -is=xx ISP\n"; |
396 | print " -as=xx AS Number\n"; |
397 | print "\n"; |
398 | print " Script Options:\n"; |
399 | print " -end=dotted IP Ending IP address for a range \n"; |
400 | print " -debug=level Debug level to print\n"; |
401 | print " -overwrite Deletes any entries in the file. Otherwise appends.\n"; |
402 | print "\n"; |
403 | print "Allowable Type Values: GeoIP | GeoIPFree | GeoCity | GeoCityLite\n"; |
404 | print " GeoIPRegion | GeoIPOrg | GeoIPASN \n"; |
405 | exit 0; |
406 | } |
407 | |
408 | # check the db type |
409 | my $matched=0; |
410 | if (!$Types{lc($DBType)}){error("Invalid database type: $DBType");} |
411 | else {debug("Using Database type: $DBType");} |
412 | |
413 | # Read config file (SiteConfig must be defined) |
414 | &Read_Config($SiteConfig); |
415 | |
416 | # see if we have valid IPs |
417 | if ($IPStart =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/) { # IPv4 address |
418 | debug("Starting IPv4 Address: $IPStart"); |
419 | } |
420 | elsif ($IPStart =~ /^[0-9A-F]*:/i) { # IPv6 address |
421 | error("Starting IPv6 Address: $IPStart"); |
422 | }else{error("Invalid starting IP address: $IPStart");} |
423 | |
424 | # for the end IP, if it's empty, we copy the start |
425 | if ($IPEnd eq ""){ |
426 | $IPEnd = $IPStart; |
427 | debug ("Using IPStart for IPEnd: $IPEnd"); |
428 | } |
429 | elsif($IPEnd =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/) { # IPv4 address |
430 | debug("Ending IPv4 Address: $IPEnd"); |
431 | } |
432 | elsif ($IPEnd =~ /^[0-9A-F]*:/i) { # IPv6 address |
433 | error("Ending IPv6 Address: $IPEnd"); |
434 | }else{error("Invalid ending IP address: $IPEnd");} |
435 | |
436 | # load the file before anything happens |
437 | if (!$Overwrite){ Load_File(); } |
438 | |
439 | # get the start and end IPs as integers |
440 | my $start = addr_to_num($IPStart); |
441 | my $end = addr_to_num($IPEnd); |
442 | |
443 | # loop and dump |
444 | while ($start <= $end){ |
445 | # add the IP and values to the hash |
446 | my $f = ","; |
447 | # clean start and end quotes |
448 | if ($f =~ m/^"/) {$f = substr($f, 1);} |
449 | |
450 | # build the fields by switching on the dbtype |
451 | switch (lc($DBType)){ |
452 | case "geoip" {$f .= $Values[1]; } |
453 | case "geoipfree" {$f .= $Values[1]; } |
454 | case "geoipcity" { |
455 | $f .= $Values[1].",".$Values[2].",\"".$Values[3]."\",\""; |
456 | $f .= $Values[4]."\",".$Values[5].",".$Values[6].",\""; |
457 | $f .= $Values[7]."\",\"".$Values[8]."\""; |
458 | } |
459 | case "geoipcitylite" { |
460 | $f .= $Values[1].",".$Values[2].",\"".$Values[3]."\",\""; |
461 | $f .= $Values[4]."\",".$Values[5].",".$Values[6].",\""; |
462 | $f .= $Values[7]."\",\"".$Values[8]."\""; |
463 | } |
464 | case "geoipregion" {$f .= "\"".$Values[2]."\""; } |
465 | case "geoiporg" {$f .= "\"".$Values[9]."\""; } |
466 | case "geoipasn" {$f .= "\"".$Values[10]." ".$Values[9]."\""} |
467 | } |
468 | |
469 | $temp{num_to_addr($start)} = $f; |
470 | debug("Generating: ".num_to_addr($start)."$f",2); |
471 | $start++; |
472 | } |
473 | |
474 | # write |
475 | Write_File(); |
476 | |
477 | 1; # Do not remove this line |
478 | |
479 | |
480 | |
481 |