* Several noticeable performance improvements (thanks, Hanno Hecker)

   * mIRC is no longer the default format. You have to always specify a format
     now.

docs/Changelog

@@ -1,4 +1,7 @@
 pisg (x.xx)
+   * Several noticeable performance improvements (thanks, Hanno Hecker)
+   * mIRC is no longer the default format. You have to always specify a format
+     now.
    * Fix wrong counting of most referenced nicks.
    * Remove devel-code that appeared in 0.52 pisg.cfg.
    * Added support for supybot logfiles,set format to 'supy'. (thanks, Jerome

docs/pisg-doc.sgml

@@ -439,7 +439,7 @@
     </refsect1>
     <refsect1>
     <title>Default</title>
-    <para> mIRC </para>
+    <para> Unset </para>
     </refsect1>
     </refentry>
 

modules/Pisg.pm

@@ -92,7 +92,7 @@ sub get_default_config_settings
         channel => '',
         logtype => 'Logfile',
         logfile => [],
-        format => 'mIRC',
+        format => '',
         network => 'SomeIRCNet',
         outputfile => 'index.html',
         outputtag => '',
@@ -197,7 +197,7 @@ sub get_default_config_settings
         botnicks => '',            # Needed for DCpp format (non-irc)
 
         dailyactivity => 0,
-        version => "0.52",
+        version => "0.53-cvs",
     };
 
     # This enables us to use the search_path in other modules
@@ -434,6 +434,8 @@ sub do_channel
         print STDERR "No channels defined.\n";
     } elsif ((!@{$self->{cfg}->{logfile}}) && (!@{$self->{cfg}->{logdir}})) {
         print STDERR "No logfile or logdir defined for " . $self->{cfg}->{channel} . "\n";
+    } elsif (!$self->{cfg}->{format}) {
+        print STDERR "No format defined for $self->{cfg}->{channel}.\n";
     } else {
         $self->init_pisg();        # Init some general things
 
@@ -475,10 +477,12 @@ _END
         }
 
         # Create our HTML page if the logfile has any data.
-        if (defined $stats and $stats->{parsedlines} > 0) {
-            $generator->create_output();
-        } elsif ($stats->{parsedlines} == 0) {
-            print STDERR "No parseable lines found in logfile ($stats->{totallines} total lines). Skipping.\nYou might be using the wrong format.\n";
+        if (defined $stats) {
+            if ($stats->{parsedlines} > 0) {
+                $generator->create_output();
+            } elsif ($stats->{parsedlines} == 0) {
+                print STDERR "No parseable lines found in logfile ($stats->{totallines} total lines). Skipping.\nYou might be using the wrong format.\n";
+            }
         }
 
         restore_aliases();

modules/Pisg/Parser/Format/IRCAP.pm

@@ -49,7 +49,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -78,7 +78,7 @@ sub thirdline
         $hash{hour} = $1;
         $hash{min}  = $2;
         $hash{saying}  = $3;
-        $hash{nick} = remove_prefix($line[0]);
+        ($hash{nick}  = $line[0]) =~ s/^[@%\+]//o; # Remove prefix
 	if ($#linea >= 2 and $linea[2] eq 'topic:') {
 	#print $linea[2], "\n";
 	}
@@ -159,15 +159,4 @@ sub thirdline
     return;
 }
 
-sub remove_prefix
-{
-    my $str = shift;
-
-    $str =~ s/^@//;
-    $str =~ s/^\+//;
-    $str =~ s/^%//;
-
-    return $str;
-}
-
 1;

modules/Pisg/Parser/Format/Trillian.pm

@@ -27,7 +27,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
         $hash{saying} =~ s/\(Link: ((http|https|ftp|telnet|news):\/\/.*?)\)\1/$1/;
 
@@ -45,7 +45,7 @@ sub actionline
     if ($line =~ /$self->{actionline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
         $hash{saying} =~ s/\(Link: ((http|https|ftp|telnet|news):\/\/.*?)\)\1/$1/;
 
@@ -64,18 +64,18 @@ sub thirdline
 
         $hash{hour} = $1;
         $hash{min}  = $2;
-        $hash{nick} = remove_prefix($3);
+        ($hash{nick}  = $3) =~ s/^[@%\+]//o; # Remove prefix
 
         if ($3 =~ /^(\S+) has been kicked off channel (\S+) by (\S+) .+/) {
-            $hash{nick} = remove_prefix($1);
+            ($hash{nick}  = $1) =~ s/^[@%\+]//o; # Remove prefix
             $hash{kicker} = $3;
 
         } elsif ($3 =~ /^(\S+) has changed the topic on channel (\S+) to (.+)/) {
-             $hash{nick} = remove_prefix($1);
+            ($hash{nick}  = $1) =~ s/^[@%\+]//o; # Remove prefix
              $hash{newtopic} = $3;
 
         } elsif ($3 =~ /^Mode change \"(\S+)[^\"]+\".+ by (.+)$/) {
-             $hash{nick} = remove_prefix($2);
+            ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
              $hash{newmode} = $1;
 
         } elsif ($3 =~ /^(\S+) \S+ has joined channel \S+/) {
@@ -83,7 +83,7 @@ sub thirdline
             $hash{newjoin} = $1;
 
         } elsif ($3 =~ /^(\S+) is now known as (\S+)/) {
-            $hash{nick} = remove_prefix($1);
+            ($hash{nick}  = $1) =~ s/^[@%\+]//o; # Remove prefix
             $hash{newnick} = $2;
         }
 
@@ -94,15 +94,4 @@ sub thirdline
     }
 }
 
-sub remove_prefix
-{
-    my $str = shift;
-
-    $str =~ s/^@//;
-    $str =~ s/^\+//;
-    $str =~ s/^%//;
-
-    return $str;
-}
-
 1;

modules/Pisg/Parser/Format/Vision.pm

@@ -27,7 +27,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -44,7 +44,7 @@ sub actionline
     if ($line =~ /$self->{actionline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -64,14 +64,14 @@ sub thirdline
 
         $hash{hour} = $1;
         $hash{min}  = $2;
-        $hash{nick} = remove_prefix($line[0]);
+        ($hash{nick}  = $line[0]) =~ s/^[@%\+]//o; # Remove prefix
 
         if ($#line >= 7 && ($line[1].$line[2].$line[3]) eq 'hasbeenkicked') {
             $hash{kicker} = $line[7];
 
         } elsif ($#line >= 4 && ($line[1].$line[2] eq 'Topicchanged')) {
             $hash{newtopic} = join(' ', @line[5..$#line]);
-            $hash{nick} = remove_prefix($line[4]);
+            ($hash{nick}  = $line[4]) =~ s/^[@%\+]//o; # Remove prefix
             $hash{nick} =~ s/:$//;
 
         } elsif ($#line >= 3 && ($line[1].$line[2]) eq 'setmode') {
@@ -92,13 +92,4 @@ sub thirdline
     }
 }
 
-sub remove_prefix
-{
-    my ($str) = @_;
-
-    $str =~ s/^[@%\+]//o;
-
-    return $str;
-}
-
 1;

modules/Pisg/Parser/Format/mIRC.pm

@@ -27,7 +27,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -44,7 +44,7 @@ sub actionline
     if ($line =~ /$self->{actionline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -62,9 +62,9 @@ sub thirdline
 
         my @line = split(/\s/, $3);
 
-        $hash{hour} = $1;
-        $hash{min}  = $2;
-        $hash{nick} = remove_prefix($line[0]);
+        $hash{hour}  = $1;
+        $hash{min}   = $2;
+        ($hash{nick} = $line[0]) =~ s/^[@%\+]//o; # Remove prefix
 
         if ($#line >= 4 && ($line[1].$line[2]) eq 'waskicked') {
             $hash{kicker} = $line[4];
@@ -91,13 +91,5 @@ sub thirdline
     }
 }
 
-sub remove_prefix
-{
-    my ($str) = @_;
-
-    $str =~ s/^[@%\+]//o;
-
-    return $str;
-}
 
 1;

modules/Pisg/Parser/Format/mIRC6.pm

@@ -26,7 +26,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -55,7 +55,7 @@ sub thirdline
         $hash{hour} = $1;
         $hash{min}  = $2;
         $hash{saying}  = $3;
-        $hash{nick} = remove_prefix($line[0]);
+        ($hash{nick}  = $line[0]) =~ s/^[@%\+]//o; # Remove prefix
 
         if ($#line >= 4 && ($line[1].$line[2]) eq 'waskicked' && ($line[$#line] =~ /\)$/)) {
                 $hash{kicker} = $line[4];
@@ -110,15 +110,4 @@ sub thirdline
     return;
 }
 
-sub remove_prefix
-{
-    my $str = shift;
-
-    $str =~ s/^@//;
-    $str =~ s/^\+//;
-    $str =~ s/^%//;
-
-    return $str;
-}
-
 1;

modules/Pisg/Parser/Format/muh2.pm

@@ -34,7 +34,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -51,7 +51,7 @@ sub actionline
     if ($line =~ /$self->{actionline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -71,7 +71,7 @@ sub thirdline
 
         $hash{hour} = $1;
         $hash{min}  = $2;
-        $hash{nick} = remove_prefix($line[0]);
+        ($hash{nick}  = $line[0]) =~ s/^[@%\+]//o; # Remove prefix
 
         if ($#line >= 4 && ($line[1].$line[2]) eq 'waskicked') {
             $hash{kicker} = $line[4];
@@ -98,13 +98,4 @@ sub thirdline
     }
 }
 
-sub remove_prefix
-{
-    my ($str) = @_;
-
-    $str =~ s/^[@%\+]//o;
-
-    return $str;
-}
-
 1;

modules/Pisg/Parser/Format/perlbot.pm

@@ -27,7 +27,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -44,7 +44,7 @@ sub actionline
     if ($line =~ /$self->{actionline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -62,7 +62,7 @@ sub thirdline
 
         $hash{hour} = $1;
         $hash{min}  = $2;
-        $hash{nick} = remove_prefix($4);
+        ($hash{nick}  = $4) =~ s/^[@%\+]//o; # Remove prefix
 
         if (($3) eq '[KICK]') {
             $hash{kicker} = $8;
@@ -87,15 +87,4 @@ sub thirdline
     }
 }
 
-sub remove_prefix
-{
-    my $str = shift;
-
-    $str =~ s/^@//;
-    $str =~ s/^\+//;
-    $str =~ s/^%//;
-
-    return $str;
-}
-
 1;

modules/Pisg/Parser/Format/virc98.pm

@@ -30,7 +30,7 @@ sub normalline
     if ($line =~ /$self->{normalline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -47,7 +47,7 @@ sub actionline
     if ($line =~ /$self->{actionline}/o) {
 
         $hash{hour}   = $1;
-        $hash{nick}   = remove_prefix($2);
+        ($hash{nick}  = $2) =~ s/^[@%\+]//o; # Remove prefix
         $hash{saying} = $3;
 
         return \%hash;
@@ -94,17 +94,6 @@ sub thirdline
     }
 }
 
-sub remove_prefix
-{
-    my $str = shift;
-    
-    $str =~ s/^@//;
-    $str =~ s/^\+//;
-    $str =~ s/^%//;
-
-    return $str;
-}
-
 sub remove_braces
 {
     my $str = shift;

modules/Pisg/Parser/Logfile.pm

@@ -315,7 +315,7 @@ sub _parse_file
 
                     $stats->{shouts}{$nick}++
                         if (index($saying, '!') > -1);
-
+                    study $saying;
                     if ($saying !~ /[a-z]/o && $saying =~ /[A-Z]/o) {
                         # Ignore single smileys on a line. eg. '<user> :P'
                         if ($saying !~ /^[8;:=][ ^-o]?[)pPD}\]>]$/o) {
@@ -539,12 +539,18 @@ sub _parse_words
     my ($stats, $saying, $nick, $ignorewords, $hour) = @_;
     # Cache time of day
     my $tod = int($hour/6);
+    my $word;
 
-    foreach my $word (split(/[\s,!?.:;)(\"]+/o, $saying)) {
+    study $saying;
+    $saying =~ s/[\s,!?.:;)(\"]+//o;
+    while ($saying =~ s/([^\s,!?.:;)(\"]+)[\s,!?.:;)(\"]+//og) {
+        $word = $1;
         $stats->{words}{$nick}++;
         $stats->{word_times}{$nick}[$tod]++;
         # remove uninteresting words
-        next if $ignorewords and $word =~ m/$ignorewords/i;
+        next if $ignorewords and $word =~ m/$ignorewords/io;
+        # TODO: does the /o above provide the correct behaviour?
+        # next if $ignorewords and $word =~ m/$ignorewords/i;
 
         # ignore contractions
         next if ($word =~ m/'.{1,2}$/o);

pisg.cfg

@@ -10,10 +10,9 @@
 #
 # <set FoulWords="ass bitch fuck" Maintainer="SomeMaintainer">
 #
-# <channel="#channel">
-#   Logfile="channel2.log"
-#   Format="mIRC"
-#   Network="SomeIRCNet"
-#   OutputFile="index.html"
-# </channel>
+<channel="#channel">
+   Logfile="../logsamples/bxlog-sample.log"
+   Network="SomeIRCNet"
+   OutputFile="index.html"
+</channel>
 #