#!%PERL%
# Copyright (c) vhffs project and its contributors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without 
# modification, are permitted provided that the following conditions 
# are met:
#
# 1. Redistributions of source code must retain the above copyright 
#   notice, this list of conditions and the following disclaimer.
#2. Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in 
#   the documentation and/or other materials provided with the 
#   distribution.
#3. Neither the name of vhffs nor the names of its contributors 
#   may be used to endorse or promote products derived from this 
#   software without specific prior written permission.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
#"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
#LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
#FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
#COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
#INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
#BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
#CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
# POSSIBILITY OF SUCH DAMAGE.


#Your logs must be in format : "%V %h %l %u %t \"%r\" %>s %b"
#So, add in your httpd.conf following lines :
#LogFormat "%V %h %l %u %t \"%r\" %>s %b" vhffs
#CustomLog /var/log/apache2/http.log vhffs

use strict;
use utf8;
use lib '%VHFFS_LIB_DIR%';
use Vhffs::Main;
use Vhffs::Conf;
use Vhffs::Functions;
use Vhffs::Services::Repository;
use Vhffs::Robots;


my $vhffs = init Vhffs::Main;
die "ERROR: Cannot init VHFFS !!!" if( ! defined $vhffs );

my $repos = Vhffs::Services::Repository::getall( $vhffs , Vhffs::Constants::ACTIVATED );
die "ERROR: Cannot fetch the list of repository\n" if( ! defined $repos );

my $repoconf = $vhffs->get_config->get_service('repository');
die "ERROR: A full repository configuration is needed to generate stats\n" if ( ! defined $repoconf );

my $conf_datadir = Vhffs::Conf::get_datadir;
my $log_incoming_root = $repoconf->{"log_incoming_root"};
my $log_incoming_rotations = $repoconf->{"log_incoming_rotations"};
my $log_incoming_compress = ( $repoconf->{"log_incoming_compress"} eq 'yes' ) ? 1 : 0;
my $log_parsed_root = $repoconf->{"log_parsed_root"};
my $log_parsed_rotation = $repoconf->{"log_parsed_rotation"};
my $log_parsed_compress = ( $repoconf->{"log_parsed_compress"} eq 'yes' ) ? 1 : 0;
my $log_awstats = $repoconf->{"log_awstats"};
my $awstats_sample = $repoconf->{"log_awstats_sample"};
my $log_apachelocal = ( $repoconf->{"log_apachelocal"} eq 'yes' ) ? 1 : 0;
my $log_apachectl = $repoconf->{"log_apachectl"};

die "ERROR: ".$log_incoming_root." is not a directory" if( ! -d $log_incoming_root );
die "ERROR: ".$log_parsed_root." is not a directory" if( ! -d $log_parsed_root );
die "ERROR: ".$log_awstats." does no exist" if( ! -f $log_awstats );
die "ERROR: cannot find the awstat sample at ".$awstats_sample if( ! -f $awstats_sample );
die "ERROR: cannot find the apache2ctl binary at ".$log_apachectl if( $log_apachelocal  &&  ! -f $log_apachectl );

Vhffs::Robots::lock( $vhffs , "repositorystats" );

my %repositorys;
foreach ( @{$repos} )  {
	$repositorys{$_->get_name} = 1;
}

my @downloadservers;
my $downloadserver;
my @files;

# -- Rotate servers logs (all *.log files, I know that the suffix is hardcoded but I don't
#    bother to add a configuration entry for that, it's already too complicated, and, who would
#    like anything other than .log ? ).

opendir( DIR , $log_incoming_root );
@files = readdir( DIR );

foreach( @files )  {
	next if( /\./ );
	if( -d $log_incoming_root."/".$_ )  {
		push @downloadservers, $_;
	}
}
closedir( DIR );

foreach $downloadserver ( @downloadservers )  {
	opendir ( DIR , $log_incoming_root."/".$downloadserver );
	@files = readdir( DIR );

	foreach ( @files )  {
		if( /.*\.log$/ )  {
			Vhffs::Functions::rotate_log( $log_incoming_root."/".$downloadserver."/".$_ , $log_incoming_rotations , $log_incoming_compress );
		}
	}

	# put a file to tell webserver to restart (ugly signaling over nfs, I know I know... but IT WORKS, it is secure, and doesn't consume too much CPU !)
	open( FILECYCLE , "> ".$log_incoming_root."/".$downloadserver."/cycleok" );
	print FILECYCLE "";
	close( FILECYCLE );
}

# Restart locally or wait 180 seconds to be sure all apache are restarted (welcome to the land of pigs)
if( $log_apachelocal )  {
	my $cmd = $log_apachectl." graceful";
	system ( $cmd );
} else {
	sleep ( 180 );
}

# Deleting previous logs
unlink $log_incoming_root."/mergedlog" if( -f $log_incoming_root."/mergedlog" );
unlink $log_incoming_root."/rejectlog" if( -f $log_incoming_root."/rejectlog" );

# Merge all logs
my $cmd = "mergelog ".$log_incoming_root."/*/http.log.0 ".$log_incoming_root."/*/ftp.log.0 > ".$log_incoming_root."/mergedlog";
system ( $cmd );

# Parse http logs
my $prev = "";
my $ddir = $conf_datadir."/repository/";
$ddir =~ s%[/]{2,}%/%g;

open( MERGEDIN , "< ".$log_incoming_root."/mergedlog" );
open( REJECTOUT, "> ".$log_incoming_root."/rejectlog" );

while( <MERGEDIN> )  {

	my ( $remotehost , $rfc931 , $authuser , $date , $request , $status , $size , $referer , $useragent ) = ( $_ =~ /^([^\s]*)\s+([^\s]*)\s+([^\s]*)\s+\[([^\]]*)\]\s+\"([^\"]*)\"\s+([^\s]*)\s+([^\s]*)(?:\s+\"([^\"]*)\")?(?:\s+\"([^\"]*)\")?$/ );

	next unless ( defined $remotehost && defined $rfc931 && defined $authuser && defined $date && defined $request && defined $status && defined $size );

	# define referer and useragent (convert common to combined log)
	$referer = '-' unless defined $referer;
	$useragent = '-' unless defined $useragent;

	# remove the "/data/repository/" part of the query
	$request =~ s%$ddir/*%/%;

	# remove the http:// part of the query if it exists
	$request =~ s%http://[^/]+/*%/%;

	# add HTTP/1.0 at the end of the query if needed
	$request .= ' HTTP/1.0' if( $request && $request !~ /\ HTTP\/1.[01]$/ );

	# fetch the group
	my ( $area ) = ( $request =~ /^[^\/]*\/([^\/]+)/ ); 

	# rebuild
	my $log = $remotehost.' '.$rfc931.' '.$authuser.' ['.$date.'] "'.$request.'" '.$status.' '.$size.' "'.$referer.'" "'.$useragent.'"';

	# append log line to the concerned download area
	if ( defined $area && defined $log )  {

		# We are _NOT_ hosting this repository
		if( ! exists $repositorys{$area} )  {

			print REJECTOUT $area." ".$log."\n";
		}

		# We host this repository
		else  {
			# the repository changed
			if ( $prev ne $area )  {
				my $dir = $log_parsed_root."/".$area."/logs";
				Vhffs::Functions::create_dir($dir) if ( ! -d $dir );

				my $lff = $dir."/access.log";
				close(FILEOUT);
				open( FILEOUT , ">> ".$lff );
				$prev = $area;
			}

			print FILEOUT $log."\n";
		}
	}
}

close(MERGEDIN);
close(REJECTOUT);
close(FILEOUT);


# Create a configuration file and generate statistic for each website
foreach ( @{$repos} )
{
	my $reponame = $_->get_name;

	my $weblogdir = $log_parsed_root."/".$reponame;
	my $logpath = $weblogdir."/logs/access.log";
	my $datadir = $weblogdir."/awstats";
	my $conffile = $datadir."/awstats.".$reponame.".conf";

	next if ( ! -f $logpath );
	Vhffs::Functions::create_dir($datadir) if ( ! -d $datadir );

	# Create the config file
	open( AWFILEIN , "< ".$awstats_sample );
	open( AWFILEOUT , "> ".$conffile );

	while( my $line = <AWFILEIN> )
	{
		$line =~ s/MY_DOMAINNAME/$reponame/g;
		$line =~ s/MY_LOGPATH/$logpath/g;
		$line =~ s/MY_DATADIR/$datadir/g;
		
		print AWFILEOUT $line;
	}

	close( AWFILEOUT );
	close( AWFILEIN );

	# Generate statistics
	$cmd = $log_awstats." -config=".$reponame." -update 2>/dev/null 1>/dev/null";
	system( $cmd );

	# Rotate logs for this website
	Vhffs::Functions::rotate_log( $logpath , $log_parsed_rotation , $log_parsed_compress );
}

Vhffs::Robots::unlock( $vhffs , "repositorystats" );
