#!/usr/bin/perl
# @(#) newsdump Version 1.0
# dump contents of a news server into a directory.
#
# The NNTP protocol can be found in RFC977
# RFC1036 describes the format of the messages.
#
# Copyright (C) Ver 1.0 Jun 25 1996 Gaspar Sinai <gsinai@gol.com>
# Copyright (C) Ver 0.9 Jun 19 1996 Gaspar Sinai <gsinai@gol.com>
# Copyright (C) Ver 0.8 Jun  2 1996 Gaspar Sinai <gsinai@gol.com>
# Copyright (C) Ver 0.6 Feb  6 1996 Gaspar Sinai <gsinai@gol.com>
#
# Don't worry - this program is all yours! Please read the separate
# file called COPYING
#

$VERBOSE=0;

@day = ('Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri','Sat');
@month = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec');

$ALARM=300;
#
# 
while ($#ARGV >=0)
{
	if( $#ARGV > 0 && $ARGV[0] eq "-o")
	{
		$NWDIR=$ARGV[1];shift;shift;
	}
	elsif( $#ARGV > 0 && $ARGV[0] eq "-f")
	{
		$STATFILE=$ARGV[1];shift;shift;
	}
	elsif( $#ARGV > 0 && $ARGV[0] eq "-s")
	{
		$SERVER=$ARGV[1];shift;shift;
	}
	elsif( $#ARGV > 0 && $ARGV[0] eq "-e")
	{
		$FILTER=$ARGV[1];shift;shift;
	}
	elsif( $#ARGV > 0 && $ARGV[0] eq "-t")
	{
		$ALARM=$ARGV[1];shift;shift;
	}
	elsif( $#ARGV > 0 && $ARGV[0] eq "-i")
	{
		$NEWSPORT=$ARGV[1];shift;shift;
	}
	elsif( $ARGV[0] eq "-v")
	{
		$VERBOSE++;shift;
	}
	elsif( $ARGV[0] eq "-m")
	{
		$MAILFILE=1;shift;
	}
	elsif( $ARGV[0] eq "-g")
	{
		$GROUPS=1;shift;
	}
	elsif( $ARGV[0] eq "-u")
	{
		$UPDATE++;shift;
	}
	else
	{
		last;
	}
}

if ( defined ($GROUPS))
{
	if (!defined ($SERVER))
	{
		&NWUsage;
		exit (255);
	}
}
elsif ( defined ($UPDATE))
{
	if (!defined ($STATFILE) || !defined ($SERVER))
	{
		&NWUsage;
		exit (255);
	}
}
elsif (!defined ($NWDIR) || !defined ($STATFILE) || !defined ($SERVER))
{
	&NWUsage;
	exit (255);
}
$NEWSPORT="nntp" unless (defined ($NEWSPORT));

if ( defined ($GROUPS))
{
	&NWOpen ("$SERVER", $NEWSPORT);
	($status, $smsg) = &NWCommand ("MODE READER");
	($status, $smsg) = &NWCommand ("LIST");
	if ($status != 215)
	{
		($status, $smsg) = &NWCommand ("QUIT");
		exit (1);
	}
	print @ARTICLE 	if (&NWGetArticle);
	($status, $smsg) = &NWCommand ("QUIT");
	exit (2);
}

if ( -f $STATFILE)
{
	die "$0: Can not open $STATFILE\n" unless open (NWSTATFILE, "<$STATFILE");
	while (<NWSTATFILE>)
	{
		next if ( /^#.*/o );
		unless ( /(\S*)[\t ]*(\S*)/ )
		{
			print STDERR "$0: Bad line in $STATFILE: $_";
			next;
		}
		$WNSTATUS{"$1"} = sprintf ("%d", $2);
	}
	close (NWSTATFILE);
}

&NWAddSignals ('NWDie');

#
# If the program is stuck for more than 5 minutes than bailing out.
#
alarm ($ALARM);

$UID = (getpwuid($<))[2];
$LOCK = "/tmp/loadnews.$UID";

# No locking yet. For future releases.

&NWOpen ("$SERVER", $NEWSPORT);

($status, $smsg) = &NWCommand ("MODE READER");

#
# Newsgroup names come 
#
while (<>)
{
	next if ( /^#.*/o );
	unless(/(\S*)[\t ]*.*/o)
	{
		print STDERR "$0: Bad line: $_";
		next;
	}
	
	$group=$1;
	if ( defined(%WNSTATUS) && defined ( $WNSTATUS{ "$group" }) 
		&& $WNSTATUS{ "$group" } > 0)
	{
		$start=$WNSTATUS{ "$group" };
	}
	else
	{
		$start=0;
	}

	$NUMARTICLES=0;
	$stop= &NWLoadGroupFrom($group, $start); 

	if ($stop > $start)
	{
		print "loaded $NUMARTICLES articles from $group \n" if($VERBOSE);
	}
}

&NWSaveStatus;

exit(0);

#-------------------------------------------------------------------------------
#  S U B R O U T I N E S
#-------------------------------------------------------------------------------

#
# Load a newsgroup from an indexpoint
#
sub
NWLoadGroupFrom
{
	local ($to)=-1;
	local ($newsgroup, $from) = @_;

	($status, $smsg) = &NWCommand ("GROUP $newsgroup");
	if ($status!=211 || $smsg !~ /^(\d*) (\d*) (\d*) .*/ )
	{
		print STDERR "$0: Can not retrieve articles from $newsgroup\n";
		print STDERR "$status $smsg\n";
		return -1;
	}

	if ($VERBOSE)
	{
		print "$newsgroup articles from $from ($2-$3)\n";
	}

	if ( defined ($UPDATE) )
	{
		if ( !defined ($WNSTATUS{"$newsgroup"}) 
			|| $WNSTATUS{"$newsgroup"} < $3 )
		{
			$WNSTATUS{"$newsgroup"} = $3;
		}
		
		return $3;
	}

	if ($3 == $from)
	{
		print "No new articles in $newsgroup\n" if ($VERBOSE);
		return 0;
	}
	if ($3 < $from)
	{
		print "No new articles in $newsgroup\n" if ($VERBOSE);
		$WNSTATUS{"$newsgroup"}=$3;

		# Articles might have been renumbered or expired
		if ($VERBOSE)
		{
			print STDERR "WARNING: reset $newsgroup counter to $3 (expected $from).\n";
		}
		return $3;
	}
	
	## print "listgroup\n"; 
	($status, $smsg) = &NWCommand ("LISTGROUP $newsgroup");
	if ($status!=211)
	{
		print STDERR "$0: Can not retrieve articles from $newsgroup\n";
		print STDERR "$status $smsg\n";
		return -1;
	}
	(@list)=&NWGetList;
	if ( $VERBOSE && $#LIST < 0)
	{
		print "No articles in $newsgroup\n";
	}
	## print "index $from\n";
	foreach $index (@list)
	{
		$index=sprintf("%d", $index);
		next if ($index <= $from);
		($status, $smsg) = &NWCommand ("ARTICLE $index");
		if ($status!=220)
		{
			print STDERR "$0: Can not retrieve $index from $newsgroup\n";
			print STDERR "$status $smsg\n";
			next;
		}
		unless (&NWGetArticle)
		{
			printf "Skipping $index from $newsgroup\n" if ($VERBOSE > 1);
			next;
		}
		printf "Loading $index from $newsgroup\n" if ($VERBOSE > 1);
		alarm (0);
		&NWAddSignals ('IGNORE');
		if ( defined ($MAILFILE))
		{
			$WCSAVE = "$NWDIR/NEWS.$newsgroup";
			$OPENWC = ">>$WCSAVE";
		}
		else
		{
			$WCSAVE = "$NWDIR/$newsgroup:$index";
			$OPENWC = ">$WCSAVE";
		}
		unless ( open (WCSAVE, "$OPENWC"))
		{
			print STDERR "$0: Can not open file $WCSAVE";
			undef ($WCSAVE);
			&NWAddSignals ('NWDie');
			alarm ($ALARM);
			next;
		}
		if ( defined ($MAILFILE))
		{
			($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) 
				= gmtime(time);
			printf (WCSAVE "From newsdump %s %s %2d %02d:%02d:%02d GMT 19%02d\n",
			$day[$wday],$month[$mon],$mday,$hour,$min,$sec,$year);
		}
		print WCSAVE @ARTICLE;
		print WCSAVE "\n" if ( defined ($MAILFILE));

		close ($WCSAVE);
		undef ($WCSAVE);

		$NUMARTICLES++;
		if ( !defined ($WNSTATUS{"$newsgroup"}) 
			|| $WNSTATUS{"$newsgroup"} < $index )
		{
			$WNSTATUS{"$newsgroup"} = $index;
		}
		&NWAddSignals ('NWDie');
		alarm ($ALARM);
	}
	return -1 if ( !defined ($WNSTATUS{"$newsgroup"}));
	$WNSTATUS{"$newsgroup"};
}

#
# Save the status file
#
sub
NWSaveStatus
{
	unless ( open (NWSTATFILE, ">$STATFILE"))
	{
		print STDERR "$0: Can not save $STATFILE";
		unlink "$LOCK";
		exit (255);
	}
	while( (($key,$vle) = each( %WNSTATUS)))
	{
		print NWSTATFILE "$key\t$vle\n";
	}
	close (NWSTATFILE);
}

#
# The connection is broken with the news feeed. Update files and exit.
#
sub 
NWDie 
{
	local($msg) = $_[0];

	print STDERR "An error occurred: $msg\n";
	exit (255) unless defined ($STATFILE);

	print STDERR "Saving $STATFILE.\n";
	&NWSaveStatus();
	exit (255);
}


#
# Send a command to the server and wait for the reply
# Two strings are returned. The first is the reply number
# and the second is the reply string. 
#
sub
NWCommand
{
	print NW "$_[0]\r\n" if ($_[0] ne "NOP");
	$_=<NW>;
	/(\d\d\d) (.*)\r\n/;
	return ($1,$2);
}

#
# usage: NWOpen server port 
#
# Open an NNTP server with a port number.
# If user and password is omitted no authentication is done.
# port can be specified by its name like "nntp"
#
sub 
NWOpen 
{
	local($server, $port, $user, $pass) = @_;

	$sockaddr = 'S n a4 x8';

	if ($server =~ /^(\d+)+\.(\d+)\.(\d+)\.(\d+)$/)
	{
		$serveraddr = pack('C4', $1, $2, $3, $4);
	}
	else
	{
    		&NWDie ("$0: Unkown host: $server.") 
			unless (local(@arr) = gethostbyname($server));
		$serveraddr = $arr[4];
	}

    	unless (($name, $aliases, $proto) = getprotobyname('tcp'))
	{
		&NWDie ("$0: Unkown protocol 'tcp'");
	}
	if ($port !~ /^\d+$/ )
	{
		unless (($name, $aliases, $np) = getservbyname($port,'tcp'))
		{
			&NWDie ("$0: Unkown service '$port'");
		}
		$port=$np;
	}

	$serverproc = pack($sockaddr, 2, $port, $serveraddr);

	#
	# Hardwire to AF_SOCKET, $SOCK_STREAM, 'tcp'
	# Well, it used to be constant on all platforms but now, 
        # on Solaris $SOCK_STREAM is 2 that's why the second "try"
        # I dont want to include any perl files for only this...
	&NWDie ("$0: Can not make socket.") 
		unless (socket(NW, 2, 1, 6) || socket(NW, 2, 2, 6));
	
	unless (connect(NW, $serverproc))
	{
		close (NW);
		&NWDie ("$0: Can not connect socket");
	}

	select((select(NW), $| = 1)[0]);

	($status, $msg) = &NWCommand ( "NOP" );
	unless ($status == 200 || $status == 201)
	{
		&NWDie ( "Could not contact nntpd. $msg");
	}

	if ( defined ($user) && defined ($pass))
	{
		($status, $msg) = &NWCommand ( "AUTHINFO user $user" );
		&NWDie ("Could not Authenticate. $msg") unless ($status == 381);
		($status, $msg) = &NWCommand ( "AUTHINFO pass $pass" );
		&NWDie ("Could not Authenticate. $msg") unless ($status == 281);
	}
}

#
# Get a long text buffer into list.
#
sub
NWGetList
{
	$_ = <NW>;
	s/\r$//;
	@LIST=($_);

	while ($_ ne ".\n")
	{
		alarm ($ALARM);
		$_ = <NW>;
		s/\r$//;
	##	print "$_";
		@LIST=(@LIST, $_);
	}
	$#LIST=$#LIST-1;
	@LIST;
}

#
# Print out usage message
#
sub
NWUsage
{
print <<EOTXT;
newsdump version 1.0 usage:
   list groups:  newsdump -g [-e filter] [-i port] -s server
   update state: newsdump -u [-i port] [-v] [-m] -f state-file \\
                 -s server [newsgroups-file]
   news dump:    newsdump [-e filter] [-v]  [-m] [-t timeout] [-i port] \\
                 -o dump-directory -f state-file -s  server [newsgroups-file]
EOTXT
}

#
# Get a long text buffer into list.
#
sub
NWGetArticle
{
	alarm ($ALARM);
	$retrieved=0;
	$#ARTICLE=-1;

	do {
		$_ = <NW>;
		s/\r$//;
		alarm ($ALARM);

		$match=0;
		$match=1 if (defined ($FILTER) && /$FILTER/);

		if ( defined ($GROUPS) && defined ($FILTER))
		{
			@ARTICLE=(@ARTICLE, $_) if ($match);
		}
		else
		{
			@ARTICLE=(@ARTICLE, $_);
		}

		$retrieved=$retrieved | $match;
	} while ($_ ne ".\n");

	$#ARTICLE=$#ARTICLE-1 if ($ARTICLE[$#ARTICLE]  eq ".\n");
	return ($retrieved) if (defined ($FILTER));
	return 1;
}

#
# Assign signal handlers for some signals
#
sub
NWAddSignals
{
	local ($signal) = $_[0];
	if ($signal eq 'IGNORE')
	{
		$SIG{'INT'} = 'NWSigSave';
	}
	elsif (defined ($SIGCAUGHT))
	{
		&NWDie ($SIGCAUGHT);
	}
	else 
	{
		$SIG{'INT'} = $signal;
		$SIG{'QUIT'} = $signal;
		$SIG{'TERM'} = $signal;
		$SIG{'ALRM'} = $signal;
	}
}

#
# This handler is invoked when we don't want any signal handler.
# It saves the last incoming signal in $SIGCAUGHT
#
sub
NWSigSave
{
	$SIGCAUGHT=$_[0];
}

