cronjob

#!/usr/bin/perl

chdir "/home/govtrack/scripts/gather";

require "general.pl";
require "persondb.pl";
require "db.pl";

@A = @ARGV;
undef @ARGV;

my $today = DateToDBString(time);
my $session = SessionFromDateTime($today);

print `date`;

GovDBOpen();

&TestIsWeekly;

$OUTPUT_ERRORS_ONLY = 1;

if ($A[0] eq "EARLYGATHER") { DoEarlyGather(); }
if ($A[0] eq "" || $A[0] eq "GATHER") { DoNoonGather(); }
if ($A[0] eq "" || $A[0] eq "INDEX") { DoIndex(); }
if ($A[0] eq "EMAILS") { DoEmails(); }

print "$HTTP_BYTES_FETCHED bytes fetched.\n";
print `date`;

DBClose();

# Touch the website to force the cache to clear.
system("touch ../../website/style/master.xsl");

sub DoEarlyGather {
	# Update any bills whose THOMAS summary changed.
	require "parse_status.pl";
	UpdateBills($session);
	
	RefreshBills($session, "not(starts-with(\@updated, '${today}T')) and count(subjects/*)=0");

	# Get roll call votes
	DownloadRollCallVotesAll(undef, 1, 1, 1);

	require "parse_committeeschedule.pl";
	FetchCommitteeSchedule();

	require "fetchbilltext.pl";
	GetBillFullText($session);
	CreateGeneratedBillTexts($session);
}

sub DoNoonGather {
	DoEarlyGather();

	#require "parse_record.pl";
	#GetCR('s', $retdate, 1);
	#GetCR('h', $retdate, 1);

	require "fetchcbo.pl";
	GetCBO();
	
	require "fetchomb.pl";
	#GetOMB($retdate);
	
	#require "technorati.pl";
	#ScanTechnorati();
	
	#system("perl googlesitemap.pl");
	
	#system("perl congresspedia.pl > /dev/null");
}

sub DoIndex {
	# INDEXES
	
	# NEW STATS
	system("python ../analysis/sponsorshipanalysis.py $session");

	# REP STATS
	print "RepStats\n";
	require "repstat.pl";
	DoRepStats($session);
	print "Done RepStats\n";
	
	# SUMMARY
	print "StatsSummary\n";
	require "statsummary.pl";
	DoStatsSummary($session);
	
	system("perl repstat-vimg.pl $session");
	
	# LUCENE
	
	print "Lucene\n";
	system("sh lucene.sh $session");
	
	require "getpopularmonitors.pl";
	GetPopularMonitors();
	
	require "indexusc.pl";
	IndexBillsUSC($session);
	
	system('mysql -u govtrack_sandbox govtrack -Be "SELECT DISTINCT value FROM billindex WHERE idx = \"crs\"" > /home/govtrack/data/us/crs_terms');
	
	system('tail -10000000 ~/logs/access_log |perl access_log_smarts.pl');
	
	print `date`;
}


sub DoEmails {
	my $bin = "cd ../../website/bin; /usr/bin/mono GovTrackEmailUpdates.exe";
	system("$bin daily");

	if ($Weekly) { system("$bin weekly"); }
}

sub TestIsWeekly {
	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	$Weekly = ($wday == 0);
}