git-svn-id: svn://occams/govtrack/gather/us@1304 6c5aeb89-041d-0410-8…

…b88-fd54b8401f9f

database.people.sql

@@ -597,7 +597,7 @@ INSERT INTO `people` VALUES (400001,'Neil',NULL,NULL,'Abercrombie',NULL,'Abercro
 (300093,'Debbie Ann|Deborah',NULL,NULL,'Stabenow',NULL,'Stabenow',NULL,'1950-04-29','F','United Methodist','N00004118','S000770',515,'S8MI00281','Debbie_Stabenow','SenatorStabenow','StabenowPress','S284',NULL),
 (300094,'Ted','F.',NULL,'Stevens',NULL,'Stevens',NULL,'1923-11-18','M','Episcopalian','N00007997','S000888',53268,'S2AK00010','Ted_Stevens',NULL,NULL,'S090',12109),
 (300095,'John','E.',NULL,'Sununu',NULL,'Sununu',NULL,'1964-09-10','M','Catholic','N00000397','S001078',617,'S0NH00201','John_Sununu',NULL,NULL,'S296',NULL),
-(300096,'James','M.','Jim','Talent',NULL,'Talent',NULL,'1956-10-18','M','Presbyterian','N00005004','T000024',NULL,NULL,'James_Talent',NULL,NULL,NULL,NULL),
+(300096,'James','M.','Jim','Talent',NULL,'Talent',NULL,'1956-10-18','M','Presbyterian','N00005004','T000024',NULL,NULL,'James_Talent',NULL,NULL,'S286',NULL),
 (300097,'Craig',NULL,NULL,'Thomas',NULL,'Thomas',NULL,'1933-02-17','M','Methodist','N00006246','T000162',NULL,NULL,'Craig_Thomas',NULL,NULL,'S246',15633),
 (300098,'George','V.',NULL,'Voinovich',NULL,'Voinovich',NULL,'1936-07-15','M','Roman Catholic','N00003583','V000126',21745,'S8OH00011','George_Voinovich','SenVoinovich',NULL,'S271',NULL),
 (300099,'John','W.',NULL,'Warner',NULL,'Warner',NULL,'1927-02-18','M','Episcopalian','N00002061','W000154',53356,'S8VA00107','John_Warner',NULL,NULL,'S143',14712),
@@ -1251,7 +1251,7 @@ INSERT INTO `people` VALUES (400001,'Neil',NULL,NULL,'Abercrombie',NULL,'Abercro
 (401144,'David',NULL,NULL,'Barker','Jr.','Barker',NULL,'1797-01-08','M',NULL,NULL,'B000143',NULL,NULL,NULL,NULL,NULL,NULL,435),
 (401145,'Joseph',NULL,NULL,'Barker',NULL,'Barker',NULL,'1751-10-19','M',NULL,NULL,'B000144',NULL,NULL,NULL,NULL,NULL,NULL,436),
 (401146,'Alben','William',NULL,'Barkley',NULL,'Barkley',NULL,'1877-11-24','',NULL,NULL,'B000145',NULL,NULL,NULL,NULL,NULL,NULL,437),
-(401147,'Dean',NULL,NULL,'Barkley',NULL,'Barkley',NULL,'1950-08-31','M',NULL,NULL,'B001237',NULL,NULL,NULL,NULL,NULL,NULL,NULL),
+(401147,'Dean',NULL,NULL,'Barkley',NULL,'Barkley',NULL,'1950-08-31','M',NULL,NULL,'B001237',NULL,NULL,NULL,NULL,NULL,'S285',NULL),
 (401148,'Ethelbert',NULL,NULL,'Barksdale',NULL,'Barksdale',NULL,'1824-01-04','',NULL,NULL,'B000146',NULL,NULL,NULL,NULL,NULL,NULL,438),
 (401149,'William',NULL,NULL,'Barksdale',NULL,'Barksdale',NULL,'1821-08-21','M',NULL,NULL,'B000147',NULL,NULL,NULL,NULL,NULL,NULL,439),
 (401150,'Bradley',NULL,NULL,'Barlow',NULL,'Barlow',NULL,'1814-05-12','',NULL,NULL,'B000148',NULL,NULL,NULL,NULL,NULL,NULL,440),
@@ -4381,7 +4381,7 @@ INSERT INTO `people` VALUES (400001,'Neil',NULL,NULL,'Abercrombie',NULL,'Abercro
 (404278,'Andrew','Fuller',NULL,'Fox',NULL,'Fox',NULL,'1849-04-26','M',NULL,NULL,'F000330',NULL,NULL,NULL,NULL,NULL,NULL,3330),
 (404279,'John',NULL,NULL,'Fox',NULL,'Fox',NULL,'1835-06-30','M',NULL,NULL,'F000331',NULL,NULL,NULL,NULL,NULL,NULL,3331),
 (404280,'Jon','D.',NULL,'Fox',NULL,'Fox',NULL,'1947-04-22','',NULL,NULL,'F000332',NULL,NULL,NULL,NULL,NULL,NULL,NULL),
-(404281,'Sheila',NULL,NULL,'Frahm',NULL,'Frahm',NULL,NULL,'',NULL,NULL,'F000438',NULL,NULL,NULL,NULL,NULL,NULL,NULL),
+(404281,'Sheila',NULL,NULL,'Frahm',NULL,'Frahm',NULL,NULL,'',NULL,NULL,'F000438',NULL,NULL,NULL,NULL,NULL,'S248',NULL),
 (404282,'Joseph','Irvin',NULL,'France',NULL,'France',NULL,'1873-10-11','M',NULL,NULL,'F000333',NULL,NULL,NULL,NULL,NULL,NULL,3332),
 (404283,'Richard',NULL,NULL,'Franchot',NULL,'Franchot',NULL,'1816-06-02','M',NULL,NULL,'F000334',NULL,NULL,NULL,NULL,NULL,NULL,3333),
 (404284,'George','Blinn',NULL,'Francis',NULL,'Francis',NULL,'1883-08-12','M',NULL,NULL,'F000335',NULL,NULL,NULL,NULL,NULL,NULL,3334),
@@ -6585,7 +6585,7 @@ INSERT INTO `people` VALUES (400001,'Neil',NULL,NULL,'Abercrombie',NULL,'Abercro
 (406484,'George','Frederic',NULL,'Kribbs',NULL,'Kribbs',NULL,NULL,'M',NULL,NULL,'K000330',NULL,NULL,NULL,NULL,NULL,NULL,5368),
 (406485,'John',NULL,NULL,'Kronmiller',NULL,'Kronmiller',NULL,'1858-12-06','M',NULL,NULL,'K000331',NULL,NULL,NULL,NULL,NULL,NULL,5369),
 (406486,'Otto',NULL,NULL,'Krueger',NULL,'Krueger',NULL,NULL,'',NULL,NULL,'K000332',NULL,NULL,NULL,NULL,NULL,NULL,5370),
-(406487,'Robert','Charles',NULL,'Krueger',NULL,'Krueger',NULL,'1935-09-19','M',NULL,NULL,'K000333',NULL,NULL,NULL,NULL,NULL,NULL,14247),
+(406487,'Robert','Charles',NULL,'Krueger',NULL,'Krueger',NULL,'1935-09-19','M',NULL,NULL,'K000333',NULL,NULL,NULL,NULL,NULL,'S234',14247),
 (406488,'Edward','H.',NULL,'Kruse',NULL,'Kruse',NULL,'1918-10-22','M',NULL,NULL,'K000334',NULL,NULL,NULL,NULL,NULL,NULL,5371),
 (406489,'Thomas','Henry',NULL,'Kuchel',NULL,'Kuchel',NULL,'1910-08-15','M',NULL,NULL,'K000335',NULL,NULL,NULL,NULL,NULL,NULL,5372),
 (406490,'Joseph','Henry',NULL,'Kuhns',NULL,'Kuhns',NULL,NULL,'M',NULL,NULL,'K000337',NULL,NULL,NULL,NULL,NULL,NULL,5373),
@@ -7811,9 +7811,9 @@ INSERT INTO `people` VALUES (400001,'Neil',NULL,NULL,'Abercrombie',NULL,'Abercro
 (407712,'James','Monroe',NULL,'Miller',NULL,'Miller',NULL,'1852-05-06','M',NULL,NULL,'M000733',NULL,NULL,NULL,NULL,NULL,NULL,6488),
 (407713,'Jesse',NULL,NULL,'Miller',NULL,'Miller',NULL,NULL,'',NULL,NULL,'M000734',NULL,NULL,NULL,NULL,NULL,NULL,6489),
 (407714,'John',NULL,NULL,'Miller',NULL,'Miller',NULL,'1774-11-10','M',NULL,NULL,'M000737',NULL,NULL,NULL,NULL,NULL,NULL,6490),
-(407715,'John',NULL,NULL,'Miller',NULL,'Miller',NULL,'1781-11-25','M',NULL,NULL,'M000735',NULL,NULL,NULL,NULL,NULL,NULL,6491),
-(407716,'John','Elvis',NULL,'Miller',NULL,'Miller',NULL,'1888-05-15','M',NULL,NULL,'M000738',NULL,NULL,NULL,NULL,NULL,NULL,6492);
-INSERT INTO `people` VALUES (407717,'John','Franklin',NULL,'Miller',NULL,'Miller',NULL,'1831-11-21','M',NULL,NULL,'M000740',NULL,NULL,NULL,NULL,NULL,NULL,6493),
+(407715,'John',NULL,NULL,'Miller',NULL,'Miller',NULL,'1781-11-25','M',NULL,NULL,'M000735',NULL,NULL,NULL,NULL,NULL,NULL,6491);
+INSERT INTO `people` VALUES (407716,'John','Elvis',NULL,'Miller',NULL,'Miller',NULL,'1888-05-15','M',NULL,NULL,'M000738',NULL,NULL,NULL,NULL,NULL,NULL,6492),
+(407717,'John','Franklin',NULL,'Miller',NULL,'Miller',NULL,'1831-11-21','M',NULL,NULL,'M000740',NULL,NULL,NULL,NULL,NULL,NULL,6493),
 (407718,'John','Franklin',NULL,'Miller',NULL,'Miller',NULL,NULL,'M',NULL,NULL,'M000739',NULL,NULL,NULL,NULL,NULL,NULL,6494),
 (407719,'John','Gaines',NULL,'Miller',NULL,'Miller',NULL,'1812-11-29','M',NULL,NULL,'M000741',NULL,NULL,NULL,NULL,NULL,NULL,6495),
 (407720,'John','Krepps',NULL,'Miller',NULL,'Miller',NULL,'1819-05-25','M',NULL,NULL,'M000742',NULL,NULL,NULL,NULL,NULL,NULL,6496),
@@ -12452,7 +12452,7 @@ INSERT INTO `people` VALUES (407717,'John','Franklin',NULL,'Miller',NULL,'Miller
 (412397,'Paul',NULL,NULL,'Gosar',NULL,'Gosar',NULL,'1958-11-27','M',NULL,'N00030771','G000565',123491,NULL,NULL,'RepGosar','RepGosar',NULL,NULL),
 (412398,'Ben|Benjamin',NULL,NULL,'Quayle',NULL,'Quayle',NULL,'1976-11-05','M',NULL,'N00031723','Q000024',123506,NULL,NULL,'RepBenQuayle','benquayle',NULL,NULL),
 (412399,'David',NULL,NULL,'Schweikert',NULL,'Schweikert',NULL,'1962-03-03','M',NULL,'N00006460','S001183',106387,NULL,NULL,'RepDavidSchweikert','RepDavid',NULL,NULL),
-(412400,'Eric','A.','Rick\r\n','Crawford',NULL,'Crawford',NULL,'1966-01-22','M',NULL,'N00030770','C001087',119208,NULL,NULL,NULL,NULL,NULL,NULL),
+(412400,'Eric','A.','Rick','Crawford',NULL,'Crawford',NULL,'1966-01-22','M',NULL,'N00030770','C001087',119208,NULL,NULL,NULL,NULL,NULL,NULL),
 (412401,'Tim',NULL,NULL,'Griffin',NULL,'Griffin',NULL,'1968-08-21','M',NULL,'N00031297','G000567',119213,NULL,NULL,'RepTimGriffin','RepTimGriffin',NULL,NULL),
 (412402,'Steve',NULL,NULL,'Womack',NULL,'Womack',NULL,'1957-02-18','M',NULL,'N00031857','W000809',71815,NULL,NULL,'CongressmanWomack','rep_stevewomack',NULL,NULL),
 (412403,'Jeff',NULL,NULL,'Denham',NULL,'Denham',NULL,'1967-07-29','M',NULL,'N00031593','D000612',28769,NULL,NULL,'jeffdenham19','RepJeffDenham',NULL,NULL),
@@ -54638,4 +54638,4 @@ UNLOCK TABLES;
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2011-03-08 12:50:51
+-- Dump completed on 2011-04-03 13:34:48

database.tables.sql

@@ -99,7 +99,7 @@ CREATE TABLE `people_videos` (
   KEY `personid` (`personid`,`date`),
   KEY `date` (`date`),
   KEY `link` (`link`(127))
-) ENGINE=MyISAM AUTO_INCREMENT=12524929 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=12625615 DEFAULT CHARSET=utf8;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -169,4 +169,4 @@ CREATE TABLE `committees` (
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2011-03-08 12:50:50
+-- Dump completed on 2011-04-03 13:34:48

database.tables2.sql

@@ -211,7 +211,7 @@ CREATE TABLE `questions` (
   PRIMARY KEY (`id`),
   KEY `question` (`question`),
   KEY `topic` (`topic`(16))
-) ENGINE=MyISAM AUTO_INCREMENT=39778 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=40153 DEFAULT CHARSET=utf8;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -247,4 +247,4 @@ CREATE TABLE `votes` (
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2011-03-08 12:50:50
+-- Dump completed on 2011-04-03 13:34:48

parse_committeeschedule.pl

@@ -10,13 +10,14 @@
 
 if ($ARGV[0] eq "COMMITTEESCHEDULE") {
 	GovDBOpen();
-	FetchCommitteeSchedule();
+	FetchCommitteeSchedule($ARGV[1]);
 	DBClose();
 }
 
 1;
 
 sub FetchCommitteeSchedule {
+	my $dailydigest = shift;
 	my $SESSION = SessionFromDateTime(Now());
 
 	my $committeedata = $XMLPARSER->parse_file("../data/us/committees.xml");
@@ -48,7 +49,7 @@ sub FetchCommitteeSchedule {
 	}
 
 	FetchSenateCommitteeSchedule($xml);
-	FetchHouseCommitteeSchedule($xml);
+	FetchHouseCommitteeSchedule($xml, $dailydigest);
 
 	$xml->toFile($sfile, 1);
 }
@@ -63,8 +64,11 @@ sub ClearChamberCommitteeMeetings {
 
 sub FetchHouseCommitteeSchedule {
 	my $xml = shift;
+	my $dailydigest = shift;
+
+	if (!$dailydigest) { $dailydigest = "http://thomas.loc.gov/cgi-bin/dailydigest"; }
 
-	my ($content, $mtime) = Download("http://thomas.loc.gov/cgi-bin/dailydigest", nocache => 1);
+	my ($content, $mtime) = Download($dailydigest, nocache => 1);
 	if (!$content) { return; }
 
 	$content =~ s/:\n/: /g;
@@ -89,7 +93,7 @@ sub FetchHouseCommitteeSchedule {
 		#if ($line =~ /<center><strong>/) { last; }
 		if ($line =~ /Next Meeting/) { last; }
 
-		if ($line =~ /<em>((Permanent )?Select )?Committee on (the )?([\w\W]+?)(,|:)?<\/em>/) {
+		if ($line =~ /<em>((Permanent )?Select )?Committee on (the )?([\w\W]+?)([,:;])?<\/em>/) {
 			my $typ = $1;
 			$typ =~ s/ $//;
 
@@ -167,6 +171,7 @@ sub AddCommittee {
 			$cid = $CommitteeId{"$comm (Special)"};
 		}
 	}
+	if (!defined($cid)) { warn $comm; }
 
 	if ($sub ne "") { $comm .= " -- $sub"; }
 	$comm = ToUTF8($comm);

parse_rollcall.pl

@@ -1,3 +1,5 @@
+# for y in {2011..1989}; do CACHED=1 NOMAP=1 perl parse_rollcall.pl GET_ROLLS $y 0 no; done
+
 use Time::Local;
 use LWP::UserAgent;
 use GD;
@@ -38,7 +40,7 @@ sub DownloadRollCallVotesAll {
 	my $noopendb = shift;
 	my $skipifexists = shift;
 	my $forceindex = shift;
-	if (!defined($skipifexists)) { $skipifexists = 1; }
+	if (!defined($skipifexists) && !$ENV{CACHED}) { $skipifexists = 1; }
 
 	if ($YEAR eq "") { $YEAR = YearFromDate(time); }
 
@@ -1084,17 +1086,20 @@ sub normalize_vote_type {
 
 	if ($type eq "Call of the House") {
 		if ($question eq "Call of the House: QUORUM") {
-			return ("Quorum Call", "Procedural");
+			return ("Quorum Call", "procedural");
 		} else {
 			warn "Unhandled vote type: $type: $question";
-			return ($type, "Procedural");
+			return ($type, "procedural");
 		}
 
 	} elsif ($type eq "On Passage" || $type eq "On Passage of the Bill") {
 		return ("On Passage of the Bill", "passage");
 	} elsif ($type =~ s/^On (Agreeing to )?(Article \S+ of )?the (Concurrent |Joint )?Resolution(, As Amended)?$/On the $3Resolution/i) {
-		if ($2) { $type .= " (Part)"; }
-		return ($type, "passage-part");
+		if ($2) {
+			$type .= " (Part)";
+			return ($type, "passage-part");
+		}
+		return ($type, "passage");
 	} elsif ($type =~ /^(On Motion to )?(Concur in|Agree to|On Agreeing to) (the )?Senate (Amendment|amdt|Adt)s?( with an Amendment|with Amendment.*|to House (Amendment|Adt).*)?$|^Concurring|^On Concurring|Concur in (the )?Senate (Amdt|Amendment)|Concur In /i) {
 		return ("On the Senate Amendment", "passage");
 	} elsif ($type =~ /^(On Motion to )?Suspend ((the )?Rules )?and (Agree|Pass|Concur in the Senate Amendment|Agree to Senate Amendments?|Agree to S Adt to House Adts)(, As Amended)?$/i) {

pictures.pl

@@ -10,7 +10,7 @@
 		system("wget -O ../data/photos/$ARGV[1].jpeg \"$ARGV[2]\"");
 	} else {
 		if ($pid eq "") {
-			if ($ARGV[2] !~ /(^|\/)(\d+)\.jpeg$/) { die; }
+			if ($ARGV[2] !~ /(^|\/)(\d+)\.jpe?g$/i) { die; }
 			$pid = $2;
 		}
 		system("cp $ARGV[2] ../data/photos/$pid.jpeg");