Merge pull request nikolamilosevic86#6 from dbmi-pitt/master

Some of these changes might be useful for the main - not sure

Author: nikolamilosevic86

DataBaseFileDrugs.sql

@@ -1,132 +1,126 @@
--- MySQL Script generated by MySQL Workbench
--- 02/16/16 11:27:32
--- Model: New Model    Version: 1.0
+-- MySQL Script to generate SQL database for the Table Annotator
+--
+-- NOTE: It is up to you what database you load these tables into!
+--
 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL,ALLOW_INVALID_DATES';
 
 -- -----------------------------------------------------
--- Schema table_db_drugs
+-- Table `Article`
 -- -----------------------------------------------------
-DROP SCHEMA IF EXISTS `table_db_drugs` ;
-CREATE SCHEMA IF NOT EXISTS `table_db_drugs` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci ;
-USE `table_db_drugs` ;
+DROP TABLE IF EXISTS `Article` ;
 
--- -----------------------------------------------------
--- Table `table_db_drugs`.`Article`
--- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`Article` ;
-
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`Article` (
+CREATE TABLE IF NOT EXISTS `Article` (
   `idArticle` INT NOT NULL AUTO_INCREMENT,
   `PMCID` VARCHAR(50) NULL,
   `PMID` VARCHAR(45) NULL,
   `pissn` VARCHAR(45) NULL,
   `eissn` VARCHAR(45) NULL,
-  `Title` VARCHAR(500) NULL,
-  `Abstract` VARCHAR(3000) NULL,
-  `JournalName` VARCHAR(145) NULL,
-  `JournalPublisherName` VARCHAR(245) NULL,
-  `JournalPublisherLocation` VARCHAR(200) NULL,
-  `Source` VARCHAR(100) NULL,
-  `SpecId` VARCHAR(120) NULL,
-  `PlainText` VARCHAR(10000) NULL,
+  `Title` VARCHAR(2000) NULL,
+  `Abstract` VARCHAR(5000) NULL,
+  `JournalName` VARCHAR(2000) NULL,
+  `JournalPublisherName` VARCHAR(2000) NULL,
+  `JournalPublisherLocation` VARCHAR(5000) NULL,
+  `Source` VARCHAR(500) NULL,
+  `SpecId` VARCHAR(500) NULL,
+  `PlainText` LONGTEXT NULL,
   PRIMARY KEY (`idArticle`))
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`Author`
+-- Table `Author`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`Author` ;
+DROP TABLE IF EXISTS `Author` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`Author` (
+CREATE TABLE IF NOT EXISTS `Author` (
   `idAuthor` INT NOT NULL AUTO_INCREMENT,
-  `FirstName` VARCHAR(45) NULL,
-  `LastName` VARCHAR(100) NULL,
+  `FirstName` VARCHAR(200) NULL,
+  `LastName` VARCHAR(200) NULL,
   `Article_idArticle` INT NOT NULL,
   PRIMARY KEY (`idAuthor`),
   INDEX `fk_Author_Article1_idx` (`Article_idArticle` ASC),
   CONSTRAINT `fk_Author_Article1`
     FOREIGN KEY (`Article_idArticle`)
-    REFERENCES `table_db_drugs`.`Article` (`idArticle`)
+    REFERENCES `Article` (`idArticle`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`ArtTable`
+-- Table `ArtTable`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`ArtTable` ;
+DROP TABLE IF EXISTS `ArtTable` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`ArtTable` (
+CREATE TABLE IF NOT EXISTS `ArtTable` (
   `idTable` INT NOT NULL AUTO_INCREMENT,
-  `TableOrder` VARCHAR(145) NULL,
-  `TableCaption` VARCHAR(545) NULL,
-  `TableFooter` VARCHAR(1445) NULL,
+  `TableOrder` VARCHAR(500) NULL,
+  `TableCaption` VARCHAR(5000) NULL,
+  `TableFooter` VARCHAR(5000) NULL,
   `StructureType` VARCHAR(45) NULL,
   `PragmaticType` VARCHAR(45) NULL,
   `HasXML` VARCHAR(40) NULL,
   `Article_idArticle` INT NOT NULL,
-  `Section` VARCHAR(245) NULL,
+  `Section` VARCHAR(1000) NULL,
   PRIMARY KEY (`idTable`),
   INDEX `fk_Table_Article1_idx` (`Article_idArticle` ASC),
   CONSTRAINT `fk_Table_Article1`
     FOREIGN KEY (`Article_idArticle`)
-    REFERENCES `table_db_drugs`.`Article` (`idArticle`)
+    REFERENCES `Article` (`idArticle`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`Cell`
+-- Table `Cell`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`Cell` ;
+DROP TABLE IF EXISTS `Cell` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`Cell` (
+CREATE TABLE IF NOT EXISTS `Cell` (
   `idCell` INT NOT NULL AUTO_INCREMENT,
-  `CellID` VARCHAR(45) NULL,
-  `CellType` VARCHAR(45) NULL,
+  `CellID` VARCHAR(200) NULL,
+  `CellType` VARCHAR(200) NULL,
   `Table_idTable` INT NOT NULL,
   `RowN` INT NULL,
   `ColumnN` INT NULL,
-  `HeaderRef` VARCHAR(45) NULL,
-  `StubRef` VARCHAR(45) NULL,
-  `SuperRowRef` VARCHAR(45) NULL,
-  `Content` VARCHAR(545) NULL,
-  `WholeHeader` VARCHAR(645) NULL,
-  `WholeStub` VARCHAR(655) NULL,
-  `WholeSuperRow` VARCHAR(645) NULL,
+  `HeaderRef` VARCHAR(200) NULL,
+  `StubRef` VARCHAR(200) NULL,
+  `SuperRowRef` VARCHAR(200) NULL,
+  `Content` VARCHAR(5000) NULL,
+  `WholeHeader` VARCHAR(5000) NULL,
+  `WholeStub` VARCHAR(5000) NULL,
+  `WholeSuperRow` VARCHAR(5000) NULL,
   PRIMARY KEY (`idCell`),
   INDEX `fk_Cell_Table1_idx` (`Table_idTable` ASC),
   CONSTRAINT `fk_Cell_Table1`
     FOREIGN KEY (`Table_idTable`)
-    REFERENCES `table_db_drugs`.`ArtTable` (`idTable`)
+    REFERENCES `ArtTable` (`idTable`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`CellRole`
+-- Table `CellRole`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`CellRole` ;
+DROP TABLE IF EXISTS `CellRole` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`CellRole` (
+CREATE TABLE IF NOT EXISTS `CellRole` (
   `idCellRole` INT NOT NULL AUTO_INCREMENT,
   `CellRoleName` VARCHAR(45) NULL,
   PRIMARY KEY (`idCellRole`))
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`CellRoles`
+-- Table `CellRoles`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`CellRoles` ;
+DROP TABLE IF EXISTS `CellRoles` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`CellRoles` (
+CREATE TABLE IF NOT EXISTS `CellRoles` (
   `idCellRoles` INT NOT NULL AUTO_INCREMENT,
   `CellRole_idCellRole` INT NOT NULL,
   `Cell_idCell` INT NOT NULL,
@@ -135,91 +129,91 @@ CREATE TABLE IF NOT EXISTS `table_db_drugs`.`CellRoles` (
   INDEX `fk_CellRoles_Cell1_idx` (`Cell_idCell` ASC),
   CONSTRAINT `fk_CellRoles_CellRole1`
     FOREIGN KEY (`CellRole_idCellRole`)
-    REFERENCES `table_db_drugs`.`CellRole` (`idCellRole`)
+    REFERENCES `CellRole` (`idCellRole`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION,
   CONSTRAINT `fk_CellRoles_Cell1`
     FOREIGN KEY (`Cell_idCell`)
-    REFERENCES `table_db_drugs`.`Cell` (`idCell`)
+    REFERENCES `Cell` (`idCell`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`Annotation`
+-- Table `Annotation`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`Annotation` ;
+DROP TABLE IF EXISTS `Annotation` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`Annotation` (
+CREATE TABLE IF NOT EXISTS `Annotation` (
   `idAnnotation` INT NOT NULL AUTO_INCREMENT,
-  `Content` VARCHAR(245) NULL,
+  `Content` VARCHAR(1000) NULL,
   `Start` INT NULL,
   `End` INT NULL,
-  `AnnotationID` VARCHAR(245) NULL,
-  `AnnotationSchemaVersion` VARCHAR(245) NULL,
-  `AnnotationDescription` VARCHAR(500) NULL,
-  `AnnotationURL` VARCHAR(245) NULL,
-  `AgentName` VARCHAR(345) NULL,
+  `AnnotationID` VARCHAR(500) NULL,
+  `AnnotationSchemaVersion` VARCHAR(500) NULL,
+  `AnnotationDescription` VARCHAR(1500) NULL,
+  `AnnotationURL` VARCHAR(1000) NULL,
+  `AgentName` VARCHAR(500) NULL,
   `AgentType` VARCHAR(45) NULL,
-  `EnvironmentDescription` VARCHAR(505) NULL,
+  `EnvironmentDescription` VARCHAR(1000) NULL,
   `DateOfAction` DATETIME NULL,
-  `Location` VARCHAR(200) NULL,
+  `Location` VARCHAR(500) NULL,
   `Cell_idCell` INT NOT NULL,
   PRIMARY KEY (`idAnnotation`),
   INDEX `fk_Annotation_Cell1_idx` (`Cell_idCell` ASC),
   CONSTRAINT `fk_Annotation_Cell1`
     FOREIGN KEY (`Cell_idCell`)
-    REFERENCES `table_db_drugs`.`Cell` (`idCell`)
+    REFERENCES `Cell` (`idCell`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`Affiliation`
+-- Table `Affiliation`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`Affiliation` ;
+DROP TABLE IF EXISTS `Affiliation` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`Affiliation` (
+CREATE TABLE IF NOT EXISTS `Affiliation` (
   `idAffiliation` INT NOT NULL AUTO_INCREMENT,
   `AffiliationName` VARCHAR(345) NULL,
   `Author_idAuthor` INT NOT NULL,
   PRIMARY KEY (`idAffiliation`),
   INDEX `fk_Affiliation_Author1_idx` (`Author_idAuthor` ASC),
   CONSTRAINT `fk_Affiliation_Author1`
     FOREIGN KEY (`Author_idAuthor`)
-    REFERENCES `table_db_drugs`.`Author` (`idAuthor`)
+    REFERENCES `Author` (`idAuthor`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`Email`
+-- Table `Email`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`Email` ;
+DROP TABLE IF EXISTS `Email` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`Email` (
+CREATE TABLE IF NOT EXISTS `Email` (
   `idEmail` INT NOT NULL AUTO_INCREMENT,
-  `Email` VARCHAR(245) NULL,
+  `Email` VARCHAR(1000) NULL,
   `Author_idAuthor` INT NOT NULL,
   PRIMARY KEY (`idEmail`, `Author_idAuthor`),
   INDEX `fk_Email_Author1_idx` (`Author_idAuthor` ASC),
   CONSTRAINT `fk_Email_Author1`
     FOREIGN KEY (`Author_idAuthor`)
-    REFERENCES `table_db_drugs`.`Author` (`idAuthor`)
+    REFERENCES `Author` (`idAuthor`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
 
 
 -- -----------------------------------------------------
--- Table `table_db_drugs`.`OriginalArticle`
+-- Table `OriginalArticle`
 -- -----------------------------------------------------
-DROP TABLE IF EXISTS `table_db_drugs`.`OriginalArticle` ;
+DROP TABLE IF EXISTS `OriginalArticle` ;
 
-CREATE TABLE IF NOT EXISTS `table_db_drugs`.`OriginalArticle` (
+CREATE TABLE IF NOT EXISTS `OriginalArticle` (
   `idOriginalArticle` INT NOT NULL AUTO_INCREMENT,
   `PMCID` VARCHAR(50) NULL,
   `PMID` VARCHAR(45) NULL,
@@ -232,7 +226,7 @@ CREATE TABLE IF NOT EXISTS `table_db_drugs`.`OriginalArticle` (
   INDEX `fk_OriginalArticle_Article1_idx` (`Article_idArticle` ASC),
   CONSTRAINT `fk_OriginalArticle_Article1`
     FOREIGN KEY (`Article_idArticle`)
-    REFERENCES `table_db_drugs`.`Article` (`idArticle`)
+    REFERENCES `Article` (`idArticle`)
     ON DELETE NO ACTION
     ON UPDATE NO ACTION)
 ENGINE = InnoDB;
@@ -243,14 +237,13 @@ SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;
 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;
 
 -- -----------------------------------------------------
--- Data for table `table_db_drugs`.`CellRole`
+-- Data for table `CellRole`
 -- -----------------------------------------------------
 START TRANSACTION;
-USE `table_db_drugs`;
-INSERT INTO `table_db_drugs`.`CellRole` (`idCellRole`, `CellRoleName`) VALUES (1, 'Header');
-INSERT INTO `table_db_drugs`.`CellRole` (`idCellRole`, `CellRoleName`) VALUES (2, 'Stub');
-INSERT INTO `table_db_drugs`.`CellRole` (`idCellRole`, `CellRoleName`) VALUES (3, 'Data');
-INSERT INTO `table_db_drugs`.`CellRole` (`idCellRole`, `CellRoleName`) VALUES (4, 'SuperRow');
+INSERT INTO `CellRole` (`idCellRole`, `CellRoleName`) VALUES (1, 'Header');
+INSERT INTO `CellRole` (`idCellRole`, `CellRoleName`) VALUES (2, 'Stub');
+INSERT INTO `CellRole` (`idCellRole`, `CellRoleName`) VALUES (3, 'Data');
+INSERT INTO `CellRole` (`idCellRole`, `CellRoleName`) VALUES (4, 'SuperRow');
 
 COMMIT;
 

analysis-UPCI/README

@@ -0,0 +1,43 @@
+import mysql.connector
+import csv
+import re
+
+# Get database connection settings
+settings = []
+with open("settings.cfg", "r+") as config_file:
+	hold = config_file.readlines()
+	for h in hold:
+		splits = h.split(";")
+		line = []
+		line.append(splits[0])
+		line.append(splits[1])
+		settings.append(line)
+
+for line in settings:
+	if "database_username" in line[0]:
+		usr = line[1]
+	if "database_password" in line[0]:
+		pword = line[1]
+	if "database_host" in line[0]:
+		hst = line[1]
+	if "database_name" in line[0]:
+		dbname = line[1]
+	if "database_port" in line[0]:
+		dbport = line[1]
+
+cnx = mysql.connector.connect(user=usr, password=pword, host=hst, database=dbname)
+
+cursor = cnx.cursor()
+
+setIDs_already_annotated = []
+
+check_query = (
+	"SELECT DISTINCT SpecId FROM Article; ")
+
+cursor.execute(check_query)
+
+
+with open('setIDs-already-annotated.txt', 'wb') as outfile:
+	for s in cursor:
+		outfile.write("%s\n" % s)
+