ERD is auto generated now from table

Big-Life-Lab · Mar 2, 2021 · d63e5b4 · d63e5b4
1 parent 84c1287
commit d63e5b4
Show file tree

Hide file tree

Showing 9 changed files with 2,523 additions and 465 deletions.
diff --git a/Variables.csv b/Variables.csv
diff --git a/Variables.csv.txt b/Variables.csv.txt
diff --git a/img/ERD.svg b/img/ERD.svg
diff --git a/metadata_fr.md b/metadata_fr.md
diff --git a/src/generate_db_generations_sql.R b/src/generate_db_generations_sql.R
@@ -12,6 +12,13 @@ library(RSQLite)
 library(glue)
 library(magrittr)
 library(stringr)
+library(stringr)
+library(DiagrammeR)
+library(DiagrammeRsvg)
+library(datamodelr)
+library(htmltools)
+
+
 
 #########################
 # default location for the DB creation file
@@ -212,6 +219,27 @@ wbe_sql_dataType_2_sql_type <- function(var_type){
                "integer" = "integer")
 }
 
+
+
+
+###########################################
+#' based on the variable type given in the table, this maps to a type in SQL for the ERD
+#'
+#'
+#'
+wbe_sql_dataType_2_ERD_type <- function(var_type){
+    var_type %>%
+        switch("string" = as.character(),
+               "boolean" = as.logical(),
+               "category" = as.factor(NA),
+               "float" = as.double(),
+               "date" = as.Date(NA),
+               "datetime" = as.POSIXct(NA),
+               "blob" = as.list(NA),
+               "integer" = as.integer())
+}
+
+
 ############################################
 #'
 #' generates SQL string that will create a database.
@@ -372,6 +400,109 @@ wbe_sql_write_db_creation <- function(lang = "en",
 
 
 
+
+#############################################
+#'
+#'
+#'
+#'
+wbe_generate_erd <- function(
+    lang = "fr",
+    encoding = "UTF-8",
+    tbls = read_csv(file.path(curr_wd, "Tables.csv")),
+    variables = read_csv(file.path(curr_wd, "Variables.csv")),
+    tableLabel_nm = paste0("tableLabel_", lang),
+    erd_fn = file.path("IMG", "ERD.svg")
+){
+
+    #Create blank tables
+    blank_tbls <-
+        tbls %>%
+        pull(tableName) %>%
+        unique() %>%
+        lapply(
+            function(curr_tbl){
+                cols <-
+                    variables %>%
+                    filter(tableName == curr_tbl) %>%
+                    pull(variableName) %>%
+                    unique()
+
+                new_tbl <- tibble()
+                lapply(cols, function(curr_col){
+
+                    colType <-
+                        variables %>%
+                        filter(tableName == curr_tbl) %>%
+                        filter(variableName == curr_col) %>%
+                        pull(variableType) %>%
+                        wbe_sql_dataType_2_ERD_type()
+                    new_tbl[curr_col] <<- colType
+                })
+                new_tbl
+            })
+
+    #put names on the tables
+    names(blank_tbls) <- tbls %>%
+        #mutate(tmp = paste0(tableName, " (" , !!sym(tableLabel_nm), ")")) %>%
+        mutate(tmp = tableName) %>%
+        pull(tmp) %>%
+        unique()
+
+
+
+    #create the data model
+    dm_f <- dm_from_data_frames(blank_tbls)
+
+
+
+    fk <-
+        variables %>%
+        filter(key == "Foreign key")
+
+    pk <-
+        variables %>%
+        filter(key == "Primary Key")
+
+    #make primary keys
+    walk(1:nrow(pk), function(i_row){
+        curr_pk <- pk %>% slice(i_row) #%>%
+        dm_f <<- dm_set_key(dm_f, curr_pk$tableName    , curr_pk$variableName)
+    })
+
+
+
+    #make links
+    walk(1:nrow(fk), function(i_row){
+
+        curr_fk <- fk %>% slice(i_row) #%>%
+        #mutate(tableName = stringr::str_to_title(tableName)) %>%
+        #mutate(foreignKeyTable = stringr::str_to_title(foreignKeyTable))
+        dm_f <<- dm_add_reference_(
+            dm_f,
+            curr_fk$tableName,
+            curr_fk$variableName,
+            curr_fk$foreignKeyTable,
+            curr_fk$foreignKeyVariable
+            #blank_tbls[[curr_fk$tableName]][[curr_fk$variableName]] == blank_tbls[[curr_fk$foreignKeyTable]][[curr_fk$foreignKeyVariable]]
+        )
+    })
+
+    graph <- dm_create_graph(dm_f, rankdir = "LR", col_attr = c("column", "type"), view_type = "all")
+    g <- dm_render_graph(graph)
+    g
+    svg_str <- export_svg(g)
+
+
+
+    fileConn<-file(erd_fn,  encoding = encoding)
+    writeLines(text = c(svg_str), con = fileConn)
+    close(fileConn)
+    svg_str
+}
+
+
+
 ########################################
 #'
 #' default file name for the WBE db
@@ -453,7 +584,7 @@ wbe_db_setup <- function(lang = "en"){
 langs <- c("en", "fr")
 lapply(langs, wbe_db_setup)
 lapply(langs, wbe_metadata_write)
-
+wbe_generate_erd()
 
 
 
diff --git a/src/junk.r b/src/junk.r
@@ -2,77 +2,103 @@
 
 
 
-
-
-
-
-mappers <- wbe_read_mappers()
-
-
-
-
-
-
-wbe_read_data <- function(..., nm, .mappers = "mappers"){
-    #' master function that will call a given mappers reader function
-    get(.mappers)[[nm]]$reader(...)
+library(stringr)
+library(DiagrammeR)
+library(DiagrammeRsvg)
+library(datamodelr)
+library(htmltools)
+
+wbe_generate_erd <- function(
+                            lang = "fr",
+                            encoding = "UTF-8",
+                            tbls = read_csv(file.path(curr_wd, "Tables.csv")),
+                            variables = read_csv(file.path(curr_wd, "Variables.csv")),
+                            tableLabel_nm = paste0("tableLabel_", lang),
+                            erd_fn = file.path("IMG", "ERD.svg")
+                            ){
+
+    #Create blank tables
+    blank_tbls <-
+    tbls %>%
+        pull(tableName) %>%
+        unique() %>%
+        lapply(
+            function(curr_tbl){
+                cols <-
+                    variables %>%
+                    filter(tableName == curr_tbl) %>%
+                    pull(variableName) %>%
+                    unique()
+
+                new_tbl <- tibble()
+                lapply(cols, function(curr_col){
+
+                    colType <-
+                        variables %>%
+                        filter(tableName == curr_tbl) %>%
+                        filter(variableName == curr_col) %>%
+                        pull(variableType) %>%
+                        wbe_sql_dataType_2_ERD_type()
+                    new_tbl[curr_col] <<- colType
+                })
+                new_tbl
+            })
+
+    #put names on the tables
+    names(blank_tbls) <- tbls %>%
+        #mutate(tmp = paste0(tableName, " (" , !!sym(tableLabel_nm), ")")) %>%
+        mutate(tmp = tableName) %>%
+        pull(tmp) %>%
+        unique()
+
+
+
+    #create the data model
+    dm_f <- dm_from_data_frames(blank_tbls)
+
+
+
+    fk <-
+        variables %>%
+        filter(key == "Foreign key")
+
+    pk <-
+        variables %>%
+        filter(key == "Primary Key")
+
+    #make primary keys
+    walk(1:nrow(pk), function(i_row){
+        curr_pk <- pk %>% slice(i_row) #%>%
+        dm_f <<- dm_set_key(dm_f, curr_pk$tableName    , curr_pk$variableName)
+    })
+
+
+
+    #make links
+    walk(1:nrow(fk), function(i_row){
+
+        curr_fk <- fk %>% slice(i_row) #%>%
+            #mutate(tableName = stringr::str_to_title(tableName)) %>%
+            #mutate(foreignKeyTable = stringr::str_to_title(foreignKeyTable))
+        dm_f <<- dm_add_reference_(
+            dm_f,
+            curr_fk$tableName,
+            curr_fk$variableName,
+            curr_fk$foreignKeyTable,
+            curr_fk$foreignKeyVariable
+            #blank_tbls[[curr_fk$tableName]][[curr_fk$variableName]] == blank_tbls[[curr_fk$foreignKeyTable]][[curr_fk$foreignKeyVariable]]
+        )
+    })
+
+    graph <- dm_create_graph(dm_f, rankdir = "LR", col_attr = c("column", "type"), view_type = "all")
+    g <- dm_render_graph(graph)
+    g
+    svg_str <- export_svg(g)
+
+
+
+    fileConn<-file(erd_fn,  encoding = encoding)
+    writeLines(text = c(svg_str), con = fileConn)
+    close(fileConn)
+    svg_str
 }
-
-
-
-wbe_map_data <- function(..., nm, .mappers = "mappers"){
-    #' master function that will call a given mappers mapper function
-    get(.mappers)[[nm]]$mapper(...)
-}
-
-mappers <- wbe_read_mappers(){
-}
-
-
-
-
-mappers$UW$mapdir
-
-
-# read all config files
-mappers <-
-    file.path("data") %>%
-    list.files(pattern = "mapper.r", full.names = TRUE, recursive = TRUE) %>%
-        map(function(x) {
-            e <- parse(x)[[1]]
-            eval(e)
-        }) %>%
-        set_names(map(., ~.$name))
-
-
-
-dfs <- wbe_xlsx_to_dfs(full_fn = file.path("data", "NML", "NML_TEMPLATE_v0.1.2.xlsx"))
-dfs2 <- wbe_xlsx_to_dfs(full_fn = file.path("data", "Ottawa", "Ottawa_TEMPLATE_v0.1.2.xlsx"))
-dfs3 <- wbe_xlsx_to_dfs(full_fn = file.path("data", "UW", "u_w_wwtp_data_V2.xlsx"))
-
-
-
-
-dfs2 <- wbe_files_to_dfs(dir = file.path("data", "db"))
-
-
-wbe_append_lookups(df = dfs$Lookups)
-dfs$Lookups <- NULL
-wbe_append_from_dfs(dfs = dfs)
-
-
-to_add <- wbe_sample_measurments_wide_2_long(df = dfs$SmplMsr)
-
-wbe_append_from_dfs(dfs = to_add)
-
-
-
-to_add$Measurement %>% count(measurementID,sort = T)
-to_add$Sample %>% count(sampleID,sort = T)
-
-wbe_tbl("sample")
-
-
-wbe_tbl("Measurement") %>% count(measureCat,  measureUnit)
-
-wbe_create_db_from_script(db_fn = db_fn )
diff --git a/src/wbe_create_table_en.sql b/src/wbe_create_table_en.sql
@@ -53,11 +53,11 @@ CREATE TABLE IF NOT EXISTS [WWMeasure] (
 	[accessToOtherProv] integer, --If this is 'no', this data will not be available to other data providers not listed before. If missing, data will be available to other data providers not listed before
 	[accessToDetails] integer, --More details on the existing confidentiality requirements of this measurement.
 	[notes] char, --Any additional notes.
-	FOREIGN KEY ([sampleID]) REFERENCES NA(NA) DEFERRABLE INITIALLY DEFERRED,
+	FOREIGN KEY ([sampleID]) REFERENCES Sample(sampleID) DEFERRABLE INITIALLY DEFERRED,
 	FOREIGN KEY ([labID]) REFERENCES Lab(labID) DEFERRABLE INITIALLY DEFERRED,
 	FOREIGN KEY ([assayID]) REFERENCES AssayMethod(assayID) DEFERRABLE INITIALLY DEFERRED,
 	FOREIGN KEY ([instrumentID]) REFERENCES Instrument(instrumentID) DEFERRABLE INITIALLY DEFERRED,
-	FOREIGN KEY ([reporterID]) REFERENCES reporter(reporterID) DEFERRABLE INITIALLY DEFERRED
+	FOREIGN KEY ([reporterID]) REFERENCES Reporter(reporterID) DEFERRABLE INITIALLY DEFERRED
 );
 
 CREATE TABLE IF NOT EXISTS [Site] (
@@ -79,7 +79,7 @@ CREATE TABLE IF NOT EXISTS [Site] (
 	[polygonID] char, --Links with the Polygon table, this should encompass the area that typically drains into this site.
 	[sewerNetworkFileLink] char, --Link to a file that has any detailed information about the sewer network associated with the site (any format).
 	[sewerNetworkFileBLOB] integer, --A file blob that has any detailed information about the sewer network associated with the site (any format).
-	FOREIGN KEY ([polygonID]) REFERENCES NA(NA) DEFERRABLE INITIALLY DEFERRED
+	FOREIGN KEY ([polygonID]) REFERENCES Polygon(polygonID) DEFERRABLE INITIALLY DEFERRED
 );
 
 CREATE TABLE IF NOT EXISTS [SiteMeasure] (
@@ -110,7 +110,7 @@ CREATE TABLE IF NOT EXISTS [SiteMeasure] (
 	[notes] char, --Any additional notes.
 	FOREIGN KEY ([siteID]) REFERENCES Site(siteID) DEFERRABLE INITIALLY DEFERRED,
 	FOREIGN KEY ([instrumentID]) REFERENCES Instrument(instrumentID) DEFERRABLE INITIALLY DEFERRED,
-	FOREIGN KEY ([reporterID]) REFERENCES reporter(reporterID) DEFERRABLE INITIALLY DEFERRED
+	FOREIGN KEY ([reporterID]) REFERENCES Reporter(reporterID) DEFERRABLE INITIALLY DEFERRED
 );
 
 CREATE TABLE IF NOT EXISTS [Reporter] (