From 13251725228a2508637d6b8a4177da7d9582c72d Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 09:45:42 -0700
Subject: [PATCH 01/12] remove Shiny version of plotEmbedding

fix mainEmbeds and matrixEmbeds to use the original version of plotEmbedding. Remove all shiny-centric updates to plotEmbedding
---
 R/ShinyArchRExports.R | 207 +++++++++++-------------------------------
 R/VisualizeData.R     | 180 +++++++++++++++---------------------
 2 files changed, 126 insertions(+), 261 deletions(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index d805b945..446bd501 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -141,12 +141,6 @@ exportShinyArchR <- function(
   # Add metadata to ArchRProjShiny
   ArchRProjShiny@projectMetadata[["groupBy"]] <- groupBy
   ArchRProjShiny@projectMetadata[["tileSize"]] <- tileSize
-  units <- tryCatch({
-    .h5read(getArrowFiles(ArchRProj)[1], paste0(colorBy, "/Info/Units"))[1]
-  },error=function(e){
-    "values"
-  })
-  ArchRProjShiny@projectMetadata[["units"]] <- units
 
   #copy the RDS corresponding to the ArchRProject to a new directory for use in the Shiny app
   file.copy(file.path(getOutputDirectory(ArchRProjShiny), savedArchRProjFile), file.path(mainOutputDir), recursive=FALSE)
@@ -183,93 +177,36 @@ exportShinyArchR <- function(
     return(NULL)
   }, threads = threads)
   
-  matrices <- list()
-  imputeMatrices <- list()
-  imputeWeights <- getImputeWeights(ArchRProj = ArchRProjShiny)
-  df <- getEmbedding(ArchRProjShiny, embedding = embedding, returnDF = TRUE)
-  
-  if(!file.exists(file.path(subOutputDir, "matrices.rds")) && !file.exists(file.path(subOutputDir, "imputeMatrices.rds"))){
-    for(matName in allMatrices){
-      if(matName %in% supportedMatrices){
-        
-        featuresNames <- getFeatures(ArchRProj = ArchRProj, useMatrix = matName)
-        dir.create(file.path(subOutputDir, matName), showWarnings = FALSE)
-        saveRDS(featuresNames, file.path(subOutputDir, matName, paste0(matName, "_names.rds")))
-        
-        if(!is.null(featuresNames)){
-          
-          mat = Matrix(.getMatrixValues(
-            ArchRProj = ArchRProjShiny, 
-            name = featuresNames,
-            matrixName = matName,
-            log2Norm = FALSE,
-            threads = threads), sparse = TRUE)
-          
-          matrices[[matName]] = mat
-          matList = mat[,rownames(df), drop=FALSE]
-          .logThis(matList, paste0(matName,"-Before-Impute"), logFile = logFile)
-          
-          if(getArchRVerbose()) message("Imputing Matrix")
-
-          imputeMat <- imputeMatrix(mat = as.matrix(matList), imputeWeights = imputeWeights, logFile = logFile)
-          
-          if(!inherits(imputeMat, "matrix")){
-            imputeMat <- matrix(imputeMat, ncol = nrow(df))
-            colnames(imputeMat) <- rownames(df)
-          }
-          imputeMatrices[[matName]] <- imputeMat
-          
-        }else{
-          stop(matName, " is NULL.")
-        }
-      }
-    }
-    
-    matrices$allColorBy= .availableArrays(head(getArrowFiles(ArchRProj), 2))
-    saveRDS(matrices, file.path(subOutputDir, "matrices.rds"))
-    saveRDS(imputeMatrices, file.path(subOutputDir, "imputeMatrices.rds"))
-  }else{
-    
-    message("Matrices and imputed matrices already exist. Reading from local files...")
-    
-    matrices <- readRDS(file.path(subOutputDir, "matrices.rds"))
-    imputeMatrices <- readRDS(file.path(subOutputDir, "imputeMatrices.rds"))
-  }
-  
+  #Create embedding plots for columns in cellColData
   message("Generating raster embedding images for cellColData entries...")
   
   # mainEmbeds will create an HDF5 file containing the nativeRaster vectors for data stored in cellColData
   if (!file.exists(file.path(subOutputDir, "mainEmbeds.h5"))) {
-    .mainEmbeds(ArchRProj = ArchRProjShiny,
-                outDirEmbed = file.path(subOutputDir),
-                colorBy = "cellColData",
-                cellColEmbeddings = cellColEmbeddings,
-                embeddingDF = df,
-                matrices =  matrices,
-                imputeMatrices = imputeMatrices,
-                logFile = createLogFile("mainEmbeds")
-    )          
+    .mainEmbeds(
+      ArchRProj = ArchRProjShiny,
+      outDirEmbed = file.path(subOutputDir),
+      colorBy = "cellColData",
+      cellColEmbeddings = cellColEmbeddings,
+      embedding = embedding,
+      logFile = logFile
+    )
   } else{
     message("H5 for main embeddings already exists...")
   }
   
+  #Create embedding plots for matrices
   message("Generating raster embedding images for matrix data...")
   
   # matrixEmbeds will create an HDF5 file containing he nativeRaster vectors for data stored in matrices
   if(!file.exists(file.path(subOutputDir, "plotBlank72.h5"))){
-    embeddingDF = df
-    
     .matrixEmbeds(
       ArchRProj = ArchRProj,
       outDirEmbed = file.path(subOutputDir),
       colorBy = intersect(supportedMatrices, allMatrices),
       embedding = embedding,
-      embeddingDF = df,
-      matrices = matrices,
-      imputeMatrices = imputeMatrices,
-      threads = getArchRThreads(),
+      threads = threads,
       verbose = TRUE,
-      logFile = createLogFile("matrixEmbeds")
+      logFile = logFile
     )
     
   }else{
@@ -306,9 +243,6 @@ exportShinyArchR <- function(
 #' @param cellColEmbeddings A character vector of columns in `cellColData` to plot as part of the Shiny app. No default is provided so this must be set.
 #' For ex. `c("Sample","Clusters","TSSEnrichment","nFrags")`.
 #' @param embedding The embedding to use. Default is "UMAP".
-#' @param embeddingDF
-#' @param matrices List of stored matrices to use for plotEmbedding so that it runs faster. 
-#' @param imputeMatrices List of stored imputed matrices to use for plotEmbedding so that it runs faster. 
 #' @param threads The number of threads to use for parallel execution.
 #' @param logFile The path to a file to be used for logging ArchR output.
 #'
@@ -318,9 +252,6 @@ exportShinyArchR <- function(
   colorBy = "cellColData", 
   cellColEmbeddings = NULL,
   embedding = "UMAP",
-  embeddingDF = NULL,
-  matrices = NULL,
-  imputeMatrices = NULL,
   threads = getArchRThreads(),
   logFile = createLogFile("mainEmbeds")
 ){
@@ -329,7 +260,6 @@ exportShinyArchR <- function(
   .validInput(input = colorBy, name = "colorBy", valid = c("character"))
   .validInput(input = cellColEmbeddings, name = "cellColEmbeddings", valid = c("character"))
   .validInput(input = embedding, name = "embedding", valid = c("character"))
-  .validInput(input = embeddingDF, name = "embeddingDF", valid = c("data.frame"))
   .validInput(input = threads, name = "threads", valid = c("numeric"))
   .validInput(input = logFile, name = "logFile", valid = c("character"))
   
@@ -339,26 +269,26 @@ exportShinyArchR <- function(
   if(!file.exists(file.path(outDirEmbed, "embeds.rds"))){  
     
     embeds <- .safelapply(1:length(cellColEmbeddings), function(x){ # 
-      name <- cellColEmbeddings[x]
+      
       tryCatch({
         named_embed <- plotEmbedding(
           ArchRProj = ArchRProj,
           baseSize = 12,
           colorBy = colorBy,
-          name = name,
+          name = cellColEmbeddings[x],
           embedding = embedding,
-          embeddingDF = embeddingDF,
           rastr = FALSE,
           size = 0.5,
-          matrices = matrices,
-          imputeMatrices = imputeMatrices,
-          Shiny = TRUE
-        )+ggtitle(paste0("Colored by ", name))+theme(text = element_text(size=12), 
-                                                     legend.title = element_text(size = 12),legend.text = element_text(size = 6))
+        ) + ggtitle(paste0("Colored by ", cellColEmbeddings[x])) + 
+        theme(
+          text = element_text(size=12),
+          legend.title = element_text(size = 12),
+          legend.text = element_text(size = 6)
+          )
       }, error = function(x){
         print(x)
       })
-      return(named_embed) 
+      return(named_embed)
     }, threads = threads)
     
     names(embeds) <- cellColEmbeddings
@@ -425,8 +355,6 @@ exportShinyArchR <- function(
 #' a data matrix in the corresponding ArrowFiles (i.e. "GeneScoreMatrix", "MotifMatrix", "PeakMatrix"). 
 #' @param supportedMatrices
 #' @param embedding The embedding to use. Default is "UMAP".
-#' @param matrices List of stored matrices to use for plotEmbedding so that it runs faster. 
-#' @param imputeMatrices List of stored imputed matrices to use for plotEmbedding so that it runs faster. 
 #' @param threads The number of threads to use for parallel execution.
 #' @param verbose A boolean value that determines whether standard output should be printed.
 #' @param logFile The path to a file to be used for logging ArchR output.
@@ -468,46 +396,33 @@ exportShinyArchR <- function(
   # save the palette
   embeds_pal_list = list()
   
-  allMatrices <- getAvailableMatrices(ArchRProj)
-  
   for(mat in colorBy){
-    
-    if(file.exists(paste0(outDirEmbed, "/",mat, "/", mat, "_names.rds"))){
       
-      dir.create(paste0(outDirEmbed, "/",mat, "/embeds"), showWarnings = FALSE)
-      
-      featureNames <- readRDS(file.path(outDirEmbed, mat, paste0(mat,"_names.rds")))
+    dir.create(paste0(outDirEmbed, "/",mat, "/embeds"), showWarnings = FALSE)
+    
+    featureNames <- getFeatures(ArchRProj = ArchRProj, useMatrix = mat)
+    featureNames <- featureNames[which(!is.na(featureNames))]
 
-      if(!is.null(featureNames)){
-        
-        embeds_points <- .safelapply(1:length(featureNames), function(x){ #length(featureNames) 
-          
-          print(paste0("Creating plots for ", mat,": ",x,": ",round((x/length(featureNames))*100,3), "%"))
+    message(paste0("Creating plots for ", mat,"..."))
 
-          if(!is.na(featureNames[x])){
-            
-            gene_plot <- plotEmbedding(
-              ArchRProj = ArchRProj,
-              colorBy = mat,
-              name = featureNames[x],
-              embedding = embedding,
-              quantCut = c(0.01, 0.95),
-              imputeWeights = getImputeWeights(ArchRProj = ArchRProj),
-              plotAs = "points",
-              embeddingDF = embeddingDF,
-              matrices = matrices,
-              imputeMatrices = imputeMatrices,
-              rastr = TRUE
-            )
-            
-            
-          }else{
-            gene_plot = NULL
-          }
-          
-          if(!is.null(gene_plot)){
+    if(!is.null(featureNames)){
+        
+        featurePlots <- plotEmbedding(
+          ArchRProj = ArchRProj,
+          colorBy = mat,
+          name = featureNames,
+          quantCut = c(0.01, 0.95),
+          imputeWeights = getImputeWeights(ArchRProj = ArchRProj),
+          plotAs = "points",
+          rastr = TRUE,
+          threads = threads
+        )
+        
+        embeds_points <- .safelapply(seq_along(featurePlots), function(x){
+          featurePlotx <- featurePlots[x][[1]]
+          if(!is.null(featurePlotx)){
             
-            gene_plot_blank <- gene_plot + theme(axis.title.x = element_blank()) +
+            featurePlotx_blank <- featurePlotx + theme(axis.title.x = element_blank()) +
               theme(axis.title.y = element_blank()) +
               theme(axis.title = element_blank()) +
               theme(legend.position = "none") +
@@ -520,22 +435,20 @@ exportShinyArchR <- function(
             
             #save plot without axes etc as a jpg.
             ggsave(filename = file.path(outDirEmbed, mat, "embeds", paste0(featureNames[x],"_blank72.jpg")),
-                   plot = gene_plot_blank, device = "jpg", width = 3, height = 3, units = "in", dpi = 72)
+                   plot = featurePlotx_blank, device = "jpg", width = 3, height = 3, units = "in", dpi = 72)
             
             #read back in that jpg because we need vector in native format
             blank_jpg72 <- jpeg::readJPEG(source = file.path(outDirEmbed, mat, "embeds", paste0(featureNames[x],"_blank72.jpg")),
-                                                        native = TRUE)
+                                          native = TRUE)
             
-            g <- ggplot_build(gene_plot)
+            g <- ggplot_build(featurePlotx)
             
-            res = list(list(plot=as.vector(blank_jpg72), min = round(min(gene_plot$data$color),1),
-                            max = round(max(gene_plot$data$color),1), pal = unique(g$data[[1]][,"colour"])))
+            res = list(list(plot=as.vector(blank_jpg72), min = round(min(featurePlotx$data$color),1),
+                            max = round(max(featurePlotx$data$color),1), pal = unique(g$data[[1]][,"colour"])))
             
             return(res)
           }
-          
-          
-        }, threads = threads) 
+        }, threads = threads)
         
         names(embeds_points) <- featureNames
         embeds_points = embeds_points[!unlist(lapply(embeds_points, is.null))]
@@ -558,27 +471,14 @@ exportShinyArchR <- function(
         
         embeds_min_max_list[[mat]] =  embeds_min_max
         embeds_pal_list[[mat]] = embeds_points[[length(embeds_points)]][[1]]$pal
-        
-        
-# 
-# 
-#       embeds_min_max_list[[mat]] =  embeds_min_max
-#       embeds_pal_list[[mat]] = embeds_points[[length(embeds_points)]][[1]]$pal
-      
+
       }else{
-        
-        message(mat,".rds file does not exist")
-      }
-        
-    }else{
-      message(mat,".rds file does not exist. This file should have been created previously be exportShinyArchR. Skipping...")
-    }
-    
+      
+      stop("Matrix ", mat,"has no features!")
+    }  
     
   }
 
-# nms = names(embeds_pal_list)  
-  
 for(i in 1:length(embeds_pal_list)){
  
   cols = embeds_pal_list[[i]]
@@ -588,7 +488,6 @@ for(i in 1:length(embeds_pal_list)){
   
 }
   
-  
 scale <- embeds_min_max_list
 pal <- embeds_pal_list
 
diff --git a/R/VisualizeData.R b/R/VisualizeData.R
index f586ab33..3a370f60 100644
--- a/R/VisualizeData.R
+++ b/R/VisualizeData.R
@@ -247,15 +247,11 @@ plotEmbedding <- function(
   keepAxis = FALSE,
   baseSize = 10,
   plotAs = NULL,
-  Shiny = FALSE,
-  matrices = NULL,
-  imputeMatrices = NULL,
-  embeddingDF = NULL,
   threads = getArchRThreads(),
   logFile = createLogFile("plotEmbedding"),
   ...
-){
-  
+  ){
+
   .validInput(input = ArchRProj, name = "ArchRProj", valid = c("ArchRProj"))
   .validInput(input = embedding, name = "reducedDims", valid = c("character"))
   .validInput(input = colorBy, name = "colorBy", valid = c("character"))
@@ -274,39 +270,34 @@ plotEmbedding <- function(
   .validInput(input = keepAxis, name = "keepAxis", valid = c("boolean"))
   .validInput(input = baseSize, name = "baseSize", valid = c("numeric"))
   .validInput(input = plotAs, name = "plotAs", valid = c("character", "null"))
-  .validInput(input = Shiny, name = "Shiny", valid = c("boolean"))
   .validInput(input = threads, name = "threads", valid = c("integer"))
   .validInput(input = logFile, name = "logFile", valid = c("character"))
-  
+
   .requirePackage("ggplot2", source = "cran")
-  
+
   .startLogging(logFile = logFile)
   .logThis(mget(names(formals()),sys.frame(sys.nframe())), "Input-Parameters", logFile=logFile)
-  
+
   ##############################
   # Get Embedding
   ##############################
-  .logMessage("Getting Embedding", logFile = logFile)
-  if(Shiny){
-     df <- embeddingDF
-  } else{
-    df <- getEmbedding(ArchRProj, embedding = embedding, returnDF = TRUE)
-  }
+  .logMessage("Getting UMAP Embedding", logFile = logFile)
+  df <- getEmbedding(ArchRProj, embedding = embedding, returnDF = TRUE)
 
   if(!all(rownames(df) %in% ArchRProj$cellNames)){
     stop("Not all cells in embedding are present in ArchRProject!")
   }
+
   .logThis(df, name = "Embedding data.frame", logFile = logFile)
-  
   if(!is.null(sampleCells)){
     if(sampleCells < nrow(df)){
       if(!is.null(imputeWeights)){
-        stop("Cannot sampleCells with imputeWeights not equal to NULL at this time!")
+        stop("Cannot sampleCells with imputeWeights not equalt to NULL at this time!")
       }
       df <- df[sort(sample(seq_len(nrow(df)), sampleCells)), , drop = FALSE]
     }
   }
-  
+
   #Parameters
   plotParams <- list(...)
   plotParams$x <- df[,1]
@@ -320,33 +311,29 @@ plotEmbedding <- function(
   plotParams$rastr <- rastr
   plotParams$size <- size
   plotParams$randomize <- randomize
-  
-  #Check if Cells To Be Highlighted
+
+  #Check if Cells To Be Highlighed
   if(!is.null(highlightCells)){
     highlightPoints <- match(highlightCells, rownames(df), nomatch = 0)
     if(any(highlightPoints==0)){
       stop("highlightCells contain cells not in Embedding cellNames! Please make sure that these match!")
     }
   }
-  
+
   #Make Sure ColorBy is valid!
   if(length(colorBy) > 1){
     stop("colorBy must be of length 1!")
   }
-
-  if(!Shiny){
-    allColorBy <- c("colData", "cellColData", .availableArrays(head(getArrowFiles(ArchRProj), 2)))
-  } else {
-    allColorBy <-  c("colData", "cellColData", matrices$allColorBy)
-  }
-  
+  allColorBy <-  c("colData", "cellColData", .availableArrays(head(getArrowFiles(ArchRProj), 2)))
   if(tolower(colorBy) %ni% tolower(allColorBy)){
     stop("colorBy must be one of the following :\n", paste0(allColorBy, sep=", "))
   }
   colorBy <- allColorBy[match(tolower(colorBy), tolower(allColorBy))]
+
   .logMessage(paste0("ColorBy = ", colorBy), logFile = logFile)
-  
+
   if(tolower(colorBy) == "coldata" | tolower(colorBy) == "cellcoldata"){
+      
     colorList <- lapply(seq_along(name), function(x){
       colorParams <- list()
       colorParams$color <- as.vector(getCellColData(ArchRProj, select = name[x], drop = FALSE)[rownames(df), 1])
@@ -363,7 +350,7 @@ plotEmbedding <- function(
       if(x == 1){
         .logThis(colorParams, name = "ColorParams 1", logFile = logFile)
       }
-      
+
       if(!is.null(imputeWeights)){
         if(getArchRVerbose()) message("Imputing Matrix")
         colorMat <- matrix(colorParams$color, nrow=1)
@@ -372,30 +359,28 @@ plotEmbedding <- function(
         colorParams$color <- as.vector(colorMat)
       }
       colorParams
-    }) 
-  }else{# plotting embedding for matrix instead of col in cellcoldata
+    })
+
+
+  }else{
+
     suppressMessages(message(logFile))
-    
-    if(!Shiny){
-      units <- tryCatch({
-      .h5read(getArrowFiles(ArchRProj)[1], paste0(colorBy, "/Info/Units"))[1]
-    },error=function(e){
-      "values"
+
+    units <- tryCatch({
+        .h5read(getArrowFiles(ArchRProj)[1], paste0(colorBy, "/Info/Units"))[1]
+      },error=function(e){
+        "values"
     })
-    }else{
-        units <- ArchRProj@projectMetadata[["units"]]
-    }
     
     if(is.null(log2Norm) & tolower(colorBy) == "genescorematrix"){
       log2Norm <- TRUE
     }
-    
+
     if(is.null(log2Norm)){
       log2Norm <- FALSE
     }
-    
-    if(!Shiny){
-      colorMat <- .getMatrixValues(
+
+    colorMat <- .getMatrixValues(
       ArchRProj = ArchRProj, 
       name = name, 
       matrixName = colorBy, 
@@ -403,47 +388,27 @@ plotEmbedding <- function(
       threads = threads,
       logFile = logFile
     )
-    }else{ 
-      #get values from pre-saved list
-      colorMat = tryCatch({
-      t(as.matrix(matrices[[colorBy]][name,]))
-      }, warning = function(warning_condition) {
-      message(paste("name doesn't exist:", name))
-      message(warning_condition)
-      return(NULL)    
-      }, error = function(error_condition) {
-      message(paste("name doesn't exist:", name))
-      message(error_condition)
-      return(NA)
-      }, finally={
-      })
-     rownames(colorMat)=name
-    }
-    
+
     if(!all(rownames(df) %in% colnames(colorMat))){
       .logMessage("Not all cells in embedding are present in feature matrix. This may be due to using a custom embedding.", logFile = logFile)
       stop("Not all cells in embedding are present in feature matrix. This may be due to using a custom embedding.")
     }
-    
+
     colorMat <- colorMat[,rownames(df), drop=FALSE]
-    
+
     .logThis(colorMat, "colorMat-Before-Impute", logFile = logFile)
-    
+
     if(!is.null(imputeWeights)){
       if(getArchRVerbose()) message("Imputing Matrix")
-        if(!Shiny){
-          colorMat <- imputeMatrix(mat = as.matrix(colorMat), imputeWeights = imputeWeights, logFile = logFile)
-        }else{
-          colorMat <- imputeMatrices[[colorBy]][name,]
-        }
-        if(!inherits(colorMat, "matrix")){
-          colorMat <- matrix(colorMat, ncol = nrow(df))
-          colnames(colorMat) <- rownames(df)
-        }     
+      colorMat <- imputeMatrix(mat = as.matrix(colorMat), imputeWeights = imputeWeights, logFile = logFile)
+      if(!inherits(colorMat, "matrix")){
+        colorMat <- matrix(colorMat, ncol = nrow(df))
+        colnames(colorMat) <- rownames(df)
+      }
     }
-    
+
     .logThis(colorMat, "colorMat-After-Impute", logFile = logFile)
-    
+
     colorList <- lapply(seq_len(nrow(colorMat)), function(x){
       colorParams <- list()
       colorParams$color <- colorMat[x, ]
@@ -465,38 +430,39 @@ plotEmbedding <- function(
       }
       colorParams
     })
+
   }
-  
+
   if(getArchRVerbose()) message("Plotting Embedding")
-  
+
   ggList <- lapply(seq_along(colorList), function(x){
-    
+
     if(getArchRVerbose()) message(x, " ", appendLF = FALSE)
-    
+
     plotParamsx <- .mergeParams(colorList[[x]], plotParams)
-    
+
     if(plotParamsx$discrete){
       plotParamsx$color <- paste0(plotParamsx$color)
     }
-    
+
     if(!plotParamsx$discrete){
-      
+
       if(!is.null(quantCut)){
         plotParamsx$color <- .quantileCut(plotParamsx$color, min(quantCut), max(quantCut))
       }
-      
+
       plotParamsx$pal <- paletteContinuous(set = plotParamsx$continuousSet)
-      
+
       if(!is.null(pal)){
-        
+
         plotParamsx$pal <- pal
         
       }
-      
+
       if(is.null(plotAs)){
         plotAs <- "hexplot"
       }
-      
+
       if(!is.null(log2Norm)){
         if(log2Norm){
           plotParamsx$color <- log2(plotParamsx$color + 1)
@@ -505,62 +471,62 @@ plotEmbedding <- function(
           plotParamsx$colorTitle <- units
         }
       }
-      
+
       if(tolower(plotAs) == "hex" | tolower(plotAs) == "hexplot"){
-        
+
         plotParamsx$discrete <- NULL
         plotParamsx$continuousSet <- NULL
         plotParamsx$rastr <- NULL
         plotParamsx$size <- NULL
         plotParamsx$randomize <- NULL
-        
+
         .logThis(plotParamsx, name = paste0("PlotParams-", x), logFile = logFile)
         gg <- do.call(ggHex, plotParamsx)
-        
+
       }else{
-        
+
         if(!is.null(highlightCells)){
           plotParamsx$highlightPoints <- highlightPoints
         }
-        
+
         .logThis(plotParamsx, name = paste0("PlotParams-", x), logFile = logFile)
         gg <- do.call(ggPoint, plotParamsx)
-        
+
       }
-      
+
     }else{
       
       if(!is.null(pal)){
         plotParamsx$pal <- pal
       }
-      
+
       if(!is.null(highlightCells)){
         plotParamsx$highlightPoints <- highlightPoints
       }
-      
+
       .logThis(plotParamsx, name = paste0("PlotParams-", x), logFile = logFile)
       gg <- do.call(ggPoint, plotParamsx)
-      
+
     }
-    
+
     if(!keepAxis){
       gg <- gg + theme(axis.text.x=element_blank(), axis.ticks.x=element_blank(), axis.text.y=element_blank(), axis.ticks.y=element_blank())
     }
-    
+
     gg
-    
+
   })
   names(ggList) <- name
   if(getArchRVerbose()) message("")
-  
+
   if(length(ggList) == 1){
     ggList <- ggList[[1]]
   }
-  
+
   .endLogging(logFile = logFile)
-  
+
   ggList
-  
+
 }
 
 #' Visualize Groups from ArchR Project

From e484d360dc048e280e91586c84ceefbefa5c382c Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 09:54:23 -0700
Subject: [PATCH 02/12] bugfix

---
 R/ShinyArchRExports.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 446bd501..fcd82c3b 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -97,7 +97,7 @@ exportShinyArchR <- function(
   
   supportedMatrices <- c("GeneScoreMatrix", "GeneIntegrationMatrix", "MotifMatrix") #only these matrices are currently supported for ShinyArchR
   #subset matrices for use in Shiny app
-  allMatrices <- getAvailableMatrices(ArchRProjShiny)
+  allMatrices <- getAvailableMatrices(ArchRProj)
   if(!is.null(matsToUse)){
     if(!all(matsToUse %in% allMatrices)){
       stop("Not all matrices defined in matsToUse exist in your ArchRProject. See getAvailableMatrices().")

From 89fd4af6cd520a02750c9e98ff624badad1d5680 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:00:32 -0700
Subject: [PATCH 03/12] bugfix

make groups apply to both frags and cov
---
 R/ShinyArchRExports.R | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index fcd82c3b..6ec72892 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -150,14 +150,15 @@ exportShinyArchR <- function(
   fragFiles <- list.files(path = file.path(fragDir, pattern = "\\_frags.rds$"))
   dir.create(file.path(mainOutputDir, "ShinyFragments"), showWarnings = FALSE)
 
+  groups <- unique(ArchRProj@cellColData[,groupBy])
+
   #check for the existence of each expected fragment file and create if not found
-  fragGroups <- unique(ArchRProj@cellColData[,groupBy])
-  fragOut <- .safelapply(seq_along(fragGroups), function(x){
-    fragGroupsx <- fragGroups[x]
-    if(!file.exists(file.path(fragDir,paste0(fragGroupsx,"_frags.rds"))) | force){
+  fragOut <- .safelapply(seq_along(groups), function(x){
+    groupsx <- groups[x]
+    if(!file.exists(file.path(fragDir,paste0(groupsx,"_frags.rds"))) | force){
       .exportGroupFragmentsRDS(ArchRProj = ArchRProjShiny, groupBy = groupBy, outDir = fragDir, threads = threads)
     } else {
-      message(paste0("Fragment file for ", fragGroupsx," already exist. Skipping fragment file generation..."))
+      message(paste0("Fragment file for ", groupsx," already exist. Skipping fragment file generation..."))
     }
     return(NULL)
   }, threads = threads)

From d5353804df9ebe703bf914ec6c4c1ed8bf4f46cf Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:06:56 -0700
Subject: [PATCH 04/12] add threads argument to exportClusterCoverage

---
 R/GroupExport.R       | 8 ++++++--
 R/ShinyArchRExports.R | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/R/GroupExport.R b/R/GroupExport.R
index 30ded3d3..652177ae 100644
--- a/R/GroupExport.R
+++ b/R/GroupExport.R
@@ -576,6 +576,7 @@ getGroupFragments <- function(
 #' user-supplied `cellColData` metadata columns (for example, "Clusters"). Cells with the same value annotated in this metadata
 #' column will be grouped together and their fragments exported to `outputDirectory`/GroupFragments.
 #' @param outDir the directory to output the group fragment files.
+#' @param threads An integer specifying the number of threads for parallel.
 #' 
 #' @examples
 #'
@@ -629,6 +630,7 @@ getGroupFragments <- function(
 #' column will be grouped together and the average signal will be plotted.
 #' @param fragDir The path to the directory containing fragment files.
 #' @param outDir The path to the desired output directory for storage of coverage files.
+#' @param threads An integer specifying the number of threads for parallel.
 #'
 .exportClusterCoverageRDS <- function(
   ArchRProj = NULL,
@@ -636,7 +638,8 @@ getGroupFragments <- function(
   scaleFactor = 1,
   groupBy = "Clusters",
   fragDir = file.path(getOutputDirectory(ArchRProj), "fragments"),
-  outDir = file.path(getOutputDirectory(ArchRProj), "coverage")
+  outDir = file.path(getOutputDirectory(ArchRProj), "coverage"),
+  threads = getArchRThreads()
 ){
   fragFiles = list.files(path = fragDir, pattern = "_frags.rds", full.names = TRUE)
   if(length(fragFiles) < 1){
@@ -687,5 +690,6 @@ getGroupFragments <- function(
 
     binnedCoverage <- coverage(bins, weight = bins$reads * scaleFactor )
     saveRDS(binnedCoverage, file.path(outDir, paste0(groupID, "_cvg.rds")))
-  }, threads = threads)  
+  }, threads = threads)
+  return(NULL)
 }
diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 6ec72892..d9e8a6ef 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -171,7 +171,7 @@ exportShinyArchR <- function(
   covOut <- .safelapply(seq_along(groups), function(x){
     groupsx <- groups[x]
     if(!file.exists(file.path(covDir,paste0(groupsx,"_cvg.rds"))) | force){
-      .exportClusterCoverageRDS(ArchRProj = ArchRProjShiny, tileSize = tileSize, groupBy = groupBy, outDir = covDir, fragDir = fragDir)
+      .exportClusterCoverageRDS(ArchRProj = ArchRProjShiny, tileSize = tileSize, groupBy = groupBy, outDir = covDir, fragDir = fragDir, threads = threads)
     } else {
       message(paste0("Coverage file for ", groupsx," already exist. Skipping coverage file generation..."))
     }

From bbb2c9fe3ed7d0680f55cb7b043e8bb5837618c9 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:12:20 -0700
Subject: [PATCH 05/12] bugfix addSeqLengths is hidden

correct function name
---
 R/GroupExport.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/GroupExport.R b/R/GroupExport.R
index 652177ae..f7714ec7 100644
--- a/R/GroupExport.R
+++ b/R/GroupExport.R
@@ -683,7 +683,7 @@ getGroupFragments <- function(
         minoverlap = 0L,
         type = "any"
       )
-    addSeqLengths(bins, genome)
+    .addSeqLengths(bins, genome)
     
     groupReadsInTSS <-
       ArchRProj@cellColData$ReadsInTSS[cells %in% cellGroups$groupID]

From 415a8a4c24297c39e892fac1464eb1a489feeda8 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:17:24 -0700
Subject: [PATCH 06/12] bugfix old arguments

---
 R/ShinyArchRExports.R | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index d9e8a6ef..4bc9a023 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -354,7 +354,6 @@ exportShinyArchR <- function(
 #' @param outDirEmbed Where the HDF5 and the jpgs will be saved.
 #' @param colorBy A string indicating whether points in the plot should be colored by a column in `cellColData` ("cellColData") or by
 #' a data matrix in the corresponding ArrowFiles (i.e. "GeneScoreMatrix", "MotifMatrix", "PeakMatrix"). 
-#' @param supportedMatrices
 #' @param embedding The embedding to use. Default is "UMAP".
 #' @param threads The number of threads to use for parallel execution.
 #' @param verbose A boolean value that determines whether standard output should be printed.
@@ -363,12 +362,8 @@ exportShinyArchR <- function(
 .matrixEmbeds <- function(
   ArchRProj = NULL,
   outDirEmbed = NULL,
-  colorBy = "cellColData",
-  supportedMatrices = c("GeneScoreMatrix", "GeneIntegrationMatrix", "MotifMatrix"),
+  colorBy = NULL,
   embedding = "UMAP",
-  embeddingDF = NULL,
-  matrices = NULL,
-  imputeMatrices = NULL,
   threads = getArchRThreads(),
   verbose = TRUE,
   logFile = createLogFile("matrixEmbeds")
@@ -376,11 +371,7 @@ exportShinyArchR <- function(
   .validInput(input = ArchRProj, name = "ArchRProj", valid = c("ArchRProj"))
   .validInput(input = outDirEmbed, name = "outDirEmbed", valid = c("character"))
   .validInput(input = colorBy, name = "colorBy", valid = c("character"))
-  .validInput(input = supportedMatrices, name = "supportedMatrices", valid = c("character"))
   .validInput(input = embedding, name = "embedding", valid = c("character"))
-  .validInput(input = embeddingDF, name = "embeddingDF", valid = c("data.frame"))
-  .validInput(input = matrices, name = "matrices", valid = c("list"))
-  .validInput(input = imputeMatrices, name = "imputeMatrices", valid = c("list"))
   .validInput(input = threads, name = "threads", valid = c("numeric"))
   .validInput(input = verbose, name = "verbose", valid = c("boolean"))
   .validInput(input = logFile, name = "logFile", valid = c("character"))

From 0cc745734da634ebb6ee481c9d18c5314e69425b Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:34:45 -0700
Subject: [PATCH 07/12] add back feature name storage

---
 R/ShinyArchRExports.R | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 4bc9a023..61ce649e 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -394,6 +394,8 @@ exportShinyArchR <- function(
     
     featureNames <- getFeatures(ArchRProj = ArchRProj, useMatrix = mat)
     featureNames <- featureNames[which(!is.na(featureNames))]
+    dir.create(file.path(subOutputDir, mat), showWarnings = FALSE)
+    saveRDS(featuresNames, file.path(subOutputDir, mat, paste0(mat, "_names.rds")))
 
     message(paste0("Creating plots for ", mat,"..."))
 

From 2b1d8fcd0ecdbfba1dea7b6748132da6e5a791d9 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:39:39 -0700
Subject: [PATCH 08/12] remove explicit namespace loading from global

is this needed? seems like it should not be needed
---
 Shiny/global.R | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/Shiny/global.R b/Shiny/global.R
index 2b850fae..f6e8bc5e 100644
--- a/Shiny/global.R
+++ b/Shiny/global.R
@@ -27,15 +27,6 @@ library(htmltools)
 
 ############# NEW ADDITIONS (start) ###############################
 
-# Adjusting ArchR functions
-fn <- base::unclass(utils::lsf.str(envir = base::asNamespace("ArchR"), all = TRUE))
-for (i in base::seq_along(fn)) {
-  base::tryCatch({
-    base::eval(base::parse(text = base::paste0(fn[i], "<-ArchR:::", fn[i])))
-  }, error = function(x) {
-  })
-}
-
 # Calling ArchRProj
 ArchRProj <- ArchR::loadArchRProject(path = ".", shiny = TRUE)
 ArchRProj <- ArchR::addImputeWeights(ArchRProj = ArchRProj)

From fc3f673902af324104c1820a9bef0d5162b69ef1 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:43:23 -0700
Subject: [PATCH 09/12] bugfix dir name

---
 R/ShinyArchRExports.R | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 61ce649e..7a81abe1 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -389,13 +389,12 @@ exportShinyArchR <- function(
   embeds_pal_list = list()
   
   for(mat in colorBy){
-      
-    dir.create(paste0(outDirEmbed, "/",mat, "/embeds"), showWarnings = FALSE)
+    dir.create(file.path(outDirEmbed, mat), showWarnings = FALSE)
+    dir.create(file.path(outDirEmbed, mat, "/embeds"), showWarnings = FALSE)
     
     featureNames <- getFeatures(ArchRProj = ArchRProj, useMatrix = mat)
     featureNames <- featureNames[which(!is.na(featureNames))]
-    dir.create(file.path(subOutputDir, mat), showWarnings = FALSE)
-    saveRDS(featuresNames, file.path(subOutputDir, mat, paste0(mat, "_names.rds")))
+    saveRDS(featuresNames, file.path(outDirEmbed, mat, paste0(mat, "_names.rds")))
 
     message(paste0("Creating plots for ", mat,"..."))
 

From 11039ee02eb796f9ed8b6ddbc33fb7b579143e70 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 10:48:50 -0700
Subject: [PATCH 10/12] typo

---
 R/ShinyArchRExports.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 7a81abe1..07e668c6 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -394,7 +394,7 @@ exportShinyArchR <- function(
     
     featureNames <- getFeatures(ArchRProj = ArchRProj, useMatrix = mat)
     featureNames <- featureNames[which(!is.na(featureNames))]
-    saveRDS(featuresNames, file.path(outDirEmbed, mat, paste0(mat, "_names.rds")))
+    saveRDS(featureNames, file.path(outDirEmbed, mat, paste0(mat, "_names.rds")))
 
     message(paste0("Creating plots for ", mat,"..."))
 

From e742abb32738eb6b79266c62612055a2838db8fc Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 11:39:42 -0700
Subject: [PATCH 11/12] add status reporting for feature plots

---
 R/ShinyArchRExports.R | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 07e668c6..253064a2 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -412,6 +412,9 @@ exportShinyArchR <- function(
         )
         
         embeds_points <- .safelapply(seq_along(featurePlots), function(x){
+          if((x %% 100) == 0) {
+            message("Processing feature #",x," of ",length(featurePlots)," for ", mat,".")
+          }
           featurePlotx <- featurePlots[x][[1]]
           if(!is.null(featurePlotx)){
             

From 4b374760b1333d4fbf327ed11ac822d04f0becd7 Mon Sep 17 00:00:00 2001
From: Ryan Corces <ryancorces@gmail.com>
Date: Tue, 28 Mar 2023 11:42:51 -0700
Subject: [PATCH 12/12] pass logFile to plotEmbedding

---
 R/ShinyArchRExports.R | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/R/ShinyArchRExports.R b/R/ShinyArchRExports.R
index 253064a2..a7062dae 100644
--- a/R/ShinyArchRExports.R
+++ b/R/ShinyArchRExports.R
@@ -280,6 +280,8 @@ exportShinyArchR <- function(
           embedding = embedding,
           rastr = FALSE,
           size = 0.5,
+          threads = threads,
+          logFile = logFile
         ) + ggtitle(paste0("Colored by ", cellColEmbeddings[x])) + 
         theme(
           text = element_text(size=12),
@@ -408,7 +410,8 @@ exportShinyArchR <- function(
           imputeWeights = getImputeWeights(ArchRProj = ArchRProj),
           plotAs = "points",
           rastr = TRUE,
-          threads = threads
+          threads = threads,
+          logFile = logFile
         )
         
         embeds_points <- .safelapply(seq_along(featurePlots), function(x){