Big composite and mb2

src/hg/bigCompositeUpdate/README.md

@@ -0,0 +1,81 @@
+# About bigComposite
+
+*(UPDATE 2025-09-14 at bottom)*
+
+For whomever is testing.
+
+- `bigComposite` should be in the `src/hg/makefile` among targets, and in my
+  mirror it gets installed to `apache/cgi-bin-${USER}` as expected.
+
+- For the bigComposite controls page to work, the `src/hg/js/bigComposite.js`
+  needs to be in the `apache/htdocs/js` dir, and the
+  `src/hg/htdocs/style/bigComposite.css` needs to be in
+  `apache/htdocs/style`. I don't know how to modify the build process for that
+  to be automatic.
+
+- Testing data for the trackDb entries can be found at:
+  ```console
+  https://smithlabresearch.org/data/mb2_trackDb.txt
+  ```
+
+  This is for hg38. It can be entered into a `hg38.trackDb` table like this:
+  ```console
+  mysql hg38 --local-infile=1 -e "
+      LOAD DATA LOCAL INFILE 'mb2_trackDb.txt'
+      REPLACE
+      INTO TABLE trackDb
+      FIELDS TERMINATED BY '\t'
+      LINES TERMINATED BY '\n';
+  "
+  ```
+
+- This file has one entry (the first) named `mb2`, which is the parent and has
+  type bigComposite. Then it has many entries named like
+  `mb2_dataElement_dataType`, where the `dataElement` is an SRA experiment
+  accession (for a WGBS methylome), and `dataType` is in
+  `{hmr,levels,reads}`. The bigBed/bigWig files are hosted on
+  smithlab.usc.edu.
+
+- The corresponding metadata can be found at:
+  ```console
+  https://smithlabresearch.org/data/mb2_metadata.json
+  ```
+
+- The metadata file includes some "settings" near the top, and a sub-dict
+  named `metadata` which has entries for all the dataElements (SRA
+  accessions). Note that not every dataElement from the `mb2_trackDb.txt` file
+  has an entry in the metadata: some were poorly annotated.
+
+- The files are large. For testing purposes you can use the script in this dir
+  that downsamples these to at most a specified number of dataElements.
+
+- In order to test the checkbox sets, and the paging of the data table, make
+  sure to get enough data to populate those.
+
+- The metadata in principle can be hosted anywhere, but CORS restrictions
+  currently block all but the local server. So putting the `mb2_metadata.json`
+  in the local `apache/htdocs` dir, and assigning
+  ```console
+  metaDataUrl /mb2_metadata.json\
+  ```
+  in the "settings" field of the trackDb entry for the mb2 bigComposite track
+  works.
+
+# Update (2025-09-14)
+ - `bigComposite.js` has been updated to use TSV for the metadata table.
+ - All config info is in the `trackDb.settings` field and parsed within `hgTrackUi.c`
+ - Correspondingly, `hgTrackUi.c` has been updated.
+ - Colors can be used or completely ignored. If used, they need a separate
+   JSON file, and checkbox filters for multiple columns can have associated
+   colors.
+ - The updated data files (corresponding to those above and the new ones):
+   ```console
+   https://smithlabresearch.org/data/mb2_trackDb_updated.txt
+   https://smithlabresearch.org/data/mb2_metadata.tsv
+   https://smithlabresearch.org/data/mb2_color.json
+   ```
+   The `_trackDb_updated.txt` is used to fill the trackDb entries, and only
+   differs by specific lines in the `settings` field. The other two files go
+   in `htdocs` or can be put anywhere if the `metaDataUrl` and
+   `colorSettingsUrl` are changed in the first logical line of
+   `mb2_trackDb_updated.txt`.

src/hg/bigCompositeUpdate/bigCompositeUpdate.c

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: MIT; (c) 2025 Andrew D Smith (author)
+/* bigCompositeUpdate - CGI to update bigComposite track visibilities */
+#include "common.h"
+#include "hash.h"
+#include "jksql.h"
+#include "htmshell.h"
+#include "web.h"
+#include "cheapcgi.h"
+#include "cart.h"
+#include "hui.h"
+
+/* Global Variables */
+struct cart *cart;             /* CGI and other variables */
+struct hash *oldVars = NULL;
+
+void doMiddle(struct cart *theCart) {
+  const int buffer_size = 128;
+  cart = theCart;
+
+  char *mdid = cgiString("mdid");  // metadata id (e.g., mb2 for MethBase2)
+  char mdid_dt[buffer_size];
+  snprintf(mdid_dt, buffer_size, "%s_%s", mdid, "dt");
+  char mdid_de[buffer_size];
+  snprintf(mdid_de, buffer_size, "%s_%s", mdid, "de");
+  char wc_mdid[buffer_size];
+  snprintf(wc_mdid, buffer_size, "*_%s", "mb2");
+
+  // clear out previous variables to avoid having to use =0 for
+  // non-visible data-type / data set combinations
+  cartRemoveLike(cart, wc_mdid);
+  cartRemovePrefix(cart, mdid);
+
+  struct dyString *cgi_encoded = cgiUrlString();
+  struct slName *dt_list = NULL;
+  struct slName *de_list = NULL;
+  char *raw_cgi = dyStringContents(cgi_encoded);
+  char *var = NULL, *val = NULL;
+  struct dyString *updated_cgi = newDyString(1024);
+  while (cgiParseNext(&raw_cgi, &var, &val)) {
+    if (sameWord(var, mdid_dt))
+      slAddHead(&dt_list, newSlName(val));
+    else if (sameWord(var, mdid_de))
+      slAddHead(&de_list, newSlName(val));
+    else cgiEncodeIntoDy(var, val, updated_cgi);
+  }
+
+  char track_name[buffer_size];
+  struct slName *de_itr = NULL;
+  for (de_itr = de_list; de_itr; de_itr = de_itr->next) {
+    struct slName *dt_itr = NULL;
+    for (dt_itr = dt_list; dt_itr; dt_itr = dt_itr->next) {
+      snprintf(track_name, buffer_size,
+	       "%s_%s_%s_sel", mdid, de_itr->name, dt_itr->name);
+      cgiEncodeIntoDy(track_name, "1", updated_cgi);
+    }
+  }
+  cartParseOverHash(cart, dyStringContents(updated_cgi));
+  cartRemove(cart, "mdid");  // could clash someday
+  cartCheckout(&cart);
+
+  dyStringFree(&cgi_encoded);
+  dyStringFree(&updated_cgi);
+
+  // send response: success, and no content
+  printf("Status: 204 No Content\r\n");
+  printf("\r\n");
+  fflush(stdout);
+}
+
+/* Null terminated list of CGI Variables we don't want to save
+ * permanently. */
+char *excludeVars[] = {"Submit", "submit", NULL,};
+
+int main(int argc, char *argv[])
+/* Process command line. */
+{
+  cgiSpoof(&argc, argv);
+  cartEmptyShell(doMiddle, hUserCookie(), excludeVars, oldVars);
+  return 0;
+}

src/hg/bigCompositeUpdate/makefile

@@ -0,0 +1,18 @@
+kentSrc = ../..
+include ../../inc/common.mk
+
+L += $(MYSQLLIBS) -lm
+MYLIBDIR = ../../lib/${MACHTYPE}
+MYLIBS =  $(MYLIBDIR)/jkhgap.a ${MYLIBDIR}/jkweb.a
+
+A = bigCompositeUpdate
+O = bigCompositeUpdate.o
+
+A = bigCompositeUpdate
+
+include ../../inc/cgi_build_rules.mk
+
+compile:: $O
+	${CC} $O ${MYLIBS} ${L}
+	mv ${AOUT} $A${EXE}
+