config-able vector graph sizes

INSTRUCTIONS/CUSTOMIZATION.md

@@ -12,8 +12,9 @@ Items of most interest in the configuration file:
 * `DefCorp` sets which corpora are active on a reset. So if you almost never search Greek, you can set `gr` to `false`, for example. On a slow machine, this would significantly speed up `in every active author` searches.
 * `Font` sets the interface font. `Noto`, `Fira`, and `Roboto` are embedded in the program. If you pick another name, you need to have it installed on your machine. 
 * `QuietStart` spares you the copyright notice.
-* `WorkerCount` sets the number of cores of your CPU to use when searching. You will be sorry if you pick a number that is larger than what the machine actually has installed. `WorkerCount` = `CoreCount` is probably the best choice unless you know why it is not.
+* `VectorChtHt` and `VectorChWd` will set the height and width of the vector charts. Note that `px` is required as are the quotation marks: `"1000px"`, e.g.
 * `VectorMaxlines` sets the maximum scope of a vector search. `1000000` will let you model all of Latin. All of Greek is about 10x larger.
+* `WorkerCount` sets the number of cores of your CPU to use when searching. You will be sorry if you pick a number that is larger than what the machine actually has installed. `WorkerCount` = `CoreCount` is probably the best choice unless you know why it is not.
 * `ZapLunates` lets you see σ/ς instead of lunate sigmas. But why would you want that?
 
 
@@ -37,32 +38,38 @@ Items of most interest in the configuration file:
     "Gzip": false,
     "HostIP": "127.0.0.1",
     "HostPort": 8000,
+    "LdaGraph": false,
+    "LdaTopics": 8,
     "LogLevel": 0,
     "ManualGC": true,
+    "MaxText": 35000,
     "MaxSrchIP": 2,
     "MaxSrchTot": 4,
-    "MaxText": 35000,
     "PGLogin": {
       "Host": "127.0.0.1",
       "Port": 5432,
       "User": "hippa_wr",
-      "Pass": "mypass",
+      "Pass": "myrandompassword",
       "DBName": "hipparchiaDB"
     },
+    "ProfileCPU": false,
+    "ProfileMEM": false,
     "ResetVectors": false,
     "QuietStart": false,
     "SelfTest": 0,
     "TickerActive": false,
     "VectorsDisabled": false,
     "VectorBot": false,
+    "VectorChtHt": "1200px",
+    "VectorChtWd": "1500px",
     "VectorMaxlines": 1000000,
     "VectorModel": "w2v",
-    "VectorNeighb": 18,
+    "VectorNeighb": 16,
     "VectorTextPrep": "winner",
     "VectorWebExt": false,
     "VocabByCt": false,
     "VocabScans": false,
-    "WorkerCount": 6,
+    "WorkerCount": 20,
     "ZapLunates": false
   }
 ```

configatlaunch.go

@@ -46,6 +46,8 @@ type CurrentConfiguration struct {
 	TickerActive    bool
 	VectorsDisabled bool
 	VectorBot       bool
+	VectorChtHt     string
+	VectorChtWd     string
 	VectorMaxlines  int
 	VectorModel     string
 	VectorNeighb    int
@@ -138,7 +140,7 @@ func ConfigAtLaunch() {
 		FAIL1 = "Could not parse your information as a valid collection of credentials. Use the following template:"
 		FAIL2 = `"{\"Pass\": \"YOURPASSWORDHERE\" ,\"Host\": \"127.0.0.1\", \"Port\": 5432, \"DBName\": \"hipparchiaDB\" ,\"User\": \"hippa_wr\"}"`
 		FAIL3 = `Could not parse the information in '%s'. Skipping and attempting to use built-in defaults instead.`
-		FAIL5 = "Improperly formatted corpus list. Using:\n\t%s"
+		FAIL5 = "Refusing to set a workercount greater than NumCPU: %d > %d ---> setting workercount value to NumCPU: %d"
 		FAIL6 = "Could not open '%s'"
 		FAIL7 = "ConfigAtLaunch() failed to execute help text template"
 		FAIL8 = "Cannot find current working directory"
@@ -333,6 +335,11 @@ func ConfigAtLaunch() {
 	if Config.VectorMaxlines == 0 {
 		Config.VectorMaxlines = VECTORMAXLINES
 	}
+
+	if Config.WorkerCount > runtime.NumCPU() {
+		msg(fmt.Sprintf(FAIL5, Config.WorkerCount, runtime.NumCPU(), runtime.NumCPU()), MSGCRIT)
+		Config.WorkerCount = runtime.NumCPU()
+	}
 }
 
 // BuildDefaultConfig - return a CurrentConfiguration filled out with various default values
@@ -363,6 +370,8 @@ func BuildDefaultConfig() CurrentConfiguration {
 	c.SelfTest = 0
 	c.TickerActive = TICKERISACTIVE
 	c.VectorBot = false
+	c.VectorChtHt = DEFAULTCHRTHEIGHT
+	c.VectorChtWd = DEFAULTCHRTWIDTH
 	c.VectorMaxlines = VECTORMAXLINES
 	c.VectorModel = VECTORMODELDEFAULT
 	c.VectorNeighb = VECTORNEIGHBORS

constants.go

@@ -30,6 +30,8 @@ const (
 	DBLMMAPSIZE              = 151701 //[HGS] [B1: 0.310s][Δ: 0.310s] unnested lemma map built (151701 items)
 	DBWKMAPSIZE              = 236835 //[HGS] [A1: 0.385s][Δ: 0.385s] 236835 works built: map[string]DbWork
 	DEFAULTBROWSERCTX        = 14
+	DEFAULTCHRTWIDTH         = "1500px"
+	DEFAULTCHRTHEIGHT        = "1200px"
 	DEFAULTCOLUMN            = "stripped_line"
 	DEFAULTCORPORA           = "{\"gr\": true, \"lt\": true, \"in\": false, \"ch\": false, \"dp\": false}"
 	DEFAULTECHOLOGLEVEL      = 0
@@ -81,7 +83,7 @@ const (
 	MINORGENREWTCAP          = 250
 	NESTEDLEMMASIZE          = 543
 	ORDERBY                  = "index"
-	POLLEVERYNTABLES         = 50 // 3455 is the max number of tables in a search...
+	POLLEVERYNTABLES         = 34 // 3455 is the max number of tables in a search...
 	SERVEDFROMHOST           = "127.0.0.1"
 	SERVEDFROMPORT           = 8000
 	SIMULTANEOUSSEARCHES     = 3 // cap on the number of db connections at (S * Config.WorkerCount)

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/e-gun/sparse v0.0.0-20230418220937-07063da15582 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 // indirect
+	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect

initializeglobals.go

@@ -49,8 +49,8 @@ var (
 	AuLocs        = make(map[string]bool)
 	WkLocs        = make(map[string]bool)
 	StatCounter   = make(map[string]*atomic.Int32)
-	TheCorpora    = [5]string{"gr", "lt", "in", "ch", "dp"}
-	TheLanguages  = [2]string{"greek", "latin"}
+	TheCorpora    = []string{"gr", "lt", "in", "ch", "dp"}
+	TheLanguages  = []string{"greek", "latin"}
 	ServableFonts = map[string]FontTempl{"Noto": NotoFont, "Roboto": RobotoFont, "Fira": FiraFont}
 	LaunchTime    = time.Now()
 	WebsocketPool = WSFillNewPool()

main.go

@@ -25,22 +25,30 @@ func main() {
 		QUIT = "to stop the server press Control-C or close this window"
 	)
 
-	LaunchTime = time.Now()
-
-	LookForConfigFile()
-	ConfigAtLaunch()
+	//
+	// [0] debugging code block #1
+	//
 
-	// [a] memory debugging runs...
+	// memory use debugging runs have to be custom-built
 
 	// uncomment next and then: "curl http://localhost:8080/debug/pprof/heap > heap.0.pprof"
-	// "go tool pprof heap.0.pprof" -> "top 20"
+	// "go tool pprof heap.0.pprof" -> "top 20", etc.
 
 	//go func() {
 	//	msg("**THIS BUILD IS NOT FOR RELEASE** PPROF server is active", MSGCRIT)
 	//	http.ListenAndServe("localhost:8080", nil)
 	//}()
 
-	// [b] profiling runs...
+	LaunchTime = time.Now()
+
+	//
+	// [1] set up the runtime configuration
+	//
+
+	LookForConfigFile()
+	ConfigAtLaunch()
+
+	// profiling runs are requested from the command line
 
 	// e.g. running: ./HipparchiaGoServer -pc -st
 	// vectorless: ./HipparchiaGoServer -pc -st -dv
@@ -60,7 +68,6 @@ func main() {
 
 	messenger.Cfg = Config
 	messenger.Lnc.LaunchTime = LaunchTime
-	// messenger.Ctr = StatCounter
 	messenger.ResetScreen()
 
 	printversion()
@@ -70,6 +77,10 @@ func main() {
 		msg(fmt.Sprintf(TERMINALTEXT, PROJYEAR, PROJAUTH, PROJMAIL), MSGMAND)
 	}
 
+	//
+	// [2] set up things that will run forever in the background
+	//
+
 	SQLPool = FillPSQLPoool()
 	go WebsocketPool.WSPoolStartListening()
 
@@ -78,7 +89,10 @@ func main() {
 
 	go messenger.Ticker(TICKERDELAY)
 
-	// concurrent launching
+	//
+	// [3] concurrent loading of the core data
+	//
+
 	var awaiting sync.WaitGroup
 	awaiting.Add(1)
 	go func(awaiting *sync.WaitGroup) {
@@ -87,7 +101,12 @@ func main() {
 		start := time.Now()
 		previous := time.Now()
 
+		// would save 100MB or RAM if you only load 'gr' and 'lt' instead of everything; but dynamic loading is a PITA.
+		// a lazyworkmapper() vs workmapper() is easy enough; but a loadworksifneeded() at RtSetOption() will not fix the
+		// buildwkcorpusmap()+ problem that will remain; too many other globals are in play
+
 		AllWorks = workmapper()
+
 		messenger.Timer("A1", fmt.Sprintf(MSG1, len(AllWorks)), start, previous)
 
 		previous = time.Now()
@@ -134,12 +153,20 @@ func main() {
 	messenger.LogPaths("main() post-initialization")
 	msg(messenger.ColStyle(fmt.Sprintf(SUMM, time.Now().Sub(LaunchTime).Seconds())), -999)
 
-	// uncomment one or more of the next if debugging; they are very spammy for the console...
+	//
+	// [4] debugging code block #2
+	// uncomment one or more; they are very spammy in the console...
+	//
 
-	// go searchvaultreport()
-	// go wsclientreport()
+	// go searchvaultreport(2 * time.Second)
+	// go wsclientreport(2 * time.Second)
 
 	msg(QUIT, MSGMAND)
+
+	//
+	// [5] done: start the server (which will never return)
+	//
+
 	StartEchoServer()
 }
 

rt-websocket.go

@@ -294,7 +294,7 @@ func WSFillNewPool() *WSPool {
 //
 
 // wsclientreport - report the # and names of the active wsclients every N seconds
-func wsclientreport() {
+func wsclientreport(d time.Duration) {
 	// add the following to main.go: "go wsclientreport()"
 	for {
 		cl := WebsocketPool.ClientMap
@@ -303,6 +303,6 @@ func wsclientreport() {
 			cc = append(cc, k.ID)
 		}
 		msg(fmt.Sprintf("%d WebsocketPool clients: %s", len(cl), strings.Join(cc, ", ")), MSGNOTE)
-		time.Sleep(4 * time.Second)
+		time.Sleep(d)
 	}
 }

searchstructs.go

@@ -515,17 +515,18 @@ func SearchInfoHub() {
 //
 
 // searchvaultreport - report the # and names of the registered searches every N seconds
-func searchvaultreport() {
+func searchvaultreport(d time.Duration) {
 	// add the following to main.go: "go searchvaultreport()"
 	// it would be possible to "garbage collect" all searches where IsActive is "false" for too long
 	// but these really are not supposed to be a problem
+
 	for {
 		as := AllSearches.SearchMap
 		var ss []string
 		for k := range as {
 			ss = append(ss, k)
 		}
 		msg(fmt.Sprintf("%d in AllSearches: %s", len(as), strings.Join(ss, ", ")), MSGNOTE)
-		time.Sleep(4 * time.Second)
+		time.Sleep(d)
 	}
 }

vectorgraphing.go

@@ -23,13 +23,11 @@ import (
 )
 
 const (
-	CHRTWIDTH  = "1500px"
-	CHRTHEIGHT = "1200px"
-	DOTHUE     = 236
-	DOTSAT     = 33
-	DOTLUM     = 45
-	DOTLUMPER  = DOTLUM + 25
-	DOTSHIFT   = 0
+	DOTHUE    = 236
+	DOTSAT    = 33
+	DOTLUM    = 45
+	DOTLUMPER = DOTLUM + 25
+	DOTSHIFT  = 0
 )
 
 var (
@@ -49,12 +47,13 @@ func buildblanknngraph(settings string, coreword string, incl string) *charts.Gr
 
 	// FYI: https://echarts.apache.org/en/theme-builder.html, but there is not much room for "theming" ATM
 
+	wd, ht := getvecchrtwdht()
 	t := fmt.Sprintf(TITLESTR, coreword, incl)
 	sf := fmt.Sprintf(SAVEFILE, StripaccentsSTR(coreword))
 
 	graph := charts.NewGraph()
 	graph.SetGlobalOptions(
-		charts.WithInitializationOpts(opts.Initialization{Width: CHRTWIDTH, Height: CHRTHEIGHT}),
+		charts.WithInitializationOpts(opts.Initialization{Width: wd, Height: ht}),
 		charts.WithTitleOpts(getcharttitleopts(t, settings)),
 		charts.WithToolboxOpts(getcharttoolboxopts(sf)),
 		// charts.WithLegendOpts(opts.Legend{}),
@@ -424,13 +423,15 @@ func lda2dscatter(ntopics int, incl string, bagger string, Y, labels mat.Matrix,
 		SAVEFILE = "lda_tsne_2d_scattergraph"
 	)
 
+	wd, ht := getvecchrtwdht()
+
 	t := fmt.Sprintf(TITLE, incl)
 	st := "text prep: " + bagger
 
 	scatter := charts.NewScatter()
 	scatter.SetGlobalOptions(
 		charts.WithTitleOpts(getcharttitleopts(t, st)),
-		charts.WithInitializationOpts(opts.Initialization{Width: CHRTHEIGHT, Height: CHRTHEIGHT}), // square
+		charts.WithInitializationOpts(opts.Initialization{Width: wd, Height: ht}), // square
 		charts.WithTooltipOpts(getcharttooltip()),
 		charts.WithToolboxOpts(getcharttoolboxopts(SAVEFILE)),
 	)
@@ -484,6 +485,8 @@ func lda3dscatter(ntopics int, incl string, bagger string, Y, labels mat.Matrix,
 		SAVEFILE = "lda_tsne_3d_scattergraph"
 	)
 
+	wd, ht := getvecchrtwdht()
+
 	scatter := charts.NewScatter3D()
 	scatter.SetGlobalOptions(
 		charts.WithXAxis3DOpts(opts.XAxis3D{Name: "X-AXIS", Show: true}),
@@ -496,7 +499,7 @@ func lda3dscatter(ntopics int, incl string, bagger string, Y, labels mat.Matrix,
 
 	scatter.SetGlobalOptions(
 		charts.WithTitleOpts(getcharttitleopts(t, st)),
-		charts.WithInitializationOpts(opts.Initialization{Width: CHRTHEIGHT, Height: CHRTHEIGHT}), // square
+		charts.WithInitializationOpts(opts.Initialization{Width: wd, Height: ht}), // square
 		charts.WithTooltipOpts(getcharttooltip()),
 		charts.WithToolboxOpts(getcharttoolboxopts(SAVEFILE)),
 	)
@@ -653,6 +656,16 @@ func custom3dscatterhtmlandjs(s *charts.Scatter3D) string {
 	return htmlandjs
 }
 
+func getvecchrtwdht() (string, string) {
+	wd := Config.VectorChtWd
+	ht := Config.VectorChtHt
+	if wd == "" || ht == "" {
+		wd = DEFAULTCHRTWIDTH
+		ht = DEFAULTCHRTHEIGHT
+	}
+	return wd, ht
+}
+
 //
 // OVERRIDE GO-ECHARTS [original code at https://github.com/go-echarts/go-echarts]
 //