Integrate epoch recreation in MIES

- The recreation of epochs is by design not threadsafe, e.g. due to stimset recreation that can include a combine epoch, where a formula is executed with Execute. Thus, EP_FetchEpochs was split to have a separate threadsafe function that does not include epoch recreation. As epoch recreation requires to know the sweepDF to retrieve the DA waves, support for this argument was added in the call chain. - BSP (SB/DB) and SweepFormula support epochs from recreation - Exporting data to NWB does not support recreation as that module runs threadsafe.
AllenInstitute · Apr 15, 2024 · c63776a · c63776a
1 parent 2fe77b4
commit c63776a
Show file tree

Hide file tree

Showing 10 changed files with 193 additions and 70 deletions.
diff --git a/Packages/MIES/MIES_BrowserSettingsPanel.ipf b/Packages/MIES/MIES_BrowserSettingsPanel.ipf
@@ -1671,8 +1671,9 @@ Function BSP_AddTracesForEpochs(string win)
 		WAVE/Z/T textualValues = BSP_GetLogbookWave(win, LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, sweepNumber = sweepNumber)
 		ASSERT(WaveExists(textualValues), "Textual LabNotebook not found.")
 
+		DFREF sweepDFR = BSP_GetSweepDF(win, sweepNumber)
 		// present since a2172f03 (Added generations of epoch information wave, 2019-05-22)
-		WAVE/T/Z epochsFromLBN = EP_FetchEpochs(numericalValues, textualValues, sweepNumber, channelNumber, channelType)
+		WAVE/T/Z epochsFromLBN = EP_FetchEpochs(numericalValues, textualValues, sweepNumber, sweepDFR, channelNumber, channelType)
 		if(!WaveExists(epochsFromLBN))
 			continue
 		endif
@@ -2014,3 +2015,26 @@ static Function BSP_MemoryFreeMappedDF(string win)
 
 	AB_FreeWorkingDFs(dfList, index)
 End
+
+Function/DF BSP_GetSweepDF(string win, variable sweepNo)
+
+	variable isSweepBrowser
+	string   device
+
+	isSweepBrowser = BSP_IsSweepBrowser(win)
+	if(isSweepBrowser)
+		DFREF  sweepBrowserDFR = SB_GetSweepBrowserFolder(win)
+		WAVE/T sweepMap        = GetSweepBrowserMap(sweepBrowserDFR)
+	else
+		SFH_ASSERT(BSP_HasBoundDevice(win), "No device bound.")
+		device = BSP_GetDevice(win)
+		DFREF deviceDFR = GetDeviceDataPath(device)
+	endif
+
+	if(isSweepBrowser)
+		DFREF deviceDFR = SB_GetSweepDataFolder(sweepMap, sweepNo = sweepNo)
+	endif
+	DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo)
+
+	return sweepDFR
+End
diff --git a/Packages/MIES/MIES_Cache.ipf b/Packages/MIES/MIES_Cache.ipf
@@ -70,6 +70,31 @@
 /// @anchor CacheKeyGenerators
 /// @{
 
+/// @brief Cache key generator for recreated epochs wave
+Function/S CA_KeyRecreatedEpochs(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
+
+	variable crc
+
+	// the calculation assumes that recreated epochs are based on an old LNB
+	// thats content is treated as const (except mod time, as this check is fast)
+
+	ASSERT_TS(!IsFreeWave(numericalValues), "Numerical LNB wave must be global")
+	ASSERT_TS(!IsFreeWave(textualValues), "Textual LNB wave must be global")
+	ASSERT_TS(!IsFreeDatafolder(sweepDFR), "sweepDFR must not be free")
+
+	crc = StringCRC(0, GetWavesDataFolder(numericalValues, 2))
+	crc = StringCRC(crc, num2istr(WaveModCountWrapper(numericalValues)))
+
+	crc = StringCRC(crc, GetWavesDataFolder(textualValues, 2))
+	crc = StringCRC(crc, num2istr(WaveModCountWrapper(textualValues)))
+
+	crc = StringCRC(crc, GetDataFolder(1, sweepDFR))
+	crc = StringCRC(crc, num2istr(sweepNo))
+	crc = StringCRC(crc, num2istr(SWEEP_EPOCH_VERSION))
+
+	return num2istr(crc) + "Version 1"
+End
+
 /// @brief Cache key generator for oodDAQ offset waves
 Function/S CA_DistDAQCreateCacheKey(params)
 	STRUCT OOdDAQParams &params

diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf
@@ -142,7 +142,7 @@ static Function EP_CollectEpochInfoDA(WAVE/T epochWave, STRUCT DataConfiguration
 
 	for(i = 0; i < s.numDACEntries; i += 1)
 
-		if(WB_StimsetIsFromThirdParty(s.setName[i]) || !cmpstr(s.setName[i], STIMSET_TP_WHILE_DAQ))
+		if(IsEmpty(s.setName[i]) || WB_StimsetIsFromThirdParty(s.setName[i]) || !cmpstr(s.setName[i], STIMSET_TP_WHILE_DAQ))
 			continue
 		endif
 
@@ -248,7 +248,7 @@ static Function EP_CollectEpochInfoTTL(WAVE/T epochWave, STRUCT DataConfiguratio
 			continue
 		endif
 
-		if(WB_StimsetIsFromThirdParty(s.TTLsetName[i]))
+		if(IsEmpty(s.TTLsetName[i]) || WB_StimsetIsFromThirdParty(s.TTLsetName[i]))
 			continue
 		endif
 
@@ -1414,9 +1414,10 @@ End
 /// @param treelevel       [optional: default = not set] tree level of epochs, if not set then treelevel is ignored
 /// @param epochsWave      [optional: defaults to $""] when passed, gathers epoch information from this wave directly.
 ///                        This is required for callers who want to read epochs during MID_SWEEP_EVENT in analysis functions.
+/// @param sweepDFR        [optional: defaults to $""] when passed, allows to fetch also epoch from recreation
 ///
 /// @returns Text wave with epoch information, only rows fitting the input parameters are returned. Can also be a null wave.
-Function/WAVE EP_GetEpochs(WAVE numericalValues, WAVE textualValues, variable sweepNo, variable channelType, variable channelNumber, string shortname, [variable treelevel, WAVE/T epochsWave])
+Function/WAVE EP_GetEpochs(WAVE numericalValues, WAVE textualValues, variable sweepNo, variable channelType, variable channelNumber, string shortname, [variable treelevel, WAVE/T epochsWave, DFREF sweepDFR])
 
 	variable index, epochCnt, midSweep
 	string regexp
@@ -1429,9 +1430,16 @@ Function/WAVE EP_GetEpochs(WAVE numericalValues, WAVE textualValues, variable sw
 	else
 		midSweep = 1
 	endif
+	if(ParamIsDefault(sweepDFR))
+		DFREF sweepDFR = $""
+	endif
 
 	if(!midsweep)
-		WAVE/T/Z epochInfoChannel = EP_FetchEpochs(numericalValues, textualValues, sweepNo, channelNumber, channelType)
+		if(DataFolderExistsDFR(sweepDFR))
+			WAVE/T/Z epochInfoChannel = EP_FetchEpochs(numericalValues, textualValues, sweepNo, sweepDFR, channelNumber, channelType)
+		else
+			WAVE/T/Z epochInfoChannel = EP_FetchEpochs_TS(numericalValues, textualValues, sweepNo, channelNumber, channelType)
+		endif
 		if(!WaveExists(epochInfoChannel))
 			return $""
 		endif
@@ -1491,7 +1499,7 @@ Function/WAVE EP_GetEpochs(WAVE numericalValues, WAVE textualValues, variable sw
 	return matches
 End
 
-/// @brief Return free text wave with the epoch information of the given channel
+/// @brief Return free text wave with the epoch information of the given channel, do not attempt recreation
 ///
 /// @param numericalValues Numerical values from the labnotebook
 /// @param textualValues   Textual values from the labnotebook
@@ -1500,21 +1508,75 @@ End
 /// @param channelType     Type of channel @sa XopChannelConstants
 ///
 /// @return epochs wave, see GetEpochsWave() for the wave layout
-threadsafe Function/WAVE EP_FetchEpochs(WAVE numericalValues, WAVE/T/Z textualValues, variable sweep, variable channelNumber, variable channelType)
+threadsafe Function/WAVE EP_FetchEpochs_TS(WAVE numericalValues, WAVE/T/Z textualValues, variable sweep, variable channelNumber, variable channelType)
 
-	variable index
+	WAVE/Z epochs = EP_FetchEpochsFromLNB(numericalValues, textualValues, sweep, channelNumber, channelType)
+
+	return epochs
+End
+
+/// @brief Return free text wave with the epoch information of the given channel, attempt recreation
+///
+/// @param numericalValues Numerical values from the labnotebook
+/// @param textualValues   Textual values from the labnotebook
+/// @param sweep           Number of sweep
+/// @param singleSweepDFR  sweep DF, e.g. from GetSingleSweepFolder(deviceDFR, sweepNo)
+/// @param channelNumber   GUI channel number
+/// @param channelType     Type of channel @sa XopChannelConstants
+///
+/// @return epochs wave, see GetEpochsWave() for the wave layout
+Function/WAVE EP_FetchEpochs(WAVE numericalValues, WAVE/T/Z textualValues, variable sweep, DFREF singleSweepDFR, variable channelNumber, variable channelType)
+
+	WAVE/Z epochs = EP_FetchEpochsFromLNB(numericalValues, textualValues, sweep, channelNumber, channelType)
+	if(WaveExists(epochs))
+		return epochs
+	endif
+
+	WAVE/Z epochs = EP_FetchEpochsFromRecreation(numericalValues, textualValues, sweep, singleSweepDFR, channelNumber, channelType)
+
+	return epochs
+End
+
+static Function/WAVE EP_FetchEpochsFromRecreation(WAVE numericalValues, WAVE/T/Z textualValues, variable sweep, DFREF singleSweepDFR, variable channelNumber, variable channelType)
+
+	string epochList
+
+	ASSERT(DataFolderExistsDFR(singleSweepDFR), "Single sweep DFR is null")
+	ASSERT(channelType == XOP_CHANNEL_TYPE_DAC || channelType == XOP_CHANNEL_TYPE_ADC || channelType == XOP_CHANNEL_TYPE_TTL, "Unsupported channel type")
+
+	WAVE/Z epochs = EP_RecreateEpochsFromLoadedData(numericalValues, textualValues, singleSweepDFR, sweep)
 
-	ASSERT_TS(channelType == XOP_CHANNEL_TYPE_DAC || channelType == XOP_CHANNEL_TYPE_ADC || channelType == XOP_CHANNEL_TYPE_TTL, "Unsupported channel type")
 	if(channelType == XOP_CHANNEL_TYPE_ADC)
-		[WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", sweep, "DAC", channelNumber, XOP_CHANNEL_TYPE_ADC, DATA_ACQUISITION_MODE)
-		if(!WaveExists(setting))
+		channelNumber = EP_GetDACFromADCChannel(numericalvalues, sweep, channelNumber)
+		if(IsNaN(channelNumber))
 			return $""
 		endif
-		channelType   = XOP_CHANNEL_TYPE_DAC
-		channelNumber = setting[index]
-		if(!(IsFinite(channelNumber) && index < NUM_HEADSTAGES))
+		channelType = XOP_CHANNEL_TYPE_DAC
+	endif
+
+	epochList = EP_EpochWaveToStr(epochs, channelNumber, channelType)
+	if(IsEmpty(epochList))
+		return $""
+	endif
+	WAVE epChannel = EP_EpochStrToWave(epochList)
+	if(!DimSize(epChannel, ROWS))
+		return $""
+	endif
+
+	return epChannel
+End
+
+threadsafe static Function/WAVE EP_FetchEpochsFromLNB(WAVE numericalValues, WAVE/T/Z textualValues, variable sweep, variable channelNumber, variable channelType)
+
+	variable index
+
+	ASSERT_TS(channelType == XOP_CHANNEL_TYPE_DAC || channelType == XOP_CHANNEL_TYPE_ADC || channelType == XOP_CHANNEL_TYPE_TTL, "Unsupported channel type")
+	if(channelType == XOP_CHANNEL_TYPE_ADC)
+		channelNumber = EP_GetDACFromADCChannel(numericalvalues, sweep, channelNumber)
+		if(IsNaN(channelNumber))
 			return $""
 		endif
+		channelType = XOP_CHANNEL_TYPE_DAC
 	endif
 
 	[WAVE setting, index] = GetLastSettingChannel(numericalValues, textualValues, sweep, EPOCHS_ENTRY_KEY, channelNumber, channelType, DATA_ACQUISITION_MODE)
@@ -1535,6 +1597,21 @@ threadsafe Function/WAVE EP_FetchEpochs(WAVE numericalValues, WAVE/T/Z textualVa
 	return epochs
 End
 
+threadsafe static Function EP_GetDACFromADCChannel(WAVE numericalValues, variable sweep, variable channelNumber)
+
+	variable index
+
+	[WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", sweep, "DAC", channelNumber, XOP_CHANNEL_TYPE_ADC, DATA_ACQUISITION_MODE)
+	if(!WaveExists(setting))
+		return NaN
+	endif
+	if(!(IsFinite(setting[index]) && index < NUM_HEADSTAGES))
+		return NaN
+	endif
+
+	return setting[index]
+End
+
 /// @brief Append epoch information from the labnotebook to the newly cleared epoch wave
 Function EP_CopyLBNEpochsToEpochsWave(string device, variable sweepNo)
 	variable i, j, epochCnt, epochChannelCnt, chanType
@@ -1550,7 +1627,7 @@ Function EP_CopyLBNEpochsToEpochsWave(string device, variable sweepNo)
 
 	for(i = 0; i < NUM_DA_TTL_CHANNELS; i += 1)
 		for(chanType : channelTypes)
-			WAVE/T/Z epochChannel = EP_FetchEpochs(numericalValues, textualValues, sweepNo, i, chanType)
+			WAVE/T/Z epochChannel = EP_FetchEpochs_TS(numericalValues, textualValues, sweepNo, i, chanType)
 
 			if(!WaveExists(epochChannel))
 				continue
@@ -1566,11 +1643,11 @@ Function EP_CopyLBNEpochsToEpochsWave(string device, variable sweepNo)
 End
 
 /// @brief Helper function that returns (unintended) gaps between epochs
-static Function/WAVE EP_GetGaps(WAVE numericalValues, WAVE textualValues, variable sweepNo, variable channelType, variable channelNumber)
+static Function/WAVE EP_GetGaps(WAVE numericalValues, WAVE textualValues, variable sweepNo, DFREF sweepDFR, variable channelType, variable channelNumber)
 
 	variable i, numEpochs, index
 
-	WAVE/Z/T zeroEpochs = EP_GetEpochs(numericalValues, textualValues, sweepNo, channelType, channelNumber, ".*", treelevel = 0)
+	WAVE/Z/T zeroEpochs = EP_GetEpochs(numericalValues, textualValues, sweepNo, channelType, channelNumber, ".*", treelevel = 0, sweepDFR = sweepDFR)
 	if(!WaveExists(zeroEpochs))
 		return $""
 	endif
@@ -1606,22 +1683,22 @@ static Function/WAVE EP_GetGaps(WAVE numericalValues, WAVE textualValues, variab
 End
 
 /// @brief Returns the following epoch of a given epoch name in a specified tree level
-Function/WAVE EP_GetNextEpoch(WAVE numericalValues, WAVE textualValues, variable sweepNo, variable channelType, variable channelNumber, string shortname, variable treelevel, [variable ignoreGaps])
+Function/WAVE EP_GetNextEpoch(WAVE numericalValues, WAVE textualValues, variable sweepNo, DFREF sweepDFR, variable channelType, variable channelNumber, string shortname, variable treelevel, [variable ignoreGaps])
 
 	variable currentEnd, dim
 
 	ignoreGaps = ParamIsDefault(ignoreGaps) ? EPOCH_GAPS_WORKAROUND : !!ignoreGaps
 
-	WAVE/Z/T currentEpoch = EP_GetEpochs(numericalValues, textualValues, sweepNo, channelType, channelNumber, shortname)
+	WAVE/Z/T currentEpoch = EP_GetEpochs(numericalValues, textualValues, sweepNo, channelType, channelNumber, shortname, sweepDFR = sweepDFR)
 	ASSERT(WaveExists(currentEpoch) && DimSize(currentEpoch, ROWS) == 1, "Found multiple candidates for current epoch.")
 	currentEnd = str2numSafe(currentEpoch[0][EPOCH_COL_ENDTIME])
-	WAVE/Z/T levelEpochs = EP_GetEpochs(numericalValues, textualValues, sweepNo, channelType, channelNumber, ".*", treelevel = treelevel)
+	WAVE/Z/T levelEpochs = EP_GetEpochs(numericalValues, textualValues, sweepNo, channelType, channelNumber, ".*", treelevel = treelevel, sweepDFR = sweepDFR)
 	if(!WaveExists(levelEpochs))
 		return $""
 	endif
 
 	if(ignoreGaps)
-		WAVE/Z gaps = EP_GetGaps(numericalValues, textualValues, sweepNo, channelType, channelNumber)
+		WAVE/Z gaps = EP_GetGaps(numericalValues, textualValues, sweepNo, sweepDFR, channelType, channelNumber)
 		if(WaveExists(gaps))
 			dim = FindDimlabel(gaps, COLS, "GAPBEGIN")
 			FindValue/Z/RMD=[][dim]/V=(currentEnd) gaps
@@ -1657,10 +1734,17 @@ End
 /// @param sweepDFR        single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo)
 /// @param sweepNo         sweep number
 /// @returns recreated 4D epoch wave
-Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
+static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
 
 	STRUCT DataConfigurationResult s
 	variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex
+	string cacheKey
+
+	cacheKey = CA_KeyRecreatedEpochs(numericalValues, textualValues, sweepDFR, sweepNo)
+	WAVE/Z/T recEpochWave = CA_TryFetchingEntryFromCache(cacheKey)
+	if(WaveExists(recEpochWave))
+		return recEpochWave
+	endif
 
 	[s] = DC_RecreateDataConfigurationResultFromLNB(numericalValues, textualValues, sweepDFR, sweepNo)
 
@@ -1682,28 +1766,7 @@ Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textu
 	endfor
 	EP_SortEpochs(recEpochWave)
 
-	return recEpochWave
-End
-
-/// @brief Fetches a single epoch channel from a recreated epoch wave.
-///        The returned epoch channel wave has the same form as epoch information that was stored in the LNB returned by @ref EP_FetchEpochs
-///
-/// @param epochWave 4d epoch wave
-/// @param channelNumber GUI channel number
-/// @param channelType   channel type, one of @ref XopChannelConstants
-/// @returns epoch channel wave (2d)
-Function/WAVE EP_FetchEpochsFromRecreated(WAVE epochWave, variable channelNumber, variable channelType)
-
-	string epList
+	CA_StoreEntryIntoCache(cacheKey, recEpochWave)
 
-	epList = EP_EpochWaveToStr(epochWave, channelNumber, channelType)
-	if(IsEmpty(epList))
-		return $""
-	endif
-	WAVE epChannel = EP_EpochStrToWave(epList)
-	if(!DimSize(epChannel, ROWS))
-		return $""
-	endif
-
-	return epChannel
+	return recEpochWave
 End
diff --git a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf
@@ -1145,7 +1145,7 @@ threadsafe static Function NWB_AppendSweepLowLevel(STRUCT NWBAsyncParameters &s)
 			WAVE params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, params.channelNumber)
 			NWB_GetTimeSeriesProperties(s.nwbVersion, s.numericalKeys, s.numericalValues, params, tsp)
 			params.groupIndex = IsFinite(params.groupIndex) ? params.groupIndex : GetNextFreeGroupIndex(s.locationID, path)
-			WAVE/T/Z params.epochs = EP_FetchEpochs(s.numericalValues, s.textualValues, s.sweep, params.channelNumber, params.channelType)
+			WAVE/T/Z params.epochs = EP_FetchEpochs_TS(s.numericalValues, s.textualValues, s.sweep, params.channelNumber, params.channelType)
 			s.locationID = WriteSingleChannel(s.locationID, path, s.nwbVersion, params, tsp, compressionMode = s.compressionMode, nwbFilePath = s.nwbFilePath)
 		endif
 
@@ -1188,7 +1188,7 @@ threadsafe static Function NWB_AppendSweepLowLevel(STRUCT NWBAsyncParameters &s)
 			NWB_GetTimeSeriesProperties(s.nwbVersion, s.numericalKeys, s.numericalValues, params, tsp)
 			params.groupIndex = IsFinite(params.groupIndex) ? params.groupIndex : GetNextFreeGroupIndex(s.locationID, path)
 			WAVE     params.data   = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, i)
-			WAVE/T/Z params.epochs = EP_FetchEpochs(s.numericalValues, s.textualValues, s.sweep, i, params.channelType)
+			WAVE/T/Z params.epochs = EP_FetchEpochs_TS(s.numericalValues, s.textualValues, s.sweep, i, params.channelType)
 
 			s.locationID = WriteSingleChannel(s.locationID, path, s.nwbVersion, params, tsp, compressionMode = s.compressionMode, nwbFilePath = s.nwbFilePath)
 
@@ -1221,7 +1221,7 @@ threadsafe static Function NWB_AppendSweepLowLevel(STRUCT NWBAsyncParameters &s)
 				break
 			case IPNWB_CHANNEL_TYPE_DAC:
 				path = "/stimulus/presentation"
-				WAVE/T/Z params.epochs = EP_FetchEpochs(s.numericalValues, s.textualValues, s.sweep, params.channelNumber, params.channelType)
+				WAVE/T/Z params.epochs = EP_FetchEpochs_TS(s.numericalValues, s.textualValues, s.sweep, params.channelNumber, params.channelType)
 				break
 			default:
 				ASSERT_TS(0, "Unexpected channel type")