sfmphonemeextractor.cpp

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose: 
//
//=============================================================================

#include "sfmobjects/SFMPhonemeExtractor.h"
#include "tier2/riff.h"
#include "PhonemeConverter.h"
#include "filesystem.h"
#include "tier1/utlbuffer.h"
#include "sentence.h"
#include "movieobjects/dmesound.h"
#include "movieobjects/dmeanimationset.h"
#include "movieobjects/dmebookmark.h"
#include "movieobjects/dmeclip.h"
#include "movieobjects/dmechannel.h"
#include "soundchars.h"
#include "tier2/p4helpers.h"
#include "tier2/soundutils.h"
#include "tier1/utldict.h"

#include <windows.h>  // WAVEFORMATEX, WAVEFORMAT and ADPCM WAVEFORMAT!!!
#include <mmreg.h>

// memdbgon must be the last include file in a .cpp file!!!
#include "tier0/memdbgon.h"


static const char *s_pAttributeValueNames[LOG_PREVIEW_FLEX_CHANNEL_COUNT] = 
{
	"value",
	"balance",
	"multilevel"
};

static const char *s_pDefaultAttributeValueNames[LOG_PREVIEW_FLEX_CHANNEL_COUNT] = 
{
	"defaultValue",
	"defaultBalance",
	"defaultMultilevel"
};


struct Extractor
{
	PE_APITYPE			apitype;
	CSysModule			*module;
	IPhonemeExtractor	*extractor;
};


//-----------------------------------------------------------------------------
// Implementations of the phoneme extractor
//-----------------------------------------------------------------------------
class CSFMPhonemeExtractor : public ISFMPhonemeExtractor
{
public:
	CSFMPhonemeExtractor();

	// Inherited from ISFMPhonemeExtractor
	virtual bool Init();
	virtual void Shutdown();
	virtual int GetAPICount();
	virtual void GetAPIInfo( int index, CUtlString* pPrintName, PE_APITYPE *pAPIType );
	virtual void Extract( const PE_APITYPE& apiType, ExtractDesc_t& info, bool bWritePhonemesToWavFiles );
	virtual void ReApply( ExtractDesc_t& info );
	virtual bool GetSentence( CDmeGameSound *gameSound, CSentence& sentence );

private:
	int FindExtractor( PE_APITYPE type );
	bool GetWaveFormat( const char *filename, CUtlBuffer* pFormat, int *pDataSize, CSentence& sentence, bool &bGotSentence );
	void LogPhonemes( int nItemIndex, ExtractDesc_t& info ); 
	void ClearInterstitialSpaces( CDmeChannelsClip *pChannelsClip, CUtlDict< LogPreview_t *, int >& controlLookup, ExtractDesc_t& info );

	void StampControlValueLogs( CDmePreset *preset, DmeTime_t tHeadPosition, float flIntensity, CUtlDict< LogPreview_t *, int > &controlLookup );
	void WriteCurrentValuesIntoLogLayers( DmeTime_t tHeadPosition, const CUtlDict< LogPreview_t *, int > &controlLookup );
	void WriteDefaultValuesIntoLogLayers( DmeTime_t tHeadPosition, const CUtlDict< LogPreview_t *, int > &controlLookup );
	void BuildPhonemeLogList( CUtlVector< LogPreview_t > &list, CUtlVector< CDmeLog * > &logs );
	CDmeChannelsClip* FindFacialChannelsClip( const CUtlVector< LogPreview_t > &list );
	void BuildPhonemeToPresetMapping( const CUtlVector< CBasePhonemeTag * > &stream, CDmeAnimationSet *pSet, CDmePresetGroup * pPresetGroup, CUtlDict< CDmePreset *, unsigned short > &phonemeToPresetDict );

	CUtlVector< Extractor >	m_Extractors;
	int m_nCurrentExtractor;
};


//-----------------------------------------------------------------------------
// Singleton
//-----------------------------------------------------------------------------
static CSFMPhonemeExtractor g_ExtractorSingleton;
ISFMPhonemeExtractor *sfm_phonemeextractor = &g_ExtractorSingleton;


//-----------------------------------------------------------------------------
// Constructor
//-----------------------------------------------------------------------------
CSFMPhonemeExtractor::CSFMPhonemeExtractor() : m_nCurrentExtractor( -1 )
{
}


//-----------------------------------------------------------------------------
// Init, shutdown
//-----------------------------------------------------------------------------
bool CSFMPhonemeExtractor::Init()
{
	// Enumerate modules under bin folder of exe
	FileFindHandle_t findHandle;
	const char *pFilename = g_pFullFileSystem->FindFirstEx( "phonemeextractors/*.dll", "EXECUTABLE_PATH", &findHandle );
	while( pFilename )
	{	
		char fullpath[ 512 ];
		Q_snprintf( fullpath, sizeof( fullpath ), "phonemeextractors/%s", pFilename );

		// Msg( "Loading extractor from %s\n", fullpath );

		Extractor e;
		e.module = g_pFullFileSystem->LoadModule( fullpath );
		if ( !e.module )
		{
			pFilename = g_pFullFileSystem->FindNext( findHandle );
			continue;
		}

		CreateInterfaceFn factory = Sys_GetFactory( e.module );
		if ( !factory )
		{
			pFilename = g_pFullFileSystem->FindNext( findHandle );
			continue;
		}

		e.extractor = ( IPhonemeExtractor * )factory( VPHONEME_EXTRACTOR_INTERFACE, NULL );
		if ( !e.extractor )
		{
			Warning( "Unable to get IPhonemeExtractor interface version %s from %s\n", VPHONEME_EXTRACTOR_INTERFACE, fullpath );
			pFilename = g_pFullFileSystem->FindNext( findHandle );
			continue;
		}

		e.apitype = e.extractor->GetAPIType();

		m_Extractors.AddToTail( e );
		pFilename = g_pFullFileSystem->FindNext( findHandle );
	}

	g_pFullFileSystem->FindClose( findHandle );
	return true;
}

void CSFMPhonemeExtractor::Shutdown()
{
	int c = m_Extractors.Count();
	for ( int i = c - 1; i >= 0; i-- )
	{
		Extractor *e = &m_Extractors[ i ];
		g_pFullFileSystem->UnloadModule( e->module );
	}

	m_Extractors.RemoveAll();
}


//-----------------------------------------------------------------------------
// Finds an extractor of a particular type
//-----------------------------------------------------------------------------
int CSFMPhonemeExtractor::FindExtractor( PE_APITYPE type )
{
	for ( int i=0; i < m_Extractors.Count(); i++ )
	{
		if ( m_Extractors[i].apitype == type )
			return i;
	}
	return -1;
}


//-----------------------------------------------------------------------------
// Iterates over extractors
//-----------------------------------------------------------------------------
int CSFMPhonemeExtractor::GetAPICount()
{
	return m_Extractors.Count();
}

void CSFMPhonemeExtractor::GetAPIInfo( int index, CUtlString* pPrintName, PE_APITYPE *pAPIType )
{
	Assert( pPrintName );
	Assert( pAPIType );
	pPrintName->Set( m_Extractors[ index ].extractor->GetName() );
	*pAPIType = m_Extractors[ index ].apitype;
}

static void ParseSentence( CSentence& sentence, IterateRIFF &walk )
{
	CUtlBuffer buf( 0, 0, CUtlBuffer::TEXT_BUFFER );

	buf.EnsureCapacity( walk.ChunkSize() );
	walk.ChunkRead( buf.Base() );
	buf.SeekPut( CUtlBuffer::SEEK_HEAD, walk.ChunkSize() );

	sentence.InitFromDataChunk( buf.Base(), buf.TellPut() );
}

bool CSFMPhonemeExtractor::GetWaveFormat( const char *filename, CUtlBuffer *pBuf, int *pDataSize, CSentence& sentence, bool &bGotSentence )
{
	InFileRIFF riff( filename, *g_pFSIOReadBinary );
	Assert( riff.RIFFName() == RIFF_WAVE );

	// set up the iterator for the whole file (root RIFF is a chunk)
	IterateRIFF walk( riff, riff.RIFFSize() );

	bool gotFmt = false;
	bool gotData = false;
	bGotSentence = false;

	// Walk input chunks and copy to output
	while ( walk.ChunkAvailable() )
	{
		switch ( walk.ChunkName() )
		{
		case WAVE_FMT:
			{
				pBuf->SeekPut( CUtlBuffer::SEEK_HEAD, walk.ChunkSize() );
				walk.ChunkRead( pBuf->Base() );
				gotFmt = true;
			}
			break;
		case WAVE_DATA:
			{
				*pDataSize = walk.ChunkSize();
				gotData = true;
			}
			break;
		case WAVE_VALVEDATA:
			{
				bGotSentence = true;
				ParseSentence( sentence, walk );
			}
			break;
		default:
			break;
		}

		// Done
		if ( gotFmt && gotData && bGotSentence )
			return true;

		walk.ChunkNext();
	}
	return ( gotFmt && gotData );
}

bool CSFMPhonemeExtractor::GetSentence( CDmeGameSound *gameSound, CSentence& sentence )
{
	const char *filename = gameSound->m_SoundName.Get();
	Assert( filename && filename [ 0 ] );

	char soundname[ 512 ];
	// Note, calling PSkipSoundChars to remove any decorator characters used by the engine!!!
	Q_snprintf( soundname, sizeof( soundname ), "sound/%s", PSkipSoundChars( filename ) );
	Q_FixSlashes( soundname );

	char fullpath[ 512 ];
	g_pFullFileSystem->RelativePathToFullPath( soundname, "GAME", fullpath, sizeof( fullpath ) );

	// Get sound file metrics of interest
	CUtlBuffer buf;
	int nDataSize;
	bool bValidSentence = false;
	if ( !GetWaveFormat( soundname, &buf, &nDataSize, sentence, bValidSentence ) )
		return false;

	return bValidSentence;
}

static void BuildPhonemeStream( CSentence& in, CUtlVector< CBasePhonemeTag * >& list )
{
	for ( int i = 0; i < in.m_Words.Count(); ++i )
	{
		CWordTag *w = in.m_Words[ i ];
		if ( !w )
			continue;

		for ( int j = 0; j < w->m_Phonemes.Count(); ++j )
		{
			CPhonemeTag *ph = w->m_Phonemes[ j ];
			if ( !ph )
				continue;

			CBasePhonemeTag *newTag = new CBasePhonemeTag( *ph );
			list.AddToTail( newTag );
		}
	}

	if ( !in.m_Words.Count() && in.m_RunTimePhonemes.Count() )
	{
		for ( int i = 0 ; i < in.m_RunTimePhonemes.Count(); ++i )
		{
			CBasePhonemeTag *newTag = new CBasePhonemeTag( *in.m_RunTimePhonemes[ i ] );
			list.AddToTail( newTag );
		}
	}
}


//-----------------------------------------------------------------------------
// Purpose: Same the phoneme data into the sound files
//-----------------------------------------------------------------------------
static void StoreValveDataChunk( CSentence& sentence, IterateOutputRIFF& store )
{
	// Buffer and dump data
	CUtlBuffer buf( 0, 0, CUtlBuffer::TEXT_BUFFER );

	sentence.SaveToBuffer( buf );

	// Copy into store
	store.ChunkWriteData( buf.Base(), buf.TellPut() );
}

static bool SaveSentenceToWavFile( const char *pWavFile, CSentence& sentence )
{
	char pTempFile[ 512 ];

	Q_StripExtension( pWavFile, pTempFile, sizeof( pTempFile ) );
	Q_DefaultExtension( pTempFile, ".tmp", sizeof( pTempFile ) );

	if ( g_pFullFileSystem->FileExists( pTempFile, "GAME" ) )
	{
		g_pFullFileSystem->RemoveFile( pTempFile, "GAME" );
	}

	CP4AutoEditAddFile p4Checkout( pWavFile );
	if ( !g_pFullFileSystem->IsFileWritable( pWavFile ) )
	{
		Warning( "%s is not writable, can't save sentence data to file\n", pWavFile );
		return false;
	}

	// Rename original pWavFile to temp
	g_pFullFileSystem->RenameFile( pWavFile, pTempFile, "GAME" );

	// NOTE:  Put this in it's own scope so that the destructor for outfileRFF actually closes the file!!!!
	{
		// Read from Temp
		InFileRIFF riff( pTempFile, *g_pFSIOReadBinary );
		Assert( riff.RIFFName() == RIFF_WAVE );

		// set up the iterator for the whole file (root RIFF is a chunk)
		IterateRIFF walk( riff, riff.RIFFSize() );

		// And put data back into original pWavFile by name
		OutFileRIFF riffout( pWavFile, *g_pFSIOWriteBinary );

		IterateOutputRIFF store( riffout );

		bool bWordTrackWritten = false;

		// Walk input chunks and copy to output
		while ( walk.ChunkAvailable() )
		{
			store.ChunkStart( walk.ChunkName() );

			switch ( walk.ChunkName() )
			{
			case WAVE_VALVEDATA:
				{
					// Overwrite data
					StoreValveDataChunk( sentence, store );
					bWordTrackWritten = true;
				}
				break;
			default:
				store.CopyChunkData( walk );
				break;
			}

			store.ChunkFinish();

			walk.ChunkNext();
		}

		// If we didn't write it above, write it now
		if ( !bWordTrackWritten )
		{
			store.ChunkStart( WAVE_VALVEDATA );
			StoreValveDataChunk( sentence, store );
			store.ChunkFinish();
		}
	}

	// Remove temp file
	g_pFullFileSystem->RemoveFile( pTempFile, NULL );

	return true;
}


//-----------------------------------------------------------------------------
// Main entry point for phoneme extraction
//-----------------------------------------------------------------------------
void CSFMPhonemeExtractor::Extract( const PE_APITYPE& apiType, ExtractDesc_t& info, bool bWritePhonemesToWavFiles )
{
	if ( !info.m_pSet )
		return;

	int iExtractor = FindExtractor( apiType );
	if ( iExtractor == -1 )
		return;

	Extractor& extractor = m_Extractors[ iExtractor ];

	int nWorkItem;
	for ( nWorkItem = 0; nWorkItem < info.m_WorkList.Count(); ++nWorkItem )
	{
		CExtractInfo& workItem = info.m_WorkList[ nWorkItem ];

		workItem.m_flDuration = 0.0f;

		CSentence in;
		CSentence out;
		in.SetText( workItem.m_sHintText.String() );
		out.SetText( workItem.m_sHintText.String() );

		const char *pFileName = workItem.m_pSound->m_SoundName.Get();
		Assert( pFileName && pFileName [ 0 ] );

		char pSoundName[ 512 ];
		// Note, calling PSkipSoundChars to remove any decorator characters used by the engine!!!
		Q_snprintf( pSoundName, sizeof( pSoundName ), "sound/%s", PSkipSoundChars( pFileName ) );
		Q_FixSlashes( pSoundName );

		char pFullPath[ 512 ];
		g_pFullFileSystem->RelativePathToFullPath( pSoundName, "GAME", pFullPath, sizeof( pFullPath ) );

		// Get sound file metrics of interest
		CUtlBuffer buf;
		WAVEFORMATEX *format;
		int nDataSize;
		if ( !GetWaveFormat( pSoundName, &buf, &nDataSize, workItem.m_Sentence, workItem.m_bSentenceValid ) )
			continue;

		format = ( WAVEFORMATEX * )buf.Base();

		if ( !( format->wBitsPerSample > ( 1 << 3 ) ) )
		{
			// Have to warn and early-out here to avoid crashing with "integer divide by zero" below
			Warning( "Cannot extract phonemes from '%s', %u bits per sample.\n", pSoundName, format->wBitsPerSample );
			continue;
		}

		int nBitsPerSample = format->wBitsPerSample;
		float flSampleRate = (float)format->nSamplesPerSec;
		int nChannels = format->nChannels;
		int nSampleCount = nDataSize / ( nBitsPerSample >> 3 );

		float flTrueSampleSize = ( nBitsPerSample * nChannels ) >> 3;
		if ( format->wFormatTag == WAVE_FORMAT_ADPCM )
		{
			nBitsPerSample = 16;
			flTrueSampleSize = 0.5f;

			ADPCMWAVEFORMAT *pFormat = (ADPCMWAVEFORMAT *)buf.Base();
			int blockSize = ((pFormat->wSamplesPerBlock - 2) * pFormat->wfx.nChannels ) / 2;
			blockSize += 7 * pFormat->wfx.nChannels;

			int blockCount = nDataSize / blockSize;
			int blockRem = nDataSize % blockSize;

			// total samples in complete blocks
			nSampleCount = blockCount * pFormat->wSamplesPerBlock;

			// add remaining in a short block
			if ( blockRem )
			{
				nSampleCount += pFormat->wSamplesPerBlock - (((blockSize - blockRem) * 2) / nChannels);
			}
		}

		if ( flSampleRate > 0.0f )
		{
			workItem.m_flDuration = (float)nSampleCount / flSampleRate;
		}
		in.CreateEventWordDistribution( workItem.m_sHintText.String(), workItem.m_flDuration );
		if ( !workItem.m_bUseSentence || !workItem.m_bSentenceValid )
		{
			extractor.extractor->Extract( pFullPath,
				(int)( workItem.m_flDuration * flSampleRate * flTrueSampleSize ),
				Msg, in, out );

			// Tracker 57389:
			// Total hack to fix a bug where the Lipsinc extractor is messing up the # channels on 16 bit stereo waves
			if ( apiType == SPEECH_API_LIPSINC && nChannels == 2 && nBitsPerSample == 16 )
			{
				flTrueSampleSize *= 2.0f;
			}

			float bytespersecond = flSampleRate * flTrueSampleSize;

			int i;
			// Now convert byte offsets to times
			for ( i = 0; i < out.m_Words.Size(); i++ )
			{
				CWordTag *tag = out.m_Words[ i ];
				Assert( tag );
				if ( !tag )
					continue;

				tag->m_flStartTime = ( float )(tag->m_uiStartByte ) / bytespersecond;
				tag->m_flEndTime = ( float )(tag->m_uiEndByte ) / bytespersecond;

				for ( int j = 0; j < tag->m_Phonemes.Size(); j++ )
				{
					CPhonemeTag *ptag = tag->m_Phonemes[ j ];
					Assert( ptag );
					if ( !ptag )
						continue;

					ptag->SetStartTime( ( float )(ptag->m_uiStartByte ) / bytespersecond );
					ptag->SetEndTime( ( float )(ptag->m_uiEndByte ) / bytespersecond );
				}
			}

			if ( bWritePhonemesToWavFiles )
			{
				SaveSentenceToWavFile( pFullPath, out );
			}
		}
		else
		{
			Msg( "Using .wav file phonemes for (%s)\n", pSoundName );
			out = workItem.m_Sentence;
		}

		// Now create channel data
		workItem.ClearTags();
		BuildPhonemeStream( out, workItem.m_ApplyTags );
	}

	if ( info.m_bCreateBookmarks )
	{
		info.m_pSet->GetBookmarks().RemoveAll();
	}

	for ( nWorkItem = 0; nWorkItem < info.m_WorkList.Count(); ++nWorkItem )
	{
		LogPhonemes( nWorkItem, info );
	}
}


//-----------------------------------------------------------------------------
// 
//-----------------------------------------------------------------------------
static bool UniquePhonemeLessFunc( CBasePhonemeTag * const & lhs, CBasePhonemeTag * const & rhs )
{
	return lhs->GetPhonemeCode() < rhs->GetPhonemeCode();
}

void CSFMPhonemeExtractor::BuildPhonemeToPresetMapping( const CUtlVector< CBasePhonemeTag * > &stream, 
													   CDmeAnimationSet *pSet, CDmePresetGroup *pPresetGroup, CUtlDict< CDmePreset *, unsigned short > &phonemeToPresetDict )
{
	int i;
	CUtlRBTree< CBasePhonemeTag * > uniquePhonemes( 0, 0, UniquePhonemeLessFunc );
	for ( i = 0; i < stream.Count(); ++i )
	{
		CBasePhonemeTag *tag = stream[ i ];
		if ( uniquePhonemes.Find( tag ) == uniquePhonemes.InvalidIndex() )
		{
			uniquePhonemes.Insert( tag );
		}
	}

	for ( i = uniquePhonemes.FirstInorder(); i != uniquePhonemes.InvalidIndex(); i = uniquePhonemes.NextInorder( i ) )
	{
		CBasePhonemeTag *tag = uniquePhonemes[ i ];
		// Convert phoneme code to text
		char ph[ 32 ];
		Q_strncpy( ph, ConvertPhoneme( tag->GetPhonemeCode() ), sizeof( ph ) );

		char remappedph[ 32 ];
		// By default we search for a preset name p_xxx where xxx is the phoneme string
		Q_snprintf( remappedph, sizeof( remappedph ), "p_%s", ph );
		// Now find the preset in the animation set converter
		CDmePhonemeMapping *mapping = pSet->FindMapping( ph );
		if ( mapping )
		{
			Q_strncpy( remappedph, mapping->GetValueString( "preset" ), sizeof( remappedph ) );
		}

		// Now look up the preset, if it exists
		CDmePreset *preset = pPresetGroup->FindPreset( remappedph );
		if ( !preset )
		{
			Warning( "Animation set '%s' missing phoneme preset for '%s' -> '%s'\n", 
				pSet->GetName(), ph, remappedph );
			continue;
		}

		// Add to dictionary if it's not already there
		if ( phonemeToPresetDict.Find( ph ) == phonemeToPresetDict.InvalidIndex() )
		{
			phonemeToPresetDict.Insert( ph, preset );
		}
	}
}



//-----------------------------------------------------------------------------
// Finds the channels clip which refers to facial control values 
//-----------------------------------------------------------------------------
CDmeChannelsClip* CSFMPhonemeExtractor::FindFacialChannelsClip( const CUtlVector< LogPreview_t > &list )
{
	CDmeChannelsClip *pChannelsClip = NULL;

	int i;
	for ( i = list.Count() - 1; i >= 0; --i )
	{
		const LogPreview_t &lp = list[i];
		CDmeChannelsClip *check = FindAncestorReferencingElement< CDmeChannelsClip >( (CDmElement *)lp.m_hChannels[ 0 ].Get() );

		if ( !pChannelsClip && check )
		{
			pChannelsClip = check;
		}
		else
		{
			if ( pChannelsClip != check )
			{
				Warning( "Selected controls overlap multiple channels clips!!!\n" );
			}
		}
	}

	if ( !pChannelsClip )
	{
		Warning( "Unable to determine destination channels clip!!!\n" );
	}

	return pChannelsClip;
}


//-----------------------------------------------------------------------------
// Builds the list of logs which target facial control values 
//-----------------------------------------------------------------------------
void CSFMPhonemeExtractor::BuildPhonemeLogList( CUtlVector< LogPreview_t > &list, CUtlVector< CDmeLog * > &logs )
{
	for ( int i = 0; i < list.Count(); ++i )
	{
		LogPreview_t& p = list[ i ];

		for ( int channel = 0; channel < LOG_PREVIEW_FLEX_CHANNEL_COUNT; ++channel )
		{
			CDmeChannel *ch = p.m_hChannels[ channel ];
			if ( !ch )
				continue;

			CDmeLog *log = p.m_hChannels[ channel ]->GetLog();
			if ( !log )
				continue;

			logs.AddToTail( log );
		}
	}
}


//-----------------------------------------------------------------------------
// Writes default values into all log layers targetting facial control values
//-----------------------------------------------------------------------------
void CSFMPhonemeExtractor::WriteDefaultValuesIntoLogLayers( DmeTime_t tHeadPosition, const CUtlDict< LogPreview_t *, int > &controlLookup )
{
	// Write a zero into all relevant log layers
	for ( int j = controlLookup.First(); j != controlLookup.InvalidIndex(); j = controlLookup.Next( j ) )
	{
		LogPreview_t* lp = controlLookup[ j ];

		CDmElement *pControl = lp->m_hControl;

		for ( int chIndex = 0; chIndex < LOG_PREVIEW_FLEX_CHANNEL_COUNT; ++chIndex )
		{
			CDmeChannel *pChannel = lp->m_hChannels[ chIndex ];
			if ( !pChannel )
				continue;

			// Now get the log for the channel
			CDmeFloatLog *pFloatLog = CastElement< CDmeFloatLog >( pChannel->GetLog() );
			if ( !pFloatLog )
				continue;

			CDmeFloatLogLayer *pLayer = pFloatLog->GetLayer( pFloatLog->GetTopmostLayer() );
			if ( !pLayer )
				continue;

			float flDefaultValue = pControl->GetValue< float >( s_pDefaultAttributeValueNames[chIndex] );
			pLayer->InsertKey( tHeadPosition, flDefaultValue );
		}
	}
}


//-----------------------------------------------------------------------------
// Creates a new log key based on the interpolated value at that time
//-----------------------------------------------------------------------------
void CSFMPhonemeExtractor::WriteCurrentValuesIntoLogLayers( DmeTime_t tHeadPosition, const CUtlDict< LogPreview_t *, int > &controlLookup )
{
	// Write a zero into all relevant log layers
	for ( int j = controlLookup.First(); j != controlLookup.InvalidIndex(); j = controlLookup.Next( j ) )
	{
		LogPreview_t* lp = controlLookup[ j ];

		for ( int chIndex = 0; chIndex < LOG_PREVIEW_FLEX_CHANNEL_COUNT; ++chIndex )
		{
			CDmeChannel *pChannel = lp->m_hChannels[ chIndex ];
			if ( !pChannel )
				continue;

			// Now get the log for the channel
			CDmeFloatLog *pFloatLog = CastElement< CDmeFloatLog >( pChannel->GetLog() );
			if ( !pFloatLog )
				continue;

			CDmeFloatLogLayer *pLayer = pFloatLog->GetLayer( pFloatLog->GetTopmostLayer() );
			if ( !pLayer )
				continue;

			float flCurrentValue = pLayer->GetValue( tHeadPosition ); 
			pLayer->InsertKey( tHeadPosition, flCurrentValue );
		}
	}
}


//-----------------------------------------------------------------------------
// Samples extracted phoneme data and stamps that values into control value logs 
//-----------------------------------------------------------------------------
void CSFMPhonemeExtractor::StampControlValueLogs( CDmePreset *preset, DmeTime_t tHeadPosition, float flIntensity, CUtlDict< LogPreview_t *, int > &controlLookup )
{
	// Now walk the logs required by the preset
	const CDmrElementArray< CDmElement > &controlValues = preset->GetControlValues( );
	for ( int j = 0; j < controlValues.Count(); ++j )
	{
		// This control contains the preset value
		CDmElement *presetControl = controlValues[ j ];
		if ( !presetControl )
			continue;

		int visIndex = controlLookup.Find( presetControl->GetName() );
		if ( visIndex == controlLookup.InvalidIndex() )
			continue;

		LogPreview_t* lp = controlLookup[ visIndex ];

		for ( int chIndex = 0; chIndex < LOG_PREVIEW_FLEX_CHANNEL_COUNT; ++chIndex )
		{
			CDmeChannel *ch = lp->m_hChannels[ chIndex ];
			if ( !ch )
				continue;

			// Whereas this control contains the "default" value for the slider (since the presetControl won't have that value)
			CDmElement *defaultValueControl = lp->m_hControl.Get();
			if ( !defaultValueControl )
				continue;

			// Now get the log for the channel
			CDmeLog *log = ch->GetLog();
			if ( !log )
			{
				Assert( 0 );
				continue;
			}

			CDmeFloatLog *floatLog = CastElement< CDmeFloatLog >( log );
			if ( !floatLog )
				continue;

			CDmeFloatLogLayer *pLayer = floatLog->GetLayer( floatLog->GetTopmostLayer() );
			if ( !pLayer )
				continue;

			float flDefault = defaultValueControl->GetValue< float >( s_pDefaultAttributeValueNames[chIndex] );
			float flControlValue = presetControl->GetValue< float >( s_pAttributeValueNames[ chIndex ] );
			float flNewValue = flIntensity * ( flControlValue - flDefault );
			float flCurrent = pLayer->GetValue( tHeadPosition ) - flDefault;
			// Accumulate new value into topmost layer
			pLayer->InsertKey( tHeadPosition, flCurrent + flNewValue + flDefault );
		}
	}
}

void CSFMPhonemeExtractor::ClearInterstitialSpaces( CDmeChannelsClip *pChannelsClip, CUtlDict< LogPreview_t *, int >& controlLookup, ExtractDesc_t& info )
{
	Assert( info.m_pShot );
	Assert( pChannelsClip );

	if ( info.m_WorkList.Count() == 0 )
		return;

	// This is handled by the main layering code...
	if ( info.m_nExtractType == EXTRACT_WIPE_SOUNDS )
		return;

	// Now walk through all relevant logs
	CUtlVector< CDmeLog * > logs;
	BuildPhonemeLogList( info.m_ControlList, logs );

	DmeTime_t tMinTime( DMETIME_MAXTIME );
	DmeTime_t tMaxTime( DMETIME_MINTIME );

	int i;
	// Walk work items and figure out time bounds
	for ( i = 0; i < info.m_WorkList.Count(); ++i )
	{
		CExtractInfo &item = info.m_WorkList[ i ];

		CUtlVector< CDmeHandle< CDmeClip > > srcStack;
		CUtlVector< CDmeHandle< CDmeClip > > dstStack;

		// Convert original .wav start to animation set channels clip relative time
		item.m_pClip->BuildClipStack( &srcStack, info.m_pMovie, info.m_pShot );

		// NOTE: Time bounds measured in sound media time goes from 0 -> flWaveDuration
		DmeTime_t tSoundMediaStartTime = CDmeClip::FromChildMediaTime( srcStack, DMETIME_ZERO, false );
		DmeTime_t tSoundMediaEndTime   = CDmeClip::FromChildMediaTime( srcStack, DmeTime_t( item.m_flDuration ), false );

		// NOTE: Start and end time are measured in sound media time
		DmeTime_t tStartTime = item.m_pClip->GetStartInChildMediaTime();
		DmeTime_t tEndTime   = item.m_pClip->GetEndInChildMediaTime();

		// And convert back down into channels clip relative time
		pChannelsClip->BuildClipStack( &dstStack, info.m_pMovie, info.m_pShot );

		// Now convert back down to channels clip relative time
		DmeTime_t tChannelMediaStartTime = CDmeClip::ToChildMediaTime( dstStack, tSoundMediaStartTime, false );
		DmeTime_t tChannelMediaEndTime = CDmeClip::ToChildMediaTime( dstStack, tSoundMediaEndTime, false );

		// Find a scale + offset which transforms data in media space of the sound [namely, the phonemes]
		// into the media space of the channels [the logs that drive the facial animation]
		DmeTime_t tEndDuration = tChannelMediaEndTime - tChannelMediaStartTime;
		double flScale = ( item.m_flDuration != 0.0f ) ? tEndDuration.GetSeconds() / item.m_flDuration : 0.0f;
		DmeTime_t tOffset = tChannelMediaStartTime;

		DmeTime_t tChannelRelativeStartTime( tStartTime * flScale );
		tChannelRelativeStartTime += tOffset;
		DmeTime_t tChannelRelativeEndTime( tEndTime * flScale );
		tChannelRelativeEndTime += tOffset;

		if ( tChannelRelativeStartTime < tMinTime )
		{
			tMinTime = tChannelRelativeStartTime;
		}
		if ( tChannelRelativeEndTime > tMaxTime )
		{
			tMaxTime = tChannelRelativeEndTime;
		}
	}

	// Bloat by one quantum
	tMinTime -= DMETIME_MINDELTA;
	tMaxTime += DMETIME_MINDELTA;

	for ( i = 0; i < logs.Count(); ++i )
	{
		CDmeLog *log = logs[ i ];

		Assert( log->GetNumLayers() == 1 );
		CDmeLogLayer *layer = log->GetLayer( log->GetTopmostLayer() );

		if ( info.m_nExtractType == EXTRACT_WIPE_RANGE )
		{
			// Write default value keys into log
			// Write a default value at that time
			WriteDefaultValuesIntoLogLayers( tMinTime, controlLookup );

			// Write a default value at that time
			WriteDefaultValuesIntoLogLayers( tMaxTime, controlLookup );

			// Now discard all keys > tMinTime and < tMaxTime
			for ( int j = layer->GetKeyCount() - 1; j >= 0; --j )
			{
				DmeTime_t &t = layer->GetKeyTime( j );
				if ( t <= tMinTime )
					continue;
				if ( t >= tMaxTime )
					continue;

				layer->RemoveKey( j );
			}
		}
		else
		{
			Assert( info.m_nExtractType == EXTRACT_WIPE_CLIP );
			layer->ClearKeys();
		}
	}
}

void AddAnimSetBookmarkAtSoundMediaTime( const char *pName, DmeTime_t tStart, DmeTime_t tEnd, const CUtlVector< CDmeHandle< CDmeClip > > &srcStack, ExtractDesc_t& info )
{
	tStart = CDmeClip::FromChildMediaTime( srcStack, tStart, false );
	tEnd   = CDmeClip::FromChildMediaTime( srcStack, tEnd, false );

	tStart = info.m_pShot->ToChildMediaTime( tStart, false );
	tEnd   = info.m_pShot->ToChildMediaTime( tEnd, false );

	CDmeBookmark *pBookmark = CreateElement< CDmeBookmark >( pName );
	pBookmark->SetNote( pName );
	pBookmark->SetTime( tStart );
	pBookmark->SetDuration( tEnd - tStart );
	info.m_pSet->GetBookmarks().AddToTail( pBookmark );
}

//-----------------------------------------------------------------------------
// Main entry point for generating phoneme logs 
//-----------------------------------------------------------------------------
void CSFMPhonemeExtractor::LogPhonemes( int nItemIndex,	ExtractDesc_t& info )
{
	CExtractInfo &item = info.m_WorkList[ nItemIndex ];

	// Validate input parameters
	Assert( info.m_pSet && item.m_pClip && item.m_pSound );
	if ( !info.m_pSet || !item.m_pClip || !item.m_pSound )
		return;

	CDmePresetGroup *pPresetGroup = info.m_pSet->FindPresetGroup( "phoneme" );
	if ( !pPresetGroup )
	{
		Warning( "Animation set '%s' missing preset group 'phoneme'\n", info.m_pSet->GetName() );
		return;
	}

	if ( !info.m_pSet->GetPhonemeMap().Count() )
	{
		info.m_pSet->RestoreDefaultPhonemeMap();
	}

	// Walk through phoneme stack and build list of unique presets
	CUtlDict< CDmePreset *, unsigned short > phonemeToPresetDict;
	BuildPhonemeToPresetMapping( item.m_ApplyTags, info.m_pSet, pPresetGroup, phonemeToPresetDict );

	CDmeChannelsClip *pChannelsClip = FindFacialChannelsClip( info.m_ControlList );
	if ( !pChannelsClip )
		return;

	// Build a fast lookup of the visible sliders
	int i;
	CUtlDict< LogPreview_t *, int > controlLookup;
	for ( i = 0; i < info.m_ControlList.Count(); ++i )
	{
		controlLookup.Insert( info.m_ControlList[ i ].m_hControl->GetName(), &info.m_ControlList[ i ] );
	}

	// Only need to do this on the first item and we have multiple .wavs selected
	if ( nItemIndex == 0 && info.m_WorkList.Count() > 1 )
	{
		ClearInterstitialSpaces( pChannelsClip, controlLookup, info );
	}

	// Set up time selection, put channels into record and stamp out keyframes

	// Convert original .wav start to animation set channels clip relative time
	CUtlVector< CDmeHandle< CDmeClip > > srcStack;
	item.m_pClip->BuildClipStack( &srcStack, info.m_pMovie, info.m_pShot );
	if ( srcStack.Count() == 0 )
	{
		item.m_pClip->BuildClipStack( &srcStack, info.m_pMovie, NULL );
		if ( srcStack.Count() == 0 )
		{
			Msg( "Couldn't build stack sound clip to current shot\n" );
			return;
		}
	}

	// NOTE: Time bounds measured in sound media time goes from 0 -> flWaveDuration
	DmeTime_t tSoundMediaStartTime = CDmeClip::FromChildMediaTime( srcStack, DMETIME_ZERO, false );
	DmeTime_t tSoundMediaEndTime   = CDmeClip::FromChildMediaTime( srcStack, DmeTime_t( item.m_flDuration ), false );

	// NOTE: Start and end time are measured in sound media time
	DmeTime_t tStartTime = item.m_pClip->GetStartInChildMediaTime();
	DmeTime_t tEndTime   = item.m_pClip->GetEndInChildMediaTime();

	// And convert back down into channels clip relative time
	CUtlVector< CDmeHandle< CDmeClip > > dstStack;
	pChannelsClip->BuildClipStack( &dstStack, info.m_pMovie, info.m_pShot );

	// Now convert back down to channels clip relative time
	DmeTime_t tChannelMediaStartTime = CDmeClip::ToChildMediaTime( dstStack, tSoundMediaStartTime, false );
	DmeTime_t tChannelMediaEndTime   = CDmeClip::ToChildMediaTime( dstStack, tSoundMediaEndTime, false );

	// Find a scale + offset which transforms data in media space of the sound [namely, the phonemes]
	// into the media space of the channels [the logs that drive the facial animation]
	DmeTime_t tEndDuration = tChannelMediaEndTime - tChannelMediaStartTime;
	double flScale = ( item.m_flDuration != 0.0f ) ? tEndDuration.GetSeconds() / item.m_flDuration : 0.0f;
	DmeTime_t tOffset = tChannelMediaStartTime;

	CUtlVector< CDmeLog * > logs;
	BuildPhonemeLogList( info.m_ControlList, logs );

	// Add new write layer to each recording log
	for ( i = 0; i < logs.Count(); ++i )
	{
		logs[ i ]->AddNewLayer();
	}

	// Iterate over the entire range of the sound
	double flStartSoundTime = max( 0, tStartTime.GetSeconds() );
	double flEndSoundTime = min( item.m_flDuration, tEndTime.GetSeconds() );

	// Stamp keys right before and after the sound so as to 
	// not generate new values outside the import time range
	DmeTime_t tPrePhonemeTime( flStartSoundTime * flScale );
	tPrePhonemeTime += tOffset - DMETIME_MINDELTA;
	WriteCurrentValuesIntoLogLayers( tPrePhonemeTime, controlLookup );

	DmeTime_t tPostPhonemeTime( flEndSoundTime * flScale );
	tPostPhonemeTime += tOffset + DMETIME_MINDELTA;
	WriteCurrentValuesIntoLogLayers( tPostPhonemeTime, controlLookup );

	// add bookmarks
	if ( info.m_bCreateBookmarks )
	{
		AddAnimSetBookmarkAtSoundMediaTime( "start", tPrePhonemeTime, tPrePhonemeTime, srcStack, info );

		for ( i = 0; i < item.m_ApplyTags.Count() ; ++i )
		{
			CBasePhonemeTag *p = item.m_ApplyTags[ i ];
			const char *pPhonemeName = ConvertPhoneme( p->GetPhonemeCode() );
			DmeTime_t tStart = DmeTime_t( p->GetStartTime() );
			DmeTime_t tEnd   = DmeTime_t( p->GetEndTime() );
			AddAnimSetBookmarkAtSoundMediaTime( pPhonemeName, tStart, tEnd, srcStack, info );
		}

		AddAnimSetBookmarkAtSoundMediaTime( "end", tPostPhonemeTime, tPostPhonemeTime, srcStack, info );
	}

	if ( info.m_nFilterType == EXTRACT_FILTER_HOLD || info.m_nFilterType == EXTRACT_FILTER_LINEAR )
	{
		CDmePreset *pLastPreset = NULL;

		for ( i = 0; i < item.m_ApplyTags.Count() ; ++i )
		{
			CBasePhonemeTag *p = item.m_ApplyTags[ i ];

			DmeTime_t tStart = DmeTime_t( p->GetStartTime() );
			DmeTime_t tEnd   = DmeTime_t( p->GetEndTime() );

			int idx = phonemeToPresetDict.Find( ConvertPhoneme( p->GetPhonemeCode() ) );
			if ( idx == phonemeToPresetDict.InvalidIndex() )
				continue;

			CDmePreset *preset = phonemeToPresetDict[ idx ];
			if ( !preset )
				continue;

			DmeTime_t tKeyTime = tStart * flScale + tOffset;

			if ( info.m_nFilterType == EXTRACT_FILTER_HOLD )
			{
				// stamp value at end of phoneme (or default prior to first phoneme)
				// NOTE - this ignores phoneme length, but since all phonemes directly abut one another, this doesn't matter
				DmeTime_t tLastEnd = tKeyTime - DMETIME_MINDELTA;
				if ( tLastEnd > tPrePhonemeTime )
				{
					WriteDefaultValuesIntoLogLayers( tKeyTime - DMETIME_MINDELTA, controlLookup );
					if ( pLastPreset )
					{
						StampControlValueLogs( pLastPreset, tKeyTime - DMETIME_MINDELTA, 1.0f, controlLookup );
					}
				}
				pLastPreset = preset;
			}

			WriteDefaultValuesIntoLogLayers( tKeyTime, controlLookup );
			StampControlValueLogs( preset, tKeyTime, 1.0f, controlLookup );

			if ( info.m_nFilterType == EXTRACT_FILTER_HOLD && i == item.m_ApplyTags.Count() - 1 )
			{
				// stamp value at end of last phoneme
				tKeyTime = tEnd * flScale + tOffset;
				tKeyTime = min( tKeyTime, tPostPhonemeTime );
				WriteDefaultValuesIntoLogLayers( tKeyTime - DMETIME_MINDELTA, controlLookup );
				StampControlValueLogs( preset, tKeyTime - DMETIME_MINDELTA, 1.0f, controlLookup );

				// stamp default just after end of last phoneme to hold silence until tPostPhonemeTime
				WriteDefaultValuesIntoLogLayers( tKeyTime, controlLookup );
			}
		}
	}
	else
	{
		Assert( info.m_nFilterType == EXTRACT_FILTER_FIXED_WIDTH );

		double tStep = 1.0 / (double)clamp( info.m_flSampleRateHz, 1.0f, 1000.0f );

		float flFilter = max( info.m_flSampleFilterSize, 0.001f );
		float flOOFilter = 1.0f / flFilter;

		for ( double t = flStartSoundTime; t < flEndSoundTime; t += tStep )
		{
			DmeTime_t tPhonemeTime( t );

			// Determine the location of the sample in the channels clip
			DmeTime_t tKeyTime( t * flScale );
			tKeyTime += tOffset;

			// Write a default value at that time
			WriteDefaultValuesIntoLogLayers( tKeyTime, controlLookup );

			// Walk phonemes...
			for ( i = 0; i < item.m_ApplyTags.Count() ; ++i )
			{
				CBasePhonemeTag *p = item.m_ApplyTags[ i ];

				DmeTime_t tStart = DmeTime_t( p->GetStartTime() );
				DmeTime_t tEnd = DmeTime_t( p->GetEndTime() );

				bool bContinue = false;
				float flI = 0.0f;
				{
					DmeTime_t tFilter( flFilter );
					if ( tStart >= tPhonemeTime + tFilter || tEnd <= tPhonemeTime )
						bContinue = true;

					tStart = max( tStart, tPhonemeTime );
					tEnd = min( tEnd, tPhonemeTime + tFilter );

					flI = ( tEnd - tStart ).GetSeconds() * flOOFilter;
				}

				DmeTime_t dStart = tStart - tPhonemeTime;
				DmeTime_t dEnd = tEnd - tPhonemeTime;

				float t1 = dStart.GetSeconds() * flOOFilter;
				float t2 = dEnd.GetSeconds() * flOOFilter;

				Assert( bContinue == !( t1 < 1.0f && t2 > 0.0f ) );
				if ( !( t1 < 1.0f && t2 > 0.0f ) )
					continue;

				if ( t2 > 1 )
				{
					t2 = 1;
				}
				if ( t1 < 0 )
				{
					t1 = 0;
				}

				float flIntensity = ( t2 - t1 );
				Assert( fabs( flI - flIntensity ) < 0.000001f );

				int idx = phonemeToPresetDict.Find( ConvertPhoneme( p->GetPhonemeCode() ) );
				if ( idx == phonemeToPresetDict.InvalidIndex() )
					continue;

				CDmePreset *preset = phonemeToPresetDict[ idx ];
				if ( !preset )
					continue;

				StampControlValueLogs( preset, tKeyTime, flIntensity, controlLookup );
			}
		}
	}

	// Flatten write layers
	for ( i = 0; i < logs.Count(); ++i )
	{
		logs[ i ]->FlattenLayers( DMELOG_DEFAULT_THRESHHOLD, CDmeLog::FLATTEN_NODISCONTINUITY_FIXUP );
	}
}

void CSFMPhonemeExtractor::ReApply( ExtractDesc_t& info )
{
	if ( info.m_bCreateBookmarks )
	{
		info.m_pSet->GetBookmarks().RemoveAll();
	}

	for ( int nWorkItem = 0; nWorkItem < info.m_WorkList.Count(); ++nWorkItem )
	{
		LogPhonemes( nWorkItem, info );
	}
}