Some experimentation yields promising results...

Zach Fox committed 8 years ago

Commit 26e388b1 parent 79f30eb5 files · +769 −736Browse files

Unified · Split

5 files

interface/src/Application.cpp +1 −0
interface/src/scripting/TTSScriptingInterface.cpp +12 −5
interface/src/scripting/TTSScriptingInterface.h +9 −2
libraries/audio-client/src/AudioClient.cpp +735 −722
libraries/audio-client/src/AudioClient.h +12 −7

@@ -1182,6 +1182,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo · 1181 unmodified lines
1182		1182
1183	auto TTS = DependencyManager::get<TTSScriptingInterface>().data();	1183	auto TTS = DependencyManager::get<TTSScriptingInterface>().data();
1184	connect(TTS, &TTSScriptingInterface::ttsSampleCreated, audioIO, &AudioClient::handleTTSAudioInput);	1184	connect(TTS, &TTSScriptingInterface::ttsSampleCreated, audioIO, &AudioClient::handleTTSAudioInput);
		1185	connect(TTS, &TTSScriptingInterface::clearTTSBuffer, audioIO, &AudioClient::clearTTSBuffer);
1185		1186
1186	connect(audioIO, &AudioClient::inputReceived, [](const QByteArray& audio) {	1187	connect(audioIO, &AudioClient::inputReceived, [](const QByteArray& audio) {
1187	static auto recorder = DependencyManager::get<recording::Recorder>();	1188	static auto recorder = DependencyManager::get<recording::Recorder>();

interface/src/scripting/TTSScriptingInterface.cpp+12−5

         int16_t temp = (glm::sin(glm::radians((float)a))) * 32768;
         samples[a] = temp;
     }
-    emit ttsSampleCreated(_lastSoundByteArray);
+    emit ttsSampleCreated(_lastSoundByteArray, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * 50, 96);
     if (alsoInject) {
         AudioInjectorOptions options;
     }
 }
-void TTSScriptingInterface::speakText(const QString& textToSpeak, const bool& alsoInject) {
+void TTSScriptingInterface::speakText(const QString& textToSpeak,
+                                      const int& newChunkSize,
+                                      const int& timerInterval,
+                                      const int& sampleRate,
+                                      const int& bitsPerSample,
+                                      const bool& alsoInject) {
     WAVEFORMATEX fmt;
     fmt.wFormatTag = WAVE_FORMAT_PCM;
-    fmt.nSamplesPerSec = 24000;
+    fmt.nSamplesPerSec = sampleRate;
-    fmt.wBitsPerSample = 16;
+    fmt.wBitsPerSample = bitsPerSample;
     fmt.nChannels = 1;
     fmt.nBlockAlign = fmt.nChannels * fmt.wBitsPerSample / 8;
     fmt.nAvgBytesPerSec = fmt.nSamplesPerSec * fmt.nBlockAlign;
     _lastSoundByteArray.resize(0);
     _lastSoundByteArray.append(buf1, dwSize);
-    emit ttsSampleCreated(_lastSoundByteArray);
+    emit ttsSampleCreated(_lastSoundByteArray, newChunkSize, timerInterval);
     if (alsoInject) {
         AudioInjectorOptions options;
     if (_lastSoundAudioInjector) {
         _lastSoundAudioInjector->stop();
     }
+    emit clearTTSBuffer();
 }

interface/src/scripting/TTSScriptingInterface.h+9−2

 #include <sapi.h>      // SAPI
 #include <sphelper.h>  // SAPI Helper
 #include <AudioInjector.h>
+#include <AudioConstants.h>
 class TTSScriptingInterface : public QObject, public Dependency {
     Q_OBJECT
     ~TTSScriptingInterface();
     Q_INVOKABLE void testTone(const bool& alsoInject = false);
-    Q_INVOKABLE void speakText(const QString& textToSpeak, const bool& alsoInject = false);
+    Q_INVOKABLE void speakText(const QString& textToSpeak,
+                               const int& newChunkSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * 50),
+                               const int& timerInterval = 96,
+                               const int& sampleRate = 24000,
+                               const int& bitsPerSample = 16,
+                               const bool& alsoInject = false);
     Q_INVOKABLE void stopLastSpeech();
 signals:
-    void ttsSampleCreated(QByteArray outputArray);
+    void ttsSampleCreated(QByteArray outputArray, const int& newChunkSize, const int& timerInterval);
+    void clearTTSBuffer();
 private:
     class CComAutoInit {

libraries/audio-client/src/AudioClient.cpp+735−722

@@ -186,7 +186,7 @@ AudioClient::AudioClient() : · 185 unmodified lines
186	_networkToOutputResampler(NULL), _localToOutputResampler(NULL),	186	_networkToOutputResampler(NULL), _localToOutputResampler(NULL),
187	_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0),	187	_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0),
188	_audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), _stats(&_receivedAudioStream),	188	_audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), _stats(&_receivedAudioStream),
189	_positionGetter(DEFAULT_POSITION_GETTER),	189	_positionGetter(DEFAULT_POSITION_GETTER), _TTSTimer(this),
190	#if defined(Q_OS_ANDROID)	190	#if defined(Q_OS_ANDROID)
191	_checkInputTimer(this), _isHeadsetPluggedIn(false),	191	_checkInputTimer(this), _isHeadsetPluggedIn(false),
192	#endif	192	#endif
@@ -245,6 +245,8 @@ AudioClient::AudioClient() : · 52 unmodified lines
245	packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket");	245	packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket");
246	packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");	246	packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
247	packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat");	247	packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat");
		248
		249	connect(&_TTSTimer, &QTimer::timeout, this, &AudioClient::processTTSBuffer);
248	}	250	}
249		251
250	AudioClient::~AudioClient() {	252	AudioClient::~AudioClient() {
@@ -939,7 +941,7 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { · 688 unmodified lines
939	}	941	}
940	}	942	}
941		943
942	void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {	944	void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray, const int& sampleRate, const int& channelCount) {
943	// If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.	945	// If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.
944	bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();	946	bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();
945	if (_muted \|\| !_audioOutput \|\| (!_shouldEchoLocally && !hasReverb)) {	947	if (_muted \|\| !_audioOutput \|\| (!_shouldEchoLocally && !hasReverb)) {
@@ -949,7 +951,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { · 3 unmodified lines
949	// NOTE: we assume the inputFormat and the outputFormat are the same, since on any modern	951	// NOTE: we assume the inputFormat and the outputFormat are the same, since on any modern
950	// multimedia OS they should be. If there is a device that this is not true for, we can	952	// multimedia OS they should be. If there is a device that this is not true for, we can
951	// add back support to do resampling.	953	// add back support to do resampling.
952	if (_inputFormat.sampleRate() != _outputFormat.sampleRate()) {	954	if (sampleRate != _outputFormat.sampleRate()) {
953	return;	955	return;
954	}	956	}
955		957
@@ -972,7 +974,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { · 16 unmodified lines
972	static QByteArray loopBackByteArray;	974	static QByteArray loopBackByteArray;
973		975
974	int numInputSamples = inputByteArray.size() / AudioConstants::SAMPLE_SIZE;	976	int numInputSamples = inputByteArray.size() / AudioConstants::SAMPLE_SIZE;
975	int numLoopbackSamples = (numInputSamples * OUTPUT_CHANNEL_COUNT) / _inputFormat.channelCount();	977	int numLoopbackSamples = (numInputSamples * OUTPUT_CHANNEL_COUNT) / channelCount;
976		978
977	loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE);	979	loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE);
978		980
@@ -980,7 +982,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { · 1 unmodified line
980	int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());	982	int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());
981		983
982	// upmix mono to stereo	984	// upmix mono to stereo
983	if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, _inputFormat.channelCount(),	985	if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, channelCount,
984	OUTPUT_CHANNEL_COUNT)) {	986	OUTPUT_CHANNEL_COUNT)) {
985	// no conversion, just copy the samples	987	// no conversion, just copy the samples
986	memcpy(loopbackSamples, inputSamples, numInputSamples * AudioConstants::SAMPLE_SIZE);	988	memcpy(loopbackSamples, inputSamples, numInputSamples * AudioConstants::SAMPLE_SIZE);
@@ -1093,23 +1095,29 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { · 106 unmodified lines
1093	}	1095	}
1094	}	1096	}
1095		1097
1096	void AudioClient::processAudioAndAddToRingBuffer(QByteArray& inputByteArray,	1098	void AudioClient::handleMicAudioInput() {
1097	const uchar& channelCount,	1099	if (!_inputDevice \|\| _isPlayingBackRecording) {
1098	const qint32& bytesForDuration,	1100	return;
1099	QByteArray& rollingBuffer) {	1101	}
		1102
		1103	#if defined(Q_OS_ANDROID)
		1104	_inputReadsSinceLastCheck++;
		1105	#endif
		1106
1100	// input samples required to produce exactly NETWORK_FRAME_SAMPLES of output	1107	// input samples required to produce exactly NETWORK_FRAME_SAMPLES of output
1101	const int inputSamplesRequired =	1108	const int inputSamplesRequired =
1102	(_inputToNetworkResampler ? _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL)	1109	(_inputToNetworkResampler ? _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL)
1103	: AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *	1110	: AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
1104	channelCount;	1111	_inputFormat.channelCount();
1105		1112
1106	const auto inputAudioSamples = std::unique_ptr<int16_t[]>(new int16_t[inputSamplesRequired]);	1113	const auto inputAudioSamples = std::unique_ptr<int16_t[]>(new int16_t[inputSamplesRequired]);
		1114	QByteArray inputByteArray = _inputDevice->readAll();
1107		1115
1108	handleLocalEchoAndReverb(inputByteArray);	1116	handleLocalEchoAndReverb(inputByteArray, _inputFormat.sampleRate(), _inputFormat.channelCount());
1109		1117
1110	_inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());	1118	_inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());
1111		1119
1112	float audioInputMsecsRead = inputByteArray.size() / (float)(bytesForDuration);	1120	float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC));
1113	_stats.updateInputMsRead(audioInputMsecsRead);	1121	_stats.updateInputMsRead(audioInputMsecsRead);
1114		1122
1115	const int numNetworkBytes =	1123	const int numNetworkBytes =
@@ -1125,33 +1133,17 @@ void AudioClient::processAudioAndAddToRingBuffer(QByteArray& inputByteArray, · 9 unmodified lines
1125	} else {	1133	} else {
1126	_inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired);	1134	_inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired);
1127	possibleResampling(_inputToNetworkResampler, inputAudioSamples.get(), networkAudioSamples, inputSamplesRequired,	1135	possibleResampling(_inputToNetworkResampler, inputAudioSamples.get(), networkAudioSamples, inputSamplesRequired,
1128	numNetworkSamples, channelCount, _desiredInputFormat.channelCount());	1136	numNetworkSamples, _inputFormat.channelCount(), _desiredInputFormat.channelCount());
1129	}	1137	}
1130	int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * AudioConstants::SAMPLE_SIZE;	1138	int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * AudioConstants::SAMPLE_SIZE;
1131	float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(bytesForDuration);	1139	float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC));
1132	_stats.updateInputMsUnplayed(msecsInInputRingBuffer);	1140	_stats.updateInputMsUnplayed(msecsInInputRingBuffer);
1133		1141
1134	QByteArray audioBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);	1142	QByteArray audioBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
1135	rollingBuffer.append(audioBuffer);
1136	handleAudioInput(audioBuffer);	1143	handleAudioInput(audioBuffer);
1137	}	1144	}
1138	}	1145	}
1139		1146
1140	void AudioClient::handleMicAudioInput() {
1141	if (!_inputDevice \|\| _isPlayingBackRecording) {
1142	return;
1143	}
1144
1145	#if defined(Q_OS_ANDROID)
1146	_inputReadsSinceLastCheck++;
1147	#endif
1148
1149	QByteArray temp;
1150
1151	processAudioAndAddToRingBuffer(_inputDevice->readAll(), _inputFormat.channelCount(),
1152	_inputFormat.bytesForDuration(USECS_PER_MSEC), temp);
1153	}
1154
1155	void AudioClient::handleDummyAudioInput() {	1147	void AudioClient::handleDummyAudioInput() {
1156	const int numNetworkBytes =	1148	const int numNetworkBytes =
1157	_isStereoInput ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;	1149	_isStereoInput ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
@@ -1192,7 +1184,7 @@ int rawToWav(const char* rawData, const int& rawLength, const char* wavfn, long · 34 unmodified lines
1192	long chunksize = 0x10;	1184	long chunksize = 0x10;
1193	fwrite(&chunksize, 4, 1, wav);	1185	fwrite(&chunksize, 4, 1, wav);
1194		1186
1195	fmt.wFormatTag = 1; // WAVE_FORMAT_PCM	1187	fmt.wFormatTag = 1; // WAVE_FORMAT_PCM
1196	fmt.wChannels = channels;	1188	fmt.wChannels = channels;
1197	fmt.dwSamplesPerSec = frequency * 1;	1189	fmt.dwSamplesPerSec = frequency * 1;
1198	fmt.wBitsPerSample = 16;	1190	fmt.wBitsPerSample = 16;
@@ -1210,906 +1202,927 @@ int rawToWav(const char* rawData, const int& rawLength, const char* wavfn, long · 11 unmodified lines
1210	return 0;	1202	return 0;
1211	}	1203	}
1212		1204
1213	void AudioClient::handleTTSAudioInput(const QByteArray& audio) {	1205	void AudioClient::processTTSBuffer() {
1214	QByteArray audioBuffer(audio);	1206	Lock lock(_TTSMutex);
		1207	if (_TTSAudioBuffer.size() > 0) {
		1208	QByteArray part;
		1209	part.append(_TTSAudioBuffer.data(), _TTSChunkSize);
		1210	_TTSAudioBuffer.remove(0, _TTSChunkSize);
		1211	handleAudioInput(part);
		1212	} else {
		1213	_isProcessingTTS = false;
		1214	_TTSTimer.stop();
		1215	}
		1216	}
1215		1217
1216	QString filename = QString::number(usecTimestampNow());	1218	void AudioClient::handleTTSAudioInput(const QByteArray& audio, const int& newChunkSize, const int& timerInterval) {
1217	QString path = PathUtils::getAppDataPath() + "Audio/" + filename + "-before.wav";	1219	_TTSChunkSize = newChunkSize;
1218	rawToWav(audioBuffer.data(), audioBuffer.size(), path.toLocal8Bit(), 24000, 1);	1220	_TTSAudioBuffer.append(audio);
1219		1221
1220	QByteArray temp;	1222	handleLocalEchoAndReverb(_TTSAudioBuffer, 48000, 1);
1221		1223
1222	while (audioBuffer.size() > 0) {	1224	//QString filename = QString::number(usecTimestampNow());
1223	QByteArray part;	1225	//QString path = PathUtils::getAppDataPath() + "Audio/" + filename + "-before.wav";
1224	part.append(audioBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);	1226	//rawToWav(_TTSAudioBuffer.data(), _TTSAudioBuffer.size(), path.toLocal8Bit(), 24000, 1);
1225	audioBuffer.remove(0, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1226	processAudioAndAddToRingBuffer(part, 1, 48, temp);
1227	}
1228		1227
1229	filename = QString::number(usecTimestampNow());	1228	//QByteArray temp;
1230	path = PathUtils::getAppDataPath() + "Audio/" + filename + "-after.wav";	1229
1231	rawToWav(temp.data(), temp.size(), path.toLocal8Bit(), 12000, 1);	1230	_isProcessingTTS = true;
		1231	_TTSTimer.start(timerInterval);
		1232
		1233	//filename = QString::number(usecTimestampNow());
		1234	//path = PathUtils::getAppDataPath() + "Audio/" + filename + "-after.wav";
		1235	//rawToWav(temp.data(), temp.size(), path.toLocal8Bit(), 12000, 1);
1232	}	1236	}
1233		1237
1234	void AudioClient::prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock) {	1238	void AudioClient::clearTTSBuffer() {
1235	bool doSynchronously = localAudioLock.operator bool();	1239	_TTSAudioBuffer.resize(0);
1236	if (!localAudioLock) {	1240	_isProcessingTTS = false;
1237	localAudioLock.reset(new Lock(_localAudioMutex));	1241	_TTSTimer.stop();
1238	}	1242	}
1239		1243
1240	int samplesNeeded = std::numeric_limits<int>::max();	1244	void AudioClient::prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock) {
1241	while (samplesNeeded > 0) {	1245	bool doSynchronously = localAudioLock.operator bool();
1242	if (!doSynchronously) {	1246	if (!localAudioLock) {
1243	// unlock between every write to allow device switching	1247	localAudioLock.reset(new Lock(_localAudioMutex));
1244	localAudioLock->unlock();
1245	localAudioLock->lock();
1246	}	1248	}
1247		1249
1248	// in case of a device switch, consider bufferCapacity volatile across iterations	1250	int samplesNeeded = std::numeric_limits<int>::max();
1249	if (_outputPeriod == 0) {	1251	while (samplesNeeded > 0) {
1250	return;	1252	if (!doSynchronously) {
1251	}	1253	// unlock between every write to allow device switching
		1254	localAudioLock->unlock();
		1255	localAudioLock->lock();
		1256	}
1252		1257
1253	int bufferCapacity = _localInjectorsStream.getSampleCapacity();	1258	// in case of a device switch, consider bufferCapacity volatile across iterations
1254	int maxOutputSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * AudioConstants::STEREO;	1259	if (_outputPeriod == 0) {
1255	if (_localToOutputResampler) {	1260	return;
1256	maxOutputSamples = _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *	1261	}
1257	AudioConstants::STEREO;
1258	}
1259		1262
1260	samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);	1263	int bufferCapacity = _localInjectorsStream.getSampleCapacity();
1261	if (samplesNeeded < maxOutputSamples) {	1264	int maxOutputSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * AudioConstants::STEREO;
1262	// avoid overwriting the buffer to prevent losing frames	1265	if (_localToOutputResampler) {
1263	break;	1266	maxOutputSamples = _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
1264	}	1267	AudioConstants::STEREO;
		1268	}
1265		1269
1266	// get a network frame of local injectors' audio	1270	samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);
1267	if (!mixLocalAudioInjectors(_localMixBuffer)) {	1271	if (samplesNeeded < maxOutputSamples) {
1268	break;	1272	// avoid overwriting the buffer to prevent losing frames
1269	}	1273	break;
		1274	}
1270		1275
1271	// reverb	1276	// get a network frame of local injectors' audio
1272	if (_reverb) {	1277	if (!mixLocalAudioInjectors(_localMixBuffer)) {
1273	_localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);	1278	break;
1274	}	1279	}
1275		1280
1276	int samples;	1281	// reverb
1277	if (_localToOutputResampler) {	1282	if (_reverb) {
1278	// resample to output sample rate	1283	_localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1279	int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer,	1284	}
1280	AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1281		1285
1282	// write to local injectors' ring buffer	1286	int samples;
1283	samples = frames * AudioConstants::STEREO;	1287	if (_localToOutputResampler) {
1284	_localInjectorsStream.writeSamples(_localOutputMixBuffer, samples);	1288	// resample to output sample rate
		1289	int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer,
		1290	AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1285		1291
1286	} else {	1292	// write to local injectors' ring buffer
1287	// write to local injectors' ring buffer	1293	samples = frames * AudioConstants::STEREO;
1288	samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;	1294	_localInjectorsStream.writeSamples(_localOutputMixBuffer, samples);
1289	_localInjectorsStream.writeSamples(_localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
1290	}
1291		1295
1292	_localSamplesAvailable.fetch_add(samples, std::memory_order_release);	1296	} else {
1293	samplesNeeded -= samples;	1297	// write to local injectors' ring buffer
1294	}	1298	samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
1295	}	1299	_localInjectorsStream.writeSamples(_localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
		1300	}
1296		1301
1297	bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {	1302	_localSamplesAvailable.fetch_add(samples, std::memory_order_release);
1298	// check the flag for injectors before attempting to lock	1303	samplesNeeded -= samples;
1299	if (!_localInjectorsAvailable.load(std::memory_order_acquire)) {	1304	}
1300	return false;
1301	}	1305	}
1302		1306
1303	// lock the injectors	1307	bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
1304	Lock lock(_injectorsMutex);	1308	// check the flag for injectors before attempting to lock
1305		1309	if (!_localInjectorsAvailable.load(std::memory_order_acquire)) {
1306	QVector<AudioInjectorPointer> injectorsToRemove;	1310	return false;
		1311	}
1307		1312
1308	memset(mixBuffer, 0, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO * sizeof(float));	1313	// lock the injectors
		1314	Lock lock(_injectorsMutex);
1309		1315
1310	for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) {	1316	QVector<AudioInjectorPointer> injectorsToRemove;
1311	// the lock guarantees that injectorBuffer, if found, is invariant	1317
1312	AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();	1318	memset(mixBuffer, 0, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO * sizeof(float));
1313	if (injectorBuffer) {	1319
1314	static const int HRTF_DATASET_INDEX = 1;	1320	for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) {
1315		1321	// the lock guarantees that injectorBuffer, if found, is invariant
1316	int numChannels = injector->isAmbisonic() ? AudioConstants::AMBISONIC	1322	AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
1317	: (injector->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);	1323	if (injectorBuffer) {
1318	size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;	1324	static const int HRTF_DATASET_INDEX = 1;
1319		1325
1320	// get one frame from the injector	1326	int numChannels = injector->isAmbisonic()
1321	memset(_localScratchBuffer, 0, bytesToRead);	1327	? AudioConstants::AMBISONIC
1322	if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) {	1328	: (injector->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
1323	if (injector->isAmbisonic()) {	1329	size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
1324	// no distance attenuation	1330
1325	float gain = injector->getVolume();	1331	// get one frame from the injector
1326		1332	memset(_localScratchBuffer, 0, bytesToRead);
1327	//	1333	if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) {
1328	// Calculate the soundfield orientation relative to the listener.	1334	if (injector->isAmbisonic()) {
1329	// Injector orientation can be used to align a recording to our world coordinates.	1335	// no distance attenuation
1330	//	1336	float gain = injector->getVolume();
1331	glm::quat relativeOrientation = injector->getOrientation() * glm::inverse(_orientationGetter());	1337
1332		1338	//
1333	// convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system	1339	// Calculate the soundfield orientation relative to the listener.
1334	float qw = relativeOrientation.w;	1340	// Injector orientation can be used to align a recording to our world coordinates.
1335	float qx = -relativeOrientation.z;	1341	//
1336	float qy = -relativeOrientation.x;	1342	glm::quat relativeOrientation = injector->getOrientation() * glm::inverse(_orientationGetter());
1337	float qz = relativeOrientation.y;	1343
1338		1344	// convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system
1339	// Ambisonic gets spatialized into mixBuffer	1345	float qw = relativeOrientation.w;
1340	injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, qw, qx, qy, qz, gain,	1346	float qx = -relativeOrientation.z;
1341	AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);	1347	float qy = -relativeOrientation.x;
1342		1348	float qz = relativeOrientation.y;
1343	} else if (injector->isStereo()) {	1349
1344	// stereo gets directly mixed into mixBuffer	1350	// Ambisonic gets spatialized into mixBuffer
1345	float gain = injector->getVolume();	1351	injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, qw, qx, qy, qz, gain,
1346	for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {	1352	AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1347	mixBuffer[i] += convertToFloat(_localScratchBuffer[i]) * gain;	1353
		1354	} else if (injector->isStereo()) {
		1355	// stereo gets directly mixed into mixBuffer
		1356	float gain = injector->getVolume();
		1357	for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
		1358	mixBuffer[i] += convertToFloat(_localScratchBuffer[i]) * gain;
		1359	}
		1360
		1361	} else {
		1362	// calculate distance, gain and azimuth for hrtf
		1363	glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
		1364	float distance = glm::max(glm::length(relativePosition), EPSILON);
		1365	float gain = gainForSource(distance, injector->getVolume());
		1366	float azimuth = azimuthForSource(relativePosition);
		1367
		1368	// mono gets spatialized into mixBuffer
		1369	injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, azimuth, distance,
		1370	gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1348	}	1371	}
1349		1372
1350	} else {	1373	} else {
1351	// calculate distance, gain and azimuth for hrtf	1374	qCDebug(audioclient) << "injector has no more data, marking finished for removal";
1352	glm::vec3 relativePosition = injector->getPosition() - _positionGetter();	1375	injector->finishLocalInjection();
1353	float distance = glm::max(glm::length(relativePosition), EPSILON);	1376	injectorsToRemove.append(injector);
1354	float gain = gainForSource(distance, injector->getVolume());
1355	float azimuth = azimuthForSource(relativePosition);
1356
1357	// mono gets spatialized into mixBuffer
1358	injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, azimuth, distance, gain,
1359	AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1360	}	1377	}
1361		1378
1362	} else {	1379	} else {
1363	qCDebug(audioclient) << "injector has no more data, marking finished for removal";	1380	qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal";
1364	injector->finishLocalInjection();	1381	injector->finishLocalInjection();
1365	injectorsToRemove.append(injector);	1382	injectorsToRemove.append(injector);
1366	}	1383	}
1367
1368	} else {
1369	qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal";
1370	injector->finishLocalInjection();
1371	injectorsToRemove.append(injector);
1372	}	1384	}
1373	}
1374		1385
1375	for (const AudioInjectorPointer& injector : injectorsToRemove) {	1386	for (const AudioInjectorPointer& injector : injectorsToRemove) {
1376	qCDebug(audioclient) << "removing injector";	1387	qCDebug(audioclient) << "removing injector";
1377	_activeLocalAudioInjectors.removeOne(injector);	1388	_activeLocalAudioInjectors.removeOne(injector);
1378	}	1389	}
1379		1390
1380	// update the flag	1391	// update the flag
1381	_localInjectorsAvailable.exchange(!_activeLocalAudioInjectors.empty(), std::memory_order_release);	1392	_localInjectorsAvailable.exchange(!_activeLocalAudioInjectors.empty(), std::memory_order_release);
1382		1393
1383	return true;	1394	return true;
1384	}	1395	}
1385		1396
1386	void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {	1397	void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
1387	const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());	1398	const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
1388	assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);	1399	assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
1389		1400
1390	outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE);	1401	outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE);
1391	int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());	1402	int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
1392		1403
1393	bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();	1404	bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();
1394		1405
1395	// apply stereo reverb	1406	// apply stereo reverb
1396	if (hasReverb) {	1407	if (hasReverb) {
1397	updateReverbOptions();	1408	updateReverbOptions();
1398	int16_t* reverbSamples = _networkToOutputResampler ? _networkScratchBuffer : outputSamples;	1409	int16_t* reverbSamples = _networkToOutputResampler ? _networkScratchBuffer : outputSamples;
1399	_listenerReverb.render(decodedSamples, reverbSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);	1410	_listenerReverb.render(decodedSamples, reverbSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1400	}	1411	}
1401		1412
1402	// resample to output sample rate	1413	// resample to output sample rate
1403	if (_networkToOutputResampler) {	1414	if (_networkToOutputResampler) {
1404	const int16_t* inputSamples = hasReverb ? _networkScratchBuffer : decodedSamples;	1415	const int16_t* inputSamples = hasReverb ? _networkScratchBuffer : decodedSamples;
1405	_networkToOutputResampler->render(inputSamples, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);	1416	_networkToOutputResampler->render(inputSamples, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1406	}	1417	}
1407		1418
1408	// if no transformations were applied, we still need to copy the buffer	1419	// if no transformations were applied, we still need to copy the buffer
1409	if (!hasReverb && !_networkToOutputResampler) {	1420	if (!hasReverb && !_networkToOutputResampler) {
1410	memcpy(outputSamples, decodedSamples, decodedBuffer.size());	1421	memcpy(outputSamples, decodedSamples, decodedBuffer.size());
		1422	}
1411	}	1423	}
1412	}
1413		1424
1414	void AudioClient::sendMuteEnvironmentPacket() {	1425	void AudioClient::sendMuteEnvironmentPacket() {
1415	auto nodeList = DependencyManager::get<NodeList>();	1426	auto nodeList = DependencyManager::get<NodeList>();
1416		1427
1417	int dataSize = sizeof(glm::vec3) + sizeof(float);	1428	int dataSize = sizeof(glm::vec3) + sizeof(float);
1418		1429
1419	auto mutePacket = NLPacket::create(PacketType::MuteEnvironment, dataSize);	1430	auto mutePacket = NLPacket::create(PacketType::MuteEnvironment, dataSize);
1420		1431
1421	const float MUTE_RADIUS = 50;	1432	const float MUTE_RADIUS = 50;
1422		1433
1423	glm::vec3 currentSourcePosition = _positionGetter();	1434	glm::vec3 currentSourcePosition = _positionGetter();
1424		1435
1425	mutePacket->writePrimitive(currentSourcePosition);	1436	mutePacket->writePrimitive(currentSourcePosition);
1426	mutePacket->writePrimitive(MUTE_RADIUS);	1437	mutePacket->writePrimitive(MUTE_RADIUS);
1427		1438
1428	// grab our audio mixer from the NodeList, if it exists	1439	// grab our audio mixer from the NodeList, if it exists
1429	SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);	1440	SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
1430		1441
1431	if (audioMixer) {	1442	if (audioMixer) {
1432	// send off this mute packet	1443	// send off this mute packet
1433	nodeList->sendPacket(std::move(mutePacket), *audioMixer);	1444	nodeList->sendPacket(std::move(mutePacket), *audioMixer);
		1445	}
1434	}	1446	}
1435	}
1436		1447
1437	void AudioClient::setMuted(bool muted, bool emitSignal) {	1448	void AudioClient::setMuted(bool muted, bool emitSignal) {
1438	if (_muted != muted) {	1449	if (_muted != muted) {
1439	_muted = muted;	1450	_muted = muted;
1440	if (emitSignal) {	1451	if (emitSignal) {
1441	emit muteToggled(_muted);	1452	emit muteToggled(_muted);
		1453	}
1442	}	1454	}
1443	}	1455	}
1444	}
1445		1456
1446	void AudioClient::setNoiseReduction(bool enable, bool emitSignal) {	1457	void AudioClient::setNoiseReduction(bool enable, bool emitSignal) {
1447	if (_isNoiseGateEnabled != enable) {	1458	if (_isNoiseGateEnabled != enable) {
1448	_isNoiseGateEnabled = enable;	1459	_isNoiseGateEnabled = enable;
1449	if (emitSignal) {	1460	if (emitSignal) {
1450	emit noiseReductionChanged(_isNoiseGateEnabled);	1461	emit noiseReductionChanged(_isNoiseGateEnabled);
		1462	}
1451	}	1463	}
1452	}	1464	}
1453	}
1454		1465
1455	bool AudioClient::setIsStereoInput(bool isStereoInput) {	1466	bool AudioClient::setIsStereoInput(bool isStereoInput) {
1456	bool stereoInputChanged = false;	1467	bool stereoInputChanged = false;
1457	if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) {	1468	if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) {
1458	_isStereoInput = isStereoInput;	1469	_isStereoInput = isStereoInput;
1459	stereoInputChanged = true;	1470	stereoInputChanged = true;
1460		1471
1461	if (_isStereoInput) {	1472	if (_isStereoInput) {
1462	_desiredInputFormat.setChannelCount(2);	1473	_desiredInputFormat.setChannelCount(2);
1463	} else {	1474	} else {
1464	_desiredInputFormat.setChannelCount(1);	1475	_desiredInputFormat.setChannelCount(1);
1465	}	1476	}
1466		1477
1467	// restart the codec	1478	// restart the codec
1468	if (_codec) {	1479	if (_codec) {
1469	if (_encoder) {	1480	if (_encoder) {
1470	_codec->releaseEncoder(_encoder);	1481	_codec->releaseEncoder(_encoder);
		1482	}
		1483	_encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE,
		1484	_isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO);
1471	}	1485	}
1472	_encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE,	1486	qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput;
1473	_isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO);	1487
1474	}	1488	// restart the input device
1475	qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput;	1489	switchInputToAudioDevice(_inputDeviceInfo);
1476		1490
1477	// restart the input device	1491	emit isStereoInputChanged(_isStereoInput);
1478	switchInputToAudioDevice(_inputDeviceInfo);	1492	}
1479		1493
1480	emit isStereoInputChanged(_isStereoInput);	1494	return stereoInputChanged;
1481	}	1495	}
1482		1496
1483	return stereoInputChanged;	1497	bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) {
1484	}	1498	AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
		1499	if (injectorBuffer) {
		1500	// local injectors are on the AudioInjectorsThread, so we must guard access
		1501	Lock lock(_injectorsMutex);
		1502	if (!_activeLocalAudioInjectors.contains(injector)) {
		1503	qCDebug(audioclient) << "adding new injector";
		1504	_activeLocalAudioInjectors.append(injector);
		1505	// move local buffer to the LocalAudioThread to avoid dataraces with AudioInjector (like stop())
		1506	injectorBuffer->setParent(nullptr);
		1507
		1508	// update the flag
		1509	_localInjectorsAvailable.exchange(true, std::memory_order_release);
		1510	} else {
		1511	qCDebug(audioclient) << "injector exists in active list already";
		1512	}
		1513
		1514	return true;
1485		1515
1486	bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) {
1487	AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
1488	if (injectorBuffer) {
1489	// local injectors are on the AudioInjectorsThread, so we must guard access
1490	Lock lock(_injectorsMutex);
1491	if (!_activeLocalAudioInjectors.contains(injector)) {
1492	qCDebug(audioclient) << "adding new injector";
1493	_activeLocalAudioInjectors.append(injector);
1494	// move local buffer to the LocalAudioThread to avoid dataraces with AudioInjector (like stop())
1495	injectorBuffer->setParent(nullptr);
1496
1497	// update the flag
1498	_localInjectorsAvailable.exchange(true, std::memory_order_release);
1499	} else {	1516	} else {
1500	qCDebug(audioclient) << "injector exists in active list already";	1517	// no local buffer
		1518	return false;
1501	}	1519	}
1502
1503	return true;
1504
1505	} else {
1506	// no local buffer
1507	return false;
1508	}	1520	}
1509	}
1510		1521
1511	void AudioClient::outputFormatChanged() {	1522	void AudioClient::outputFormatChanged() {
1512	_outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) /	1523	_outputFrameSize =
1513	_desiredOutputFormat.sampleRate();	1524	(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) /
1514	_receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);	1525	_desiredOutputFormat.sampleRate();
1515	}	1526	_receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
		1527	}
1516		1528
1517	bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) {	1529	bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) {
1518	Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");	1530	Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");
1519		1531
1520	qCDebug(audioclient) << __FUNCTION__ << "inputDeviceInfo: [" << inputDeviceInfo.deviceName() << "]";	1532	qCDebug(audioclient) << __FUNCTION__ << "inputDeviceInfo: [" << inputDeviceInfo.deviceName() << "]";
1521	bool supportedFormat = false;	1533	bool supportedFormat = false;
1522		1534
1523	// NOTE: device start() uses the Qt internal device list	1535	// NOTE: device start() uses the Qt internal device list
1524	Lock lock(_deviceMutex);	1536	Lock lock(_deviceMutex);
1525		1537
1526	#if defined(Q_OS_ANDROID)	1538	#if defined(Q_OS_ANDROID)
1527	_shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged	1539	_shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged
1528	#endif	1540	#endif
1529		1541
1530	// cleanup any previously initialized device	1542	// cleanup any previously initialized device
1531	if (_audioInput) {	1543	if (_audioInput) {
1532	// The call to stop() causes _inputDevice to be destructed.	1544	// The call to stop() causes _inputDevice to be destructed.
1533	// That in turn causes it to be disconnected (see for example	1545	// That in turn causes it to be disconnected (see for example
1534	// http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect).	1546	// http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect).
1535	_audioInput->stop();	1547	_audioInput->stop();
1536	_inputDevice = NULL;	1548	_inputDevice = NULL;
1537		1549
1538	_audioInput->deleteLater();	1550	_audioInput->deleteLater();
1539	_audioInput = NULL;	1551	_audioInput = NULL;
1540	_numInputCallbackBytes = 0;	1552	_numInputCallbackBytes = 0;
1541		1553
1542	_inputDeviceInfo = QAudioDeviceInfo();	1554	_inputDeviceInfo = QAudioDeviceInfo();
1543	}	1555	}
1544		1556
1545	if (_dummyAudioInput) {	1557	if (_dummyAudioInput) {
1546	_dummyAudioInput->stop();	1558	_dummyAudioInput->stop();
1547		1559
1548	_dummyAudioInput->deleteLater();	1560	_dummyAudioInput->deleteLater();
1549	_dummyAudioInput = NULL;	1561	_dummyAudioInput = NULL;
1550	}	1562	}
1551		1563
1552	if (_inputToNetworkResampler) {	1564	if (_inputToNetworkResampler) {
1553	// if we were using an input to network resampler, delete it here	1565	// if we were using an input to network resampler, delete it here
1554	delete _inputToNetworkResampler;	1566	delete _inputToNetworkResampler;
1555	_inputToNetworkResampler = NULL;	1567	_inputToNetworkResampler = NULL;
1556	}	1568	}
1557		1569
1558	if (_audioGate) {	1570	if (_audioGate) {
1559	delete _audioGate;	1571	delete _audioGate;
1560	_audioGate = nullptr;	1572	_audioGate = nullptr;
1561	}	1573	}
1562		1574
1563	if (isShutdownRequest) {	1575	if (isShutdownRequest) {
1564	qCDebug(audioclient) << "The audio input device has shut down.";	1576	qCDebug(audioclient) << "The audio input device has shut down.";
1565	return true;	1577	return true;
1566	}	1578	}
1567		1579
1568	if (!inputDeviceInfo.isNull()) {	1580	if (!inputDeviceInfo.isNull()) {
1569	qCDebug(audioclient) << "The audio input device " << inputDeviceInfo.deviceName() << "is available.";	1581	qCDebug(audioclient) << "The audio input device " << inputDeviceInfo.deviceName() << "is available.";
1570	_inputDeviceInfo = inputDeviceInfo;	1582	_inputDeviceInfo = inputDeviceInfo;
1571	emit deviceChanged(QAudio::AudioInput, inputDeviceInfo);	1583	emit deviceChanged(QAudio::AudioInput, inputDeviceInfo);
1572		1584
1573	if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {	1585	if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {
1574	qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;	1586	qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
1575		1587
1576	// we've got the best we can get for input	1588	// we've got the best we can get for input
1577	// if required, setup a resampler for this input to our desired network format	1589	// if required, setup a resampler for this input to our desired network format
1578	if (_inputFormat != _desiredInputFormat && _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) {	1590	if (_inputFormat != _desiredInputFormat && _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) {
1579	qCDebug(audioclient) << "Attemping to create a resampler for input format to network format.";	1591	qCDebug(audioclient) << "Attemping to create a resampler for input format to network format.";
1580		1592
1581	assert(_inputFormat.sampleSize() == 16);	1593	assert(_inputFormat.sampleSize() == 16);
1582	assert(_desiredInputFormat.sampleSize() == 16);	1594	assert(_desiredInputFormat.sampleSize() == 16);
1583	int channelCount = (_inputFormat.channelCount() == 2 && _desiredInputFormat.channelCount() == 2) ? 2 : 1;	1595	int channelCount = (_inputFormat.channelCount() == 2 && _desiredInputFormat.channelCount() == 2) ? 2 : 1;
1584		1596
1585	_inputToNetworkResampler =	1597	_inputToNetworkResampler =
1586	new AudioSRC(_inputFormat.sampleRate(), _desiredInputFormat.sampleRate(), channelCount);	1598	new AudioSRC(_inputFormat.sampleRate(), _desiredInputFormat.sampleRate(), channelCount);
1587		1599
1588	} else {	1600	} else {
1589	qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";	1601	qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
1590	}	1602	}
1591		1603
1592	// the audio gate runs after the resampler	1604	// the audio gate runs after the resampler
1593	_audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());	1605	_audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());
1594	qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";	1606	qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";
1595		1607
1596	// if the user wants stereo but this device can't provide then bail	1608	// if the user wants stereo but this device can't provide then bail
1597	if (!_isStereoInput \|\| _inputFormat.channelCount() == 2) {	1609	if (!_isStereoInput \|\| _inputFormat.channelCount() == 2) {
1598	_audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);	1610	_audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);
1599	_numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat);	1611	_numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat);
1600	_audioInput->setBufferSize(_numInputCallbackBytes);	1612	_audioInput->setBufferSize(_numInputCallbackBytes);
1601	// different audio input devices may have different volumes	1613	// different audio input devices may have different volumes
1602	emit inputVolumeChanged(_audioInput->volume());	1614	emit inputVolumeChanged(_audioInput->volume());
1603		1615
1604	// how do we want to handle input working, but output not working?	1616	// how do we want to handle input working, but output not working?
1605	int numFrameSamples = calculateNumberOfFrameSamples(_numInputCallbackBytes);	1617	int numFrameSamples = calculateNumberOfFrameSamples(_numInputCallbackBytes);
1606	_inputRingBuffer.resizeForFrameSize(numFrameSamples);	1618	_inputRingBuffer.resizeForFrameSize(numFrameSamples);
1607		1619
1608	#if defined(Q_OS_ANDROID)	1620	#if defined(Q_OS_ANDROID)
1609	if (_audioInput) {	1621	if (_audioInput) {
1610	_shouldRestartInputSetup = true;	1622	_shouldRestartInputSetup = true;
1611	connect(_audioInput, &QAudioInput::stateChanged, this, &AudioClient::audioInputStateChanged);	1623	connect(_audioInput, &QAudioInput::stateChanged, this, &AudioClient::audioInputStateChanged);
1612	}	1624	}
1613	#endif	1625	#endif
1614	_inputDevice = _audioInput->start();	1626	_inputDevice = _audioInput->start();
1615		1627
1616	if (_inputDevice) {	1628	if (_inputDevice) {
1617	connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));	1629	connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
1618	supportedFormat = true;	1630	supportedFormat = true;
1619	} else {	1631	} else {
1620	qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error();	1632	qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error();
1621	_audioInput->deleteLater();	1633	_audioInput->deleteLater();
1622	_audioInput = NULL;	1634	_audioInput = NULL;
		1635	}
1623	}	1636	}
1624	}	1637	}
1625	}	1638	}
1626	}
1627		1639
1628	// If there is no working input device, use the dummy input device.	1640	// If there is no working input device, use the dummy input device.
1629	// It generates audio callbacks on a timer to simulate a mic stream of silent packets.	1641	// It generates audio callbacks on a timer to simulate a mic stream of silent packets.
1630	// This enables clients without a mic to still receive an audio stream from the mixer.	1642	// This enables clients without a mic to still receive an audio stream from the mixer.
1631	if (!_audioInput) {	1643	if (!_audioInput) {
1632	qCDebug(audioclient) << "Audio input device is not available, using dummy input.";	1644	qCDebug(audioclient) << "Audio input device is not available, using dummy input.";
1633	_inputDeviceInfo = QAudioDeviceInfo();	1645	_inputDeviceInfo = QAudioDeviceInfo();
1634	emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo);	1646	emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo);
1635		1647
1636	_inputFormat = _desiredInputFormat;	1648	_inputFormat = _desiredInputFormat;
1637	qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;	1649	qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
1638	qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";	1650	qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
1639		1651
1640	_audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());	1652	_audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());
1641	qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";	1653	qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";
1642		1654
1643	// generate audio callbacks at the network sample rate	1655	// generate audio callbacks at the network sample rate
1644	_dummyAudioInput = new QTimer(this);	1656	_dummyAudioInput = new QTimer(this);
1645	connect(_dummyAudioInput, SIGNAL(timeout()), this, SLOT(handleDummyAudioInput()));	1657	connect(_dummyAudioInput, SIGNAL(timeout()), this, SLOT(handleDummyAudioInput()));
1646	_dummyAudioInput->start((int)(AudioConstants::NETWORK_FRAME_MSECS + 0.5f));	1658	_dummyAudioInput->start((int)(AudioConstants::NETWORK_FRAME_MSECS + 0.5f));
1647	}	1659	}
1648		1660
1649	return supportedFormat;	1661	return supportedFormat;
1650	}	1662	}
1651		1663
1652	void AudioClient::audioInputStateChanged(QAudio::State state) {	1664	void AudioClient::audioInputStateChanged(QAudio::State state) {
1653	#if defined(Q_OS_ANDROID)	1665	#if defined(Q_OS_ANDROID)
1654	switch (state) {	1666	switch (state) {
1655	case QAudio::StoppedState:	1667	case QAudio::StoppedState:
1656	if (!_audioInput) {	1668	if (!_audioInput) {
1657	break;	1669	break;
1658	}
1659	// Stopped on purpose
1660	if (_shouldRestartInputSetup) {
1661	Lock lock(_deviceMutex);
1662	_inputDevice = _audioInput->start();
1663	lock.unlock();
1664	if (_inputDevice) {
1665	connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
1666	}	1670	}
1667	}	1671	// Stopped on purpose
1668	break;	1672	if (_shouldRestartInputSetup) {
1669	case QAudio::ActiveState:	1673	Lock lock(_deviceMutex);
1670	break;	1674	_inputDevice = _audioInput->start();
1671	default:	1675	lock.unlock();
1672	break;	1676	if (_inputDevice) {
1673	}	1677	connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
		1678	}
		1679	}
		1680	break;
		1681	case QAudio::ActiveState:
		1682	break;
		1683	default:
		1684	break;
		1685	}
1674	#endif	1686	#endif
1675	}	1687	}
1676		1688
1677	void AudioClient::checkInputTimeout() {	1689	void AudioClient::checkInputTimeout() {
1678	#if defined(Q_OS_ANDROID)	1690	#if defined(Q_OS_ANDROID)
1679	if (_audioInput && _inputReadsSinceLastCheck < MIN_READS_TO_CONSIDER_INPUT_ALIVE) {	1691	if (_audioInput && _inputReadsSinceLastCheck < MIN_READS_TO_CONSIDER_INPUT_ALIVE) {
1680	_audioInput->stop();	1692	_audioInput->stop();
1681	} else {	1693	} else {
1682	_inputReadsSinceLastCheck = 0;	1694	_inputReadsSinceLastCheck = 0;
1683	}	1695	}
1684	#endif	1696	#endif
1685	}	1697	}
1686		1698
1687	void AudioClient::setHeadsetPluggedIn(bool pluggedIn) {	1699	void AudioClient::setHeadsetPluggedIn(bool pluggedIn) {
1688	#if defined(Q_OS_ANDROID)	1700	#if defined(Q_OS_ANDROID)
1689	if (pluggedIn == !_isHeadsetPluggedIn && !_inputDeviceInfo.isNull()) {	1701	if (pluggedIn == !_isHeadsetPluggedIn && !_inputDeviceInfo.isNull()) {
1690	QAndroidJniObject brand = QAndroidJniObject::getStaticObjectField<jstring>("android/os/Build", "BRAND");	1702	QAndroidJniObject brand = QAndroidJniObject::getStaticObjectField<jstring>("android/os/Build", "BRAND");
1691	// some samsung phones needs more time to shutdown the previous input device	1703	// some samsung phones needs more time to shutdown the previous input device
1692	if (brand.toString().contains("samsung", Qt::CaseInsensitive)) {	1704	if (brand.toString().contains("samsung", Qt::CaseInsensitive)) {
1693	switchInputToAudioDevice(QAudioDeviceInfo(), true);	1705	switchInputToAudioDevice(QAudioDeviceInfo(), true);
1694	QThread::msleep(200);	1706	QThread::msleep(200);
1695	}	1707	}
1696		1708
1697	Setting::Handle<bool> enableAEC(SETTING_AEC_KEY, false);	1709	Setting::Handle<bool> enableAEC(SETTING_AEC_KEY, false);
1698	bool aecEnabled = enableAEC.get();	1710	bool aecEnabled = enableAEC.get();
1699		1711
1700	if ((pluggedIn \|\| !aecEnabled) && _inputDeviceInfo.deviceName() != VOICE_RECOGNITION) {	1712	if ((pluggedIn \|\| !aecEnabled) && _inputDeviceInfo.deviceName() != VOICE_RECOGNITION) {
1701	switchAudioDevice(QAudio::AudioInput, VOICE_RECOGNITION);	1713	switchAudioDevice(QAudio::AudioInput, VOICE_RECOGNITION);
1702	} else if (!pluggedIn && aecEnabled && _inputDeviceInfo.deviceName() != VOICE_COMMUNICATION) {	1714	} else if (!pluggedIn && aecEnabled && _inputDeviceInfo.deviceName() != VOICE_COMMUNICATION) {
1703	switchAudioDevice(QAudio::AudioInput, VOICE_COMMUNICATION);	1715	switchAudioDevice(QAudio::AudioInput, VOICE_COMMUNICATION);
		1716	}
1704	}	1717	}
1705	}	1718	_isHeadsetPluggedIn = pluggedIn;
1706	_isHeadsetPluggedIn = pluggedIn;
1707	#endif	1719	#endif
1708	}	1720	}
1709		1721
1710	void AudioClient::outputNotify() {	1722	void AudioClient::outputNotify() {
1711	int recentUnfulfilled = _audioOutputIODevice.getRecentUnfulfilledReads();	1723	int recentUnfulfilled = _audioOutputIODevice.getRecentUnfulfilledReads();
1712	if (recentUnfulfilled > 0) {	1724	if (recentUnfulfilled > 0) {
1713	qCDebug(audioclient, "Starve detected, %d new unfulfilled reads", recentUnfulfilled);	1725	qCDebug(audioclient, "Starve detected, %d new unfulfilled reads", recentUnfulfilled);
1714
1715	if (_outputStarveDetectionEnabled.get()) {
1716	quint64 now = usecTimestampNow() / 1000;
1717	int dt = (int)(now - _outputStarveDetectionStartTimeMsec);
1718	if (dt > STARVE_DETECTION_PERIOD) {
1719	_outputStarveDetectionStartTimeMsec = now;
1720	_outputStarveDetectionCount = 0;
1721	} else {
1722	_outputStarveDetectionCount += recentUnfulfilled;
1723	if (_outputStarveDetectionCount > STARVE_DETECTION_THRESHOLD) {
1724	int oldOutputBufferSizeFrames = _sessionOutputBufferSizeFrames;
1725	int newOutputBufferSizeFrames = setOutputBufferSize(oldOutputBufferSizeFrames + 1, false);
1726
1727	if (newOutputBufferSizeFrames > oldOutputBufferSizeFrames) {
1728	qCDebug(audioclient, "Starve threshold surpassed (%d starves in %d ms)", _outputStarveDetectionCount,
1729	dt);
1730	}
1731		1726
		1727	if (_outputStarveDetectionEnabled.get()) {
		1728	quint64 now = usecTimestampNow() / 1000;
		1729	int dt = (int)(now - _outputStarveDetectionStartTimeMsec);
		1730	if (dt > STARVE_DETECTION_PERIOD) {
1732	_outputStarveDetectionStartTimeMsec = now;	1731	_outputStarveDetectionStartTimeMsec = now;
1733	_outputStarveDetectionCount = 0;	1732	_outputStarveDetectionCount = 0;
		1733	} else {
		1734	_outputStarveDetectionCount += recentUnfulfilled;
		1735	if (_outputStarveDetectionCount > STARVE_DETECTION_THRESHOLD) {
		1736	int oldOutputBufferSizeFrames = _sessionOutputBufferSizeFrames;
		1737	int newOutputBufferSizeFrames = setOutputBufferSize(oldOutputBufferSizeFrames + 1, false);
		1738
		1739	if (newOutputBufferSizeFrames > oldOutputBufferSizeFrames) {
		1740	qCDebug(audioclient, "Starve threshold surpassed (%d starves in %d ms)",
		1741	_outputStarveDetectionCount, dt);
		1742	}
		1743
		1744	_outputStarveDetectionStartTimeMsec = now;
		1745	_outputStarveDetectionCount = 0;
		1746	}
1734	}	1747	}
1735	}	1748	}
1736	}	1749	}
1737	}	1750	}
1738	}
1739		1751
1740	bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) {	1752	bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) {
1741	Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");	1753	Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");
1742		1754
1743	qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName()	1755	qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName()
1744	<< "]";	1756	<< "]";
1745	bool supportedFormat = false;	1757	bool supportedFormat = false;
1746
1747	// NOTE: device start() uses the Qt internal device list
1748	Lock lock(_deviceMutex);
1749		1758
1750	Lock localAudioLock(_localAudioMutex);	1759	// NOTE: device start() uses the Qt internal device list
1751	_localSamplesAvailable.exchange(0, std::memory_order_release);	1760	Lock lock(_deviceMutex);
1752		1761
1753	// cleanup any previously initialized device	1762	Lock localAudioLock(_localAudioMutex);
1754	if (_audioOutput) {	1763	_localSamplesAvailable.exchange(0, std::memory_order_release);
1755	_audioOutputIODevice.close();
1756	_audioOutput->stop();
1757		1764
1758	//must be deleted in next eventloop cycle when its called from notify()	1765	// cleanup any previously initialized device
1759	_audioOutput->deleteLater();	1766	if (_audioOutput) {
1760	_audioOutput = NULL;	1767	_audioOutputIODevice.close();
		1768	_audioOutput->stop();
1761		1769
1762	_loopbackOutputDevice = NULL;	1770	//must be deleted in next eventloop cycle when its called from notify()
1763	//must be deleted in next eventloop cycle when its called from notify()	1771	_audioOutput->deleteLater();
1764	_loopbackAudioOutput->deleteLater();	1772	_audioOutput = NULL;
1765	_loopbackAudioOutput = NULL;
1766		1773
1767	delete[] _outputMixBuffer;	1774	_loopbackOutputDevice = NULL;
1768	_outputMixBuffer = NULL;	1775	//must be deleted in next eventloop cycle when its called from notify()
		1776	_loopbackAudioOutput->deleteLater();
		1777	_loopbackAudioOutput = NULL;
1769		1778
1770	delete[] _outputScratchBuffer;	1779	delete[] _outputMixBuffer;
1771	_outputScratchBuffer = NULL;	1780	_outputMixBuffer = NULL;
1772		1781
1773	delete[] _localOutputMixBuffer;	1782	delete[] _outputScratchBuffer;
1774	_localOutputMixBuffer = NULL;	1783	_outputScratchBuffer = NULL;
1775		1784
1776	_outputDeviceInfo = QAudioDeviceInfo();	1785	delete[] _localOutputMixBuffer;
1777	}	1786	_localOutputMixBuffer = NULL;
1778		1787
1779	if (_networkToOutputResampler) {	1788	_outputDeviceInfo = QAudioDeviceInfo();
1780	// if we were using an input to network resampler, delete it here	1789	}
1781	delete _networkToOutputResampler;
1782	_networkToOutputResampler = NULL;
1783		1790
1784	delete _localToOutputResampler;	1791	if (_networkToOutputResampler) {
1785	_localToOutputResampler = NULL;	1792	// if we were using an input to network resampler, delete it here
1786	}	1793	delete _networkToOutputResampler;
		1794	_networkToOutputResampler = NULL;
1787		1795
1788	if (isShutdownRequest) {	1796	delete _localToOutputResampler;
1789	qCDebug(audioclient) << "The audio output device has shut down.";	1797	_localToOutputResampler = NULL;
1790	return true;	1798	}
1791	}
1792		1799
1793	if (!outputDeviceInfo.isNull()) {	1800	if (isShutdownRequest) {
1794	qCDebug(audioclient) << "The audio output device " << outputDeviceInfo.deviceName() << "is available.";	1801	qCDebug(audioclient) << "The audio output device has shut down.";
1795	_outputDeviceInfo = outputDeviceInfo;	1802	return true;
1796	emit deviceChanged(QAudio::AudioOutput, outputDeviceInfo);	1803	}
1797		1804
1798	if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {	1805	if (!outputDeviceInfo.isNull()) {
1799	qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat;	1806	qCDebug(audioclient) << "The audio output device " << outputDeviceInfo.deviceName() << "is available.";
		1807	_outputDeviceInfo = outputDeviceInfo;
		1808	emit deviceChanged(QAudio::AudioOutput, outputDeviceInfo);
1800		1809
1801	// we've got the best we can get for input	1810	if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {
1802	// if required, setup a resampler for this input to our desired network format	1811	qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat;
1803	if (_desiredOutputFormat != _outputFormat && _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) {
1804	qCDebug(audioclient) << "Attemping to create a resampler for network format to output format.";
1805		1812
1806	assert(_desiredOutputFormat.sampleSize() == 16);	1813	// we've got the best we can get for input
1807	assert(_outputFormat.sampleSize() == 16);	1814	// if required, setup a resampler for this input to our desired network format
		1815	if (_desiredOutputFormat != _outputFormat && _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) {
		1816	qCDebug(audioclient) << "Attemping to create a resampler for network format to output format.";
1808		1817
1809	_networkToOutputResampler =	1818	assert(_desiredOutputFormat.sampleSize() == 16);
1810	new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);	1819	assert(_outputFormat.sampleSize() == 16);
1811	_localToOutputResampler =
1812	new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
1813		1820
1814	} else {	1821	_networkToOutputResampler =
1815	qCDebug(audioclient) << "No resampling required for network output to match actual output format.";	1822	new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
1816	}	1823	_localToOutputResampler =
		1824	new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
1817		1825
1818	outputFormatChanged();	1826	} else {
1819		1827	qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
1820	// setup our general output device for audio-mixer audio
1821	_audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
1822
1823	int deviceChannelCount = _outputFormat.channelCount();
1824	int frameSize =
1825	(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * deviceChannelCount * _outputFormat.sampleRate()) /
1826	_desiredOutputFormat.sampleRate();
1827	int requestedSize = _sessionOutputBufferSizeFrames * frameSize * AudioConstants::SAMPLE_SIZE;
1828	_audioOutput->setBufferSize(requestedSize);
1829
1830	// initialize mix buffers on the _audioOutput thread to avoid races
1831	connect(_audioOutput, &QAudioOutput::stateChanged, [&, frameSize, requestedSize](QAudio::State state) {
1832	if (state == QAudio::ActiveState) {
1833	// restrict device callback to _outputPeriod samples
1834	_outputPeriod = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE;
1835	// device callback may exceed reported period, so double it to avoid stutter
1836	_outputPeriod *= 2;
1837
1838	_outputMixBuffer = new float[_outputPeriod];
1839	_outputScratchBuffer = new int16_t[_outputPeriod];
1840
1841	// size local output mix buffer based on resampled network frame size
1842	int networkPeriod =
1843	_localToOutputResampler
1844	? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO)
1845	: AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
1846	_localOutputMixBuffer = new float[networkPeriod];
1847
1848	// local period should be at least twice the output period,
1849	// in case two device reads happen before more data can be read (worst case)
1850	int localPeriod = _outputPeriod * 2;
1851	// round up to an exact multiple of networkPeriod
1852	localPeriod = ((localPeriod + networkPeriod - 1) / networkPeriod) * networkPeriod;
1853	// this ensures lowest latency without stutter from underrun
1854	_localInjectorsStream.resizeForFrameSize(localPeriod);
1855
1856	int bufferSize = _audioOutput->bufferSize();
1857	int bufferSamples = bufferSize / AudioConstants::SAMPLE_SIZE;
1858	int bufferFrames = bufferSamples / (float)frameSize;
1859	qCDebug(audioclient) << "frame (samples):" << frameSize;
1860	qCDebug(audioclient) << "buffer (frames):" << bufferFrames;
1861	qCDebug(audioclient) << "buffer (samples):" << bufferSamples;
1862	qCDebug(audioclient) << "buffer (bytes):" << bufferSize;
1863	qCDebug(audioclient) << "requested (bytes):" << requestedSize;
1864	qCDebug(audioclient) << "period (samples):" << _outputPeriod;
1865	qCDebug(audioclient) << "local buffer (samples):" << localPeriod;
1866
1867	disconnect(_audioOutput, &QAudioOutput::stateChanged, 0, 0);
1868
1869	// unlock to avoid a deadlock with the device callback (which always succeeds this initialization)
1870	localAudioLock.unlock();
1871	}	1828	}
1872	});
1873	connect(_audioOutput, &QAudioOutput::notify, this, &AudioClient::outputNotify);
1874		1829
1875	_audioOutputIODevice.start();	1830	outputFormatChanged();
		1831
		1832	// setup our general output device for audio-mixer audio
		1833	_audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
		1834
		1835	int deviceChannelCount = _outputFormat.channelCount();
		1836	int frameSize =
		1837	(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * deviceChannelCount * _outputFormat.sampleRate()) /
		1838	_desiredOutputFormat.sampleRate();
		1839	int requestedSize = _sessionOutputBufferSizeFrames * frameSize * AudioConstants::SAMPLE_SIZE;
		1840	_audioOutput->setBufferSize(requestedSize);
		1841
		1842	// initialize mix buffers on the _audioOutput thread to avoid races
		1843	connect(_audioOutput, &QAudioOutput::stateChanged, [&, frameSize, requestedSize](QAudio::State state) {
		1844	if (state == QAudio::ActiveState) {
		1845	// restrict device callback to _outputPeriod samples
		1846	_outputPeriod = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE;
		1847	// device callback may exceed reported period, so double it to avoid stutter
		1848	_outputPeriod *= 2;
		1849
		1850	_outputMixBuffer = new float[_outputPeriod];
		1851	_outputScratchBuffer = new int16_t[_outputPeriod];
		1852
		1853	// size local output mix buffer based on resampled network frame size
		1854	int networkPeriod =
		1855	_localToOutputResampler
		1856	? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO)
		1857	: AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
		1858	_localOutputMixBuffer = new float[networkPeriod];
		1859
		1860	// local period should be at least twice the output period,
		1861	// in case two device reads happen before more data can be read (worst case)
		1862	int localPeriod = _outputPeriod * 2;
		1863	// round up to an exact multiple of networkPeriod
		1864	localPeriod = ((localPeriod + networkPeriod - 1) / networkPeriod) * networkPeriod;
		1865	// this ensures lowest latency without stutter from underrun
		1866	_localInjectorsStream.resizeForFrameSize(localPeriod);
		1867
		1868	int bufferSize = _audioOutput->bufferSize();
		1869	int bufferSamples = bufferSize / AudioConstants::SAMPLE_SIZE;
		1870	int bufferFrames = bufferSamples / (float)frameSize;
		1871	qCDebug(audioclient) << "frame (samples):" << frameSize;
		1872	qCDebug(audioclient) << "buffer (frames):" << bufferFrames;
		1873	qCDebug(audioclient) << "buffer (samples):" << bufferSamples;
		1874	qCDebug(audioclient) << "buffer (bytes):" << bufferSize;
		1875	qCDebug(audioclient) << "requested (bytes):" << requestedSize;
		1876	qCDebug(audioclient) << "period (samples):" << _outputPeriod;
		1877	qCDebug(audioclient) << "local buffer (samples):" << localPeriod;
		1878
		1879	disconnect(_audioOutput, &QAudioOutput::stateChanged, 0, 0);
		1880
		1881	// unlock to avoid a deadlock with the device callback (which always succeeds this initialization)
		1882	localAudioLock.unlock();
		1883	}
		1884	});
		1885	connect(_audioOutput, &QAudioOutput::notify, this, &AudioClient::outputNotify);
		1886
		1887	_audioOutputIODevice.start();
1876		1888
1877	_audioOutput->start(&_audioOutputIODevice);	1889	_audioOutput->start(&_audioOutputIODevice);
1878		1890
1879	// setup a loopback audio output device	1891	// setup a loopback audio output device
1880	_loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);	1892	_loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
1881		1893
1882	_timeSinceLastReceived.start();	1894	_timeSinceLastReceived.start();
1883		1895
1884	supportedFormat = true;	1896	supportedFormat = true;
		1897	}
1885	}	1898	}
1886	}
1887		1899
1888	return supportedFormat;	1900	return supportedFormat;
1889	}	1901	}
1890		1902
1891	int AudioClient::setOutputBufferSize(int numFrames, bool persist) {	1903	int AudioClient::setOutputBufferSize(int numFrames, bool persist) {
1892	qCDebug(audioclient) << __FUNCTION__ << "numFrames:" << numFrames << "persist:" << persist;	1904	qCDebug(audioclient) << __FUNCTION__ << "numFrames:" << numFrames << "persist:" << persist;
1893		1905
1894	numFrames = std::min(std::max(numFrames, MIN_BUFFER_FRAMES), MAX_BUFFER_FRAMES);	1906	numFrames = std::min(std::max(numFrames, MIN_BUFFER_FRAMES), MAX_BUFFER_FRAMES);
1895	qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames	1907	qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames
1896	<< "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames;	1908	<< "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames;
1897		1909
1898	if (numFrames != _sessionOutputBufferSizeFrames) {	1910	if (numFrames != _sessionOutputBufferSizeFrames) {
1899	qCInfo(audioclient, "Audio output buffer set to %d frames", numFrames);	1911	qCInfo(audioclient, "Audio output buffer set to %d frames", numFrames);
1900	_sessionOutputBufferSizeFrames = numFrames;	1912	_sessionOutputBufferSizeFrames = numFrames;
1901	if (persist) {	1913	if (persist) {
1902	_outputBufferSizeFrames.set(numFrames);	1914	_outputBufferSizeFrames.set(numFrames);
		1915	}
1903	}	1916	}
		1917	return numFrames;
1904	}	1918	}
1905	return numFrames;
1906	}
1907		1919
1908	// The following constant is operating system dependent due to differences in	1920	// The following constant is operating system dependent due to differences in
1909	// the way input audio is handled. The audio input buffer size is inversely	1921	// the way input audio is handled. The audio input buffer size is inversely
1910	// proportional to the accelerator ratio.	1922	// proportional to the accelerator ratio.
1911		1923
1912	#ifdef Q_OS_WIN	1924	#ifdef Q_OS_WIN
1913	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = IsWindows8OrGreater() ? 1.0f : 0.25f;	1925	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = IsWindows8OrGreater() ? 1.0f : 0.25f;
1914	#endif	1926	#endif
1915		1927
1916	#ifdef Q_OS_MAC	1928	#ifdef Q_OS_MAC
1917	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;	1929	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;
1918	#endif	1930	#endif
1919		1931
1920	#ifdef Q_OS_ANDROID	1932	#ifdef Q_OS_ANDROID
1921	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 0.5f;	1933	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 0.5f;
1922	#elif defined(Q_OS_LINUX)	1934	#elif defined(Q_OS_LINUX)
1923	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;	1935	const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;
1924	#endif	1936	#endif
1925		1937
1926	int AudioClient::calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const {	1938	int AudioClient::calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const {
1927	int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * format.channelCount() *	1939	int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * format.channelCount() *
1928	((float)format.sampleRate() / AudioConstants::SAMPLE_RATE)) /	1940	((float)format.sampleRate() / AudioConstants::SAMPLE_RATE)) /
1929	CALLBACK_ACCELERATOR_RATIO) +	1941	CALLBACK_ACCELERATOR_RATIO) +
1930	0.5f);	1942	0.5f);
1931		1943
1932	return numInputCallbackBytes;	1944	return numInputCallbackBytes;
1933	}	1945	}
1934		1946
1935	int AudioClient::calculateNumberOfFrameSamples(int numBytes) const {	1947	int AudioClient::calculateNumberOfFrameSamples(int numBytes) const {
1936	int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / AudioConstants::SAMPLE_SIZE;	1948	int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / AudioConstants::SAMPLE_SIZE;
1937	return frameSamples;	1949	return frameSamples;
1938	}	1950	}
1939		1951
1940	float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {	1952	float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
1941	glm::quat inverseOrientation = glm::inverse(_orientationGetter());	1953	glm::quat inverseOrientation = glm::inverse(_orientationGetter());
1942		1954
1943	glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;	1955	glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;
1944		1956
1945	// project the rotated source position vector onto the XZ plane	1957	// project the rotated source position vector onto the XZ plane
1946	rotatedSourcePosition.y = 0.0f;	1958	rotatedSourcePosition.y = 0.0f;
1947		1959
1948	static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;	1960	static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
1949		1961
1950	float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition);	1962	float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition);
1951	if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) {	1963	if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) {
1952	// produce an oriented angle about the y-axis	1964	// produce an oriented angle about the y-axis
1953	glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2));	1965	glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2));
1954	float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"	1966	float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"
1955	return (direction.x < 0.0f) ? -angle : angle;	1967	return (direction.x < 0.0f) ? -angle : angle;
1956		1968
1957	} else {	1969	} else {
1958	// no azimuth if they are in same spot	1970	// no azimuth if they are in same spot
1959	return 0.0f;	1971	return 0.0f;
		1972	}
1960	}	1973	}
1961	}
1962		1974
1963	float AudioClient::gainForSource(float distance, float volume) {	1975	float AudioClient::gainForSource(float distance, float volume) {
1964	// attenuation = -6dB * log2(distance)	1976	// attenuation = -6dB * log2(distance)
1965	// reference attenuation of 0dB at distance = 1.0m	1977	// reference attenuation of 0dB at distance = 1.0m
1966	float gain = volume / std::max(distance, HRTF_NEARFIELD_MIN);	1978	float gain = volume / std::max(distance, HRTF_NEARFIELD_MIN);
1967		1979
1968	return gain;	1980	return gain;
1969	}	1981	}
1970		1982
1971	qint64 AudioClient::AudioOutputIODevice::readData(char* data, qint64 maxSize) {	1983	qint64 AudioClient::AudioOutputIODevice::readData(char* data, qint64 maxSize) {
1972	// samples requested from OUTPUT_CHANNEL_COUNT	1984	// samples requested from OUTPUT_CHANNEL_COUNT
1973	int deviceChannelCount = _audio->_outputFormat.channelCount();	1985	int deviceChannelCount = _audio->_outputFormat.channelCount();
1974	int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;	1986	int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
1975	// restrict samplesRequested to the size of our mix/scratch buffers	1987	// restrict samplesRequested to the size of our mix/scratch buffers
1976	samplesRequested = std::min(samplesRequested, _audio->_outputPeriod);	1988	samplesRequested = std::min(samplesRequested, _audio->_outputPeriod);
1977		1989
1978	int16_t* scratchBuffer = _audio->_outputScratchBuffer;	1990	int16_t* scratchBuffer = _audio->_outputScratchBuffer;
1979	float* mixBuffer = _audio->_outputMixBuffer;	1991	float* mixBuffer = _audio->_outputMixBuffer;
1980		1992
1981	int networkSamplesPopped;	1993	int networkSamplesPopped;
1982	if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {	1994	if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
1983	qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped,	1995	qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped,
1984	_receivedAudioStream.getSamplesAvailable(), samplesRequested);	1996	_receivedAudioStream.getSamplesAvailable(), samplesRequested);
1985	AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();	1997	AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
1986	lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped);	1998	lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped);
1987	for (int i = 0; i < networkSamplesPopped; i++) {	1999	for (int i = 0; i < networkSamplesPopped; i++) {
1988	mixBuffer[i] = convertToFloat(scratchBuffer[i]);	2000	mixBuffer[i] = convertToFloat(scratchBuffer[i]);
		2001	}
		2002	samplesRequested = networkSamplesPopped;
1989	}	2003	}
1990	samplesRequested = networkSamplesPopped;
1991	}
1992		2004
1993	int injectorSamplesPopped = 0;	2005	int injectorSamplesPopped = 0;
1994	{	2006	{
1995	bool append = networkSamplesPopped > 0;	2007	bool append = networkSamplesPopped > 0;
1996	// check the samples we have available locklessly; this is possible because only two functions add to the count:	2008	// check the samples we have available locklessly; this is possible because only two functions add to the count:
1997	// - prepareLocalAudioInjectors will only increase samples count	2009	// - prepareLocalAudioInjectors will only increase samples count
1998	// - switchOutputToAudioDevice will zero samples count,	2010	// - switchOutputToAudioDevice will zero samples count,
1999	// stop the device - so that readData will exhaust the existing buffer or see a zeroed samples count,	2011	// stop the device - so that readData will exhaust the existing buffer or see a zeroed samples count,
2000	// and start the device - which can then only see a zeroed samples count	2012	// and start the device - which can then only see a zeroed samples count
2001	int samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);	2013	int samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);
2002		2014
2003	// if we do not have enough samples buffered despite having injectors, buffer them synchronously	2015	// if we do not have enough samples buffered despite having injectors, buffer them synchronously
2004	if (samplesAvailable < samplesRequested && _audio->_localInjectorsAvailable.load(std::memory_order_acquire)) {	2016	if (samplesAvailable < samplesRequested && _audio->_localInjectorsAvailable.load(std::memory_order_acquire)) {
2005	// try_to_lock, in case the device is being shut down already	2017	// try_to_lock, in case the device is being shut down already
2006	std::unique_ptr<Lock> localAudioLock(new Lock(_audio->_localAudioMutex, std::try_to_lock));	2018	std::unique_ptr<Lock> localAudioLock(new Lock(_audio->_localAudioMutex, std::try_to_lock));
2007	if (localAudioLock->owns_lock()) {	2019	if (localAudioLock->owns_lock()) {
2008	_audio->prepareLocalAudioInjectors(std::move(localAudioLock));	2020	_audio->prepareLocalAudioInjectors(std::move(localAudioLock));
2009	samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);	2021	samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);
		2022	}
2010	}	2023	}
2011	}
2012		2024
2013	samplesRequested = std::min(samplesRequested, samplesAvailable);	2025	samplesRequested = std::min(samplesRequested, samplesAvailable);
2014	if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {	2026	if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {
2015	_audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release);	2027	_audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release);
2016	qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped,	2028	qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped,
2017	_localInjectorsStream.samplesAvailable(), samplesRequested);	2029	_localInjectorsStream.samplesAvailable(), samplesRequested);
		2030	}
2018	}	2031	}
2019	}
2020		2032
2021	// prepare injectors for the next callback	2033	// prepare injectors for the next callback
2022	QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); });	2034	QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); });
2023
2024	int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
2025	int framesPopped = samplesPopped / AudioConstants::STEREO;
2026	int bytesWritten;
2027	if (samplesPopped > 0) {
2028	if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
2029	// limit the audio
2030	_audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped);
2031	} else {
2032	_audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
2033		2035
2034	// upmix or downmix to deviceChannelCount	2036	int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
2035	if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {	2037	int framesPopped = samplesPopped / AudioConstants::STEREO;
2036	int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;	2038	int bytesWritten;
2037	channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);	2039	if (samplesPopped > 0) {
		2040	if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
		2041	// limit the audio
		2042	_audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped);
2038	} else {	2043	} else {
2039	channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);	2044	_audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
		2045
		2046	// upmix or downmix to deviceChannelCount
		2047	if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
		2048	int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;
		2049	channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);
		2050	} else {
		2051	channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);
		2052	}
2040	}	2053	}
		2054
		2055	bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
		2056	} else {
		2057	// nothing on network, don't grab anything from injectors, and just return 0s
		2058	memset(data, 0, maxSize);
		2059	bytesWritten = maxSize;
2041	}	2060	}
2042		2061
2043	bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;	2062	// send output buffer for recording
2044	} else {	2063	if (_audio->_isRecording) {
2045	// nothing on network, don't grab anything from injectors, and just return 0s	2064	Lock lock(_recordMutex);
2046	memset(data, 0, maxSize);	2065	_audio->_audioFileWav.addRawAudioChunk(reinterpret_cast<char*>(scratchBuffer), bytesWritten);
2047	bytesWritten = maxSize;	2066	}
2048	}
2049		2067
2050	// send output buffer for recording	2068	int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
2051	if (_audio->_isRecording) {	2069	float msecsAudioOutputUnplayed =
2052	Lock lock(_recordMutex);	2070	bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC);
2053	_audio->_audioFileWav.addRawAudioChunk(reinterpret_cast<char*>(scratchBuffer), bytesWritten);	2071	_audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed);
2054	}
2055		2072
2056	int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();	2073	if (bytesAudioOutputUnplayed == 0) {
2057	float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC);	2074	_unfulfilledReads++;
2058	_audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed);	2075	}
2059		2076
2060	if (bytesAudioOutputUnplayed == 0) {	2077	return bytesWritten;
2061	_unfulfilledReads++;
2062	}	2078	}
2063		2079
2064	return bytesWritten;	2080	bool AudioClient::startRecording(const QString& filepath) {
2065	}	2081	if (!_audioFileWav.create(_outputFormat, filepath)) {
2066		2082	qDebug() << "Error creating audio file: " + filepath;
2067	bool AudioClient::startRecording(const QString& filepath) {	2083	return false;
2068	if (!_audioFileWav.create(_outputFormat, filepath)) {	2084	}
2069	qDebug() << "Error creating audio file: " + filepath;	2085	_isRecording = true;
2070	return false;	2086	return true;
2071	}	2087	}
2072	_isRecording = true;
2073	return true;
2074	}
2075		2088
2076	void AudioClient::stopRecording() {	2089	void AudioClient::stopRecording() {
2077	if (_isRecording) {	2090	if (_isRecording) {
2078	_isRecording = false;	2091	_isRecording = false;
2079	_audioFileWav.close();	2092	_audioFileWav.close();
		2093	}
2080	}	2094	}
2081	}
2082		2095
2083	void AudioClient::loadSettings() {	2096	void AudioClient::loadSettings() {
2084	_receivedAudioStream.setDynamicJitterBufferEnabled(dynamicJitterBufferEnabled.get());	2097	_receivedAudioStream.setDynamicJitterBufferEnabled(dynamicJitterBufferEnabled.get());
2085	_receivedAudioStream.setStaticJitterBufferFrames(staticJitterBufferFrames.get());	2098	_receivedAudioStream.setStaticJitterBufferFrames(staticJitterBufferFrames.get());
2086		2099
2087	qCDebug(audioclient) << "---- Initializing Audio Client ----";	2100	qCDebug(audioclient) << "---- Initializing Audio Client ----";
2088	auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();	2101	auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
2089	for (auto& plugin : codecPlugins) {	2102	for (auto& plugin : codecPlugins) {
2090	qCDebug(audioclient) << "Codec available:" << plugin->getName();	2103	qCDebug(audioclient) << "Codec available:" << plugin->getName();
		2104	}
2091	}	2105	}
2092	}
2093		2106
2094	void AudioClient::saveSettings() {	2107	void AudioClient::saveSettings() {
2095	dynamicJitterBufferEnabled.set(_receivedAudioStream.dynamicJitterBufferEnabled());	2108	dynamicJitterBufferEnabled.set(_receivedAudioStream.dynamicJitterBufferEnabled());
2096	staticJitterBufferFrames.set(_receivedAudioStream.getStaticJitterBufferFrames());	2109	staticJitterBufferFrames.set(_receivedAudioStream.getStaticJitterBufferFrames());
2097	}	2110	}
2098		2111
2099	void AudioClient::setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale) {	2112	void AudioClient::setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale) {
2100	avatarBoundingBoxCorner = corner;	2113	avatarBoundingBoxCorner = corner;
2101	avatarBoundingBoxScale = scale;	2114	avatarBoundingBoxScale = scale;
2102	}	2115	}
2103		2116
2104	void AudioClient::startThread() {	2117	void AudioClient::startThread() {
2105	moveToNewNamedThread(this, "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority);	2118	moveToNewNamedThread(this, "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority);
2106	}	2119	}
2107		2120
2108	void AudioClient::setInputVolume(float volume, bool emitSignal) {	2121	void AudioClient::setInputVolume(float volume, bool emitSignal) {
2109	if (_audioInput && volume != (float)_audioInput->volume()) {	2122	if (_audioInput && volume != (float)_audioInput->volume()) {
2110	_audioInput->setVolume(volume);	2123	_audioInput->setVolume(volume);
2111	if (emitSignal) {	2124	if (emitSignal) {
2112	emit inputVolumeChanged(_audioInput->volume());	2125	emit inputVolumeChanged(_audioInput->volume());
		2126	}
2113	}	2127	}
2114	}	2128	}
2115	}

libraries/audio-client/src/AudioClient.h+12−7

     void checkInputTimeout();
     void handleDummyAudioInput();
     void handleRecordedAudioInput(const QByteArray& audio);
-    void handleTTSAudioInput(const QByteArray& audio);
+    void handleTTSAudioInput(const QByteArray& audio,
+                             const int& newChunkSize,
+                             const int& timerInterval);
+    void clearTTSBuffer();
+    void processTTSBuffer();
     void reset();
     void audioMixerKilled();
     bool mixLocalAudioInjectors(float* mixBuffer);
     float azimuthForSource(const glm::vec3& relativePosition);
     float gainForSource(float distance, float volume);
-    void processAudioAndAddToRingBuffer(QByteArray& inputByteArray,
+    Mutex _TTSMutex;
-                                        const uchar& channelCount,
+    QTimer _TTSTimer;
-                                        const qint32& bytesForDuration,
+    bool _isProcessingTTS {false};
-                                        QByteArray& rollingBuffer);
+    QByteArray _TTSAudioBuffer;
+    int _TTSChunkSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * 50;
 #ifdef Q_OS_ANDROID
     QTimer _checkInputTimer;
     void configureReverb();
     void updateReverbOptions();
-    void handleLocalEchoAndReverb(QByteArray& inputByteArray);
+    void handleLocalEchoAndReverb(QByteArray& inputByteArray, const int& sampleRate, const int& channelCount);
     bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);
     bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);