Some experimentation yields promising results...

Zach Fox committed 8 years ago

Commit 26e388b1 parent 79f30eb5 files · +769 −736Browse files

Unified · Split

5 files

interface/src/Application.cpp +1 −0
interface/src/scripting/TTSScriptingInterface.cpp +12 −5
interface/src/scripting/TTSScriptingInterface.h +9 −2
libraries/audio-client/src/AudioClient.cpp +735 −722
libraries/audio-client/src/AudioClient.h +12 −7

@@ -1182,6 +1182,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo · 1181 unmodified lines
1182	1182
1183	1183	auto TTS = DependencyManager::get<TTSScriptingInterface>().data();
1184	1184	connect(TTS, &TTSScriptingInterface::ttsSampleCreated, audioIO, &AudioClient::handleTTSAudioInput);
	1185	+ connect(TTS, &TTSScriptingInterface::clearTTSBuffer, audioIO, &AudioClient::clearTTSBuffer);
1185	1186
1186	1187	connect(audioIO, &AudioClient::inputReceived, [](const QByteArray& audio) {
1187	1188	static auto recorder = DependencyManager::get<recording::Recorder>();

interface/src/scripting/TTSScriptingInterface.cpp+12−5

@@ -65,7 +65,7 @@ void TTSScriptingInterface::testTone(const bool& alsoInject) { · 64 unmodified lines
65	65	int16_t temp = (glm::sin(glm::radians((float)a))) * 32768;
66	66	samples[a] = temp;
67	67	}
68		− emit ttsSampleCreated(_lastSoundByteArray);
	68	+ emit ttsSampleCreated(_lastSoundByteArray, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * 50, 96);
69	69
70	70	if (alsoInject) {
71	71	AudioInjectorOptions options;
@@ -75,11 +75,16 @@ void TTSScriptingInterface::testTone(const bool& alsoInject) { · 3 unmodified lines
75	75	}
76	76	}
77	77
78		−void TTSScriptingInterface::speakText(const QString& textToSpeak, const bool& alsoInject) {
	78	+void TTSScriptingInterface::speakText(const QString& textToSpeak,
	79	+ const int& newChunkSize,
	80	+ const int& timerInterval,
	81	+ const int& sampleRate,
	82	+ const int& bitsPerSample,
	83	+ const bool& alsoInject) {
79	84	WAVEFORMATEX fmt;
80	85	fmt.wFormatTag = WAVE_FORMAT_PCM;
81		− fmt.nSamplesPerSec = 24000;
82		− fmt.wBitsPerSample = 16;
	86	+ fmt.nSamplesPerSec = sampleRate;
	87	+ fmt.wBitsPerSample = bitsPerSample;
83	88	fmt.nChannels = 1;
84	89	fmt.nBlockAlign = fmt.nChannels * fmt.wBitsPerSample / 8;
85	90	fmt.nAvgBytesPerSec = fmt.nSamplesPerSec * fmt.nBlockAlign;
@@ -146,7 +151,7 @@ void TTSScriptingInterface::speakText(const QString& textToSpeak, const bool& al · 60 unmodified lines
146	151	_lastSoundByteArray.resize(0);
147	152	_lastSoundByteArray.append(buf1, dwSize);
148	153
149		− emit ttsSampleCreated(_lastSoundByteArray);
	154	+ emit ttsSampleCreated(_lastSoundByteArray, newChunkSize, timerInterval);
150	155
151	156	if (alsoInject) {
152	157	AudioInjectorOptions options;
@@ -160,4 +165,6 @@ void TTSScriptingInterface::stopLastSpeech() { · 7 unmodified lines
160	165	if (_lastSoundAudioInjector) {
161	166	_lastSoundAudioInjector->stop();
162	167	}
	168	+
	169	+ emit clearTTSBuffer();
163	170	}

interface/src/scripting/TTSScriptingInterface.h+9−2

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include <sapi.h> // SAPI
20	20	#include <sphelper.h> // SAPI Helper
21	21	#include <AudioInjector.h>
	22	+#include <AudioConstants.h>
22	23
23	24	class TTSScriptingInterface : public QObject, public Dependency {
24	25	Q_OBJECT
@@ -28,11 +29,17 @@ public: · 3 unmodified lines
28	29	~TTSScriptingInterface();
29	30
30	31	Q_INVOKABLE void testTone(const bool& alsoInject = false);
31		− Q_INVOKABLE void speakText(const QString& textToSpeak, const bool& alsoInject = false);
	32	+ Q_INVOKABLE void speakText(const QString& textToSpeak,
	33	+ const int& newChunkSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * 50),
	34	+ const int& timerInterval = 96,
	35	+ const int& sampleRate = 24000,
	36	+ const int& bitsPerSample = 16,
	37	+ const bool& alsoInject = false);
32	38	Q_INVOKABLE void stopLastSpeech();
33	39
34	40	signals:
35		− void ttsSampleCreated(QByteArray outputArray);
	41	+ void ttsSampleCreated(QByteArray outputArray, const int& newChunkSize, const int& timerInterval);
	42	+ void clearTTSBuffer();
36	43
37	44	private:
38	45	class CComAutoInit {

libraries/audio-client/src/AudioClient.cpp+735−722

@@ -186,7 +186,7 @@ AudioClient::AudioClient() : · 185 unmodified lines
186	186	_networkToOutputResampler(NULL), _localToOutputResampler(NULL),
187	187	_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0),
188	188	_audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), _stats(&_receivedAudioStream),
189		− _positionGetter(DEFAULT_POSITION_GETTER),
	189	+ _positionGetter(DEFAULT_POSITION_GETTER), _TTSTimer(this),
190	190	#if defined(Q_OS_ANDROID)
191	191	_checkInputTimer(this), _isHeadsetPluggedIn(false),
192	192	#endif
@@ -245,6 +245,8 @@ AudioClient::AudioClient() : · 52 unmodified lines
245	245	packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket");
246	246	packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
247	247	packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat");
	248	+
	249	+ connect(&_TTSTimer, &QTimer::timeout, this, &AudioClient::processTTSBuffer);
248	250	}
249	251
250	252	AudioClient::~AudioClient() {
@@ -939,7 +941,7 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { · 688 unmodified lines
939	941	}
940	942	}
941	943
942		−void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
	944	+void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray, const int& sampleRate, const int& channelCount) {
943	945	// If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.
944	946	bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();
945	947	if (_muted \|\| !_audioOutput \|\| (!_shouldEchoLocally && !hasReverb)) {
@@ -949,7 +951,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { · 3 unmodified lines
949	951	// NOTE: we assume the inputFormat and the outputFormat are the same, since on any modern
950	952	// multimedia OS they should be. If there is a device that this is not true for, we can
951	953	// add back support to do resampling.
952		− if (_inputFormat.sampleRate() != _outputFormat.sampleRate()) {
	954	+ if (sampleRate != _outputFormat.sampleRate()) {
953	955	return;
954	956	}
955	957
@@ -972,7 +974,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { · 16 unmodified lines
972	974	static QByteArray loopBackByteArray;
973	975
974	976	int numInputSamples = inputByteArray.size() / AudioConstants::SAMPLE_SIZE;
975		− int numLoopbackSamples = (numInputSamples * OUTPUT_CHANNEL_COUNT) / _inputFormat.channelCount();
	977	+ int numLoopbackSamples = (numInputSamples * OUTPUT_CHANNEL_COUNT) / channelCount;
976	978
977	979	loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE);
978	980
@@ -980,7 +982,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { · 1 unmodified line
980	982	int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());
981	983
982	984	// upmix mono to stereo
983		− if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, _inputFormat.channelCount(),
	985	+ if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, channelCount,
984	986	OUTPUT_CHANNEL_COUNT)) {
985	987	// no conversion, just copy the samples
986	988	memcpy(loopbackSamples, inputSamples, numInputSamples * AudioConstants::SAMPLE_SIZE);
@@ -1093,23 +1095,29 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { · 106 unmodified lines
1093	1095	}
1094	1096	}
1095	1097
1096		−void AudioClient::processAudioAndAddToRingBuffer(QByteArray& inputByteArray,
1097		− const uchar& channelCount,
1098		− const qint32& bytesForDuration,
1099		− QByteArray& rollingBuffer) {
	1098	+void AudioClient::handleMicAudioInput() {
	1099	+ if (!_inputDevice \|\| _isPlayingBackRecording) {
	1100	+ return;
	1101	+ }
	1102	+
	1103	+#if defined(Q_OS_ANDROID)
	1104	+ _inputReadsSinceLastCheck++;
	1105	+#endif
	1106	+
1100	1107	// input samples required to produce exactly NETWORK_FRAME_SAMPLES of output
1101	1108	const int inputSamplesRequired =
1102	1109	(_inputToNetworkResampler ? _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL)
1103	1110	: AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
1104		− channelCount;
	1111	+ _inputFormat.channelCount();
1105	1112
1106	1113	const auto inputAudioSamples = std::unique_ptr<int16_t[]>(new int16_t[inputSamplesRequired]);
	1114	+ QByteArray inputByteArray = _inputDevice->readAll();
1107	1115
1108		− handleLocalEchoAndReverb(inputByteArray);
	1116	+ handleLocalEchoAndReverb(inputByteArray, _inputFormat.sampleRate(), _inputFormat.channelCount());
1109	1117
1110	1118	_inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());
1111	1119
1112		− float audioInputMsecsRead = inputByteArray.size() / (float)(bytesForDuration);
	1120	+ float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC));
1113	1121	_stats.updateInputMsRead(audioInputMsecsRead);
1114	1122
1115	1123	const int numNetworkBytes =
@@ -1125,33 +1133,17 @@ void AudioClient::processAudioAndAddToRingBuffer(QByteArray& inputByteArray, · 9 unmodified lines
1125	1133	} else {
1126	1134	_inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired);
1127	1135	possibleResampling(_inputToNetworkResampler, inputAudioSamples.get(), networkAudioSamples, inputSamplesRequired,
1128		− numNetworkSamples, channelCount, _desiredInputFormat.channelCount());
	1136	+ numNetworkSamples, _inputFormat.channelCount(), _desiredInputFormat.channelCount());
1129	1137	}
1130	1138	int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * AudioConstants::SAMPLE_SIZE;
1131		− float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(bytesForDuration);
	1139	+ float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC));
1132	1140	_stats.updateInputMsUnplayed(msecsInInputRingBuffer);
1133	1141
1134	1142	QByteArray audioBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
1135		− rollingBuffer.append(audioBuffer);
1136	1143	handleAudioInput(audioBuffer);
1137	1144	}
1138	1145	}
1139	1146
1140		−void AudioClient::handleMicAudioInput() {
1141		− if (!_inputDevice \|\| _isPlayingBackRecording) {
1142		− return;
1143		− }
1144		−
1145		−#if defined(Q_OS_ANDROID)
1146		− _inputReadsSinceLastCheck++;
1147		−#endif
1148		−
1149		− QByteArray temp;
1150		−
1151		− processAudioAndAddToRingBuffer(_inputDevice->readAll(), _inputFormat.channelCount(),
1152		− _inputFormat.bytesForDuration(USECS_PER_MSEC), temp);
1153		−}
1154		−
1155	1147	void AudioClient::handleDummyAudioInput() {
1156	1148	const int numNetworkBytes =
1157	1149	_isStereoInput ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
@@ -1192,7 +1184,7 @@ int rawToWav(const char* rawData, const int& rawLength, const char* wavfn, long · 34 unmodified lines
1192	1184	long chunksize = 0x10;
1193	1185	fwrite(&chunksize, 4, 1, wav);
1194	1186
1195		− fmt.wFormatTag = 1; // WAVE_FORMAT_PCM
	1187	+ fmt.wFormatTag = 1; // WAVE_FORMAT_PCM
1196	1188	fmt.wChannels = channels;
1197	1189	fmt.dwSamplesPerSec = frequency * 1;
1198	1190	fmt.wBitsPerSample = 16;
@@ -1210,906 +1202,927 @@ int rawToWav(const char* rawData, const int& rawLength, const char* wavfn, long · 11 unmodified lines
1210	1202	return 0;
1211	1203	}
1212	1204
1213		−void AudioClient::handleTTSAudioInput(const QByteArray& audio) {
1214		− QByteArray audioBuffer(audio);
	1205	+void AudioClient::processTTSBuffer() {
	1206	+ Lock lock(_TTSMutex);
	1207	+ if (_TTSAudioBuffer.size() > 0) {
	1208	+ QByteArray part;
	1209	+ part.append(_TTSAudioBuffer.data(), _TTSChunkSize);
	1210	+ _TTSAudioBuffer.remove(0, _TTSChunkSize);
	1211	+ handleAudioInput(part);
	1212	+ } else {
	1213	+ _isProcessingTTS = false;
	1214	+ _TTSTimer.stop();
	1215	+ }
	1216	+}
1215	1217
1216		− QString filename = QString::number(usecTimestampNow());
1217		− QString path = PathUtils::getAppDataPath() + "Audio/" + filename + "-before.wav";
1218		− rawToWav(audioBuffer.data(), audioBuffer.size(), path.toLocal8Bit(), 24000, 1);
	1218	+void AudioClient::handleTTSAudioInput(const QByteArray& audio, const int& newChunkSize, const int& timerInterval) {
	1219	+ _TTSChunkSize = newChunkSize;
	1220	+ _TTSAudioBuffer.append(audio);
1219	1221
1220		− QByteArray temp;
	1222	+ handleLocalEchoAndReverb(_TTSAudioBuffer, 48000, 1);
1221	1223
1222		− while (audioBuffer.size() > 0) {
1223		− QByteArray part;
1224		− part.append(audioBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1225		− audioBuffer.remove(0, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1226		− processAudioAndAddToRingBuffer(part, 1, 48, temp);
1227		− }
	1224	+ //QString filename = QString::number(usecTimestampNow());
	1225	+ //QString path = PathUtils::getAppDataPath() + "Audio/" + filename + "-before.wav";
	1226	+ //rawToWav(_TTSAudioBuffer.data(), _TTSAudioBuffer.size(), path.toLocal8Bit(), 24000, 1);
1228	1227
1229		− filename = QString::number(usecTimestampNow());
1230		− path = PathUtils::getAppDataPath() + "Audio/" + filename + "-after.wav";
1231		− rawToWav(temp.data(), temp.size(), path.toLocal8Bit(), 12000, 1);
	1228	+ //QByteArray temp;
	1229	+
	1230	+ _isProcessingTTS = true;
	1231	+ _TTSTimer.start(timerInterval);
	1232	+
	1233	+ //filename = QString::number(usecTimestampNow());
	1234	+ //path = PathUtils::getAppDataPath() + "Audio/" + filename + "-after.wav";
	1235	+ //rawToWav(temp.data(), temp.size(), path.toLocal8Bit(), 12000, 1);
1232	1236	}
1233	1237
1234		−void AudioClient::prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock) {
1235		− bool doSynchronously = localAudioLock.operator bool();
1236		− if (!localAudioLock) {
1237		− localAudioLock.reset(new Lock(_localAudioMutex));
1238		− }
	1238	+void AudioClient::clearTTSBuffer() {
	1239	+ _TTSAudioBuffer.resize(0);
	1240	+ _isProcessingTTS = false;
	1241	+ _TTSTimer.stop();
	1242	+}
1239	1243
1240		− int samplesNeeded = std::numeric_limits<int>::max();
1241		− while (samplesNeeded > 0) {
1242		− if (!doSynchronously) {
1243		− // unlock between every write to allow device switching
1244		− localAudioLock->unlock();
1245		− localAudioLock->lock();
	1244	+void AudioClient::prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock) {
	1245	+ bool doSynchronously = localAudioLock.operator bool();
	1246	+ if (!localAudioLock) {
	1247	+ localAudioLock.reset(new Lock(_localAudioMutex));
1246	1248	}
1247	1249
1248		− // in case of a device switch, consider bufferCapacity volatile across iterations
1249		− if (_outputPeriod == 0) {
1250		− return;
1251		− }
	1250	+ int samplesNeeded = std::numeric_limits<int>::max();
	1251	+ while (samplesNeeded > 0) {
	1252	+ if (!doSynchronously) {
	1253	+ // unlock between every write to allow device switching
	1254	+ localAudioLock->unlock();
	1255	+ localAudioLock->lock();
	1256	+ }
1252	1257
1253		− int bufferCapacity = _localInjectorsStream.getSampleCapacity();
1254		− int maxOutputSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * AudioConstants::STEREO;
1255		− if (_localToOutputResampler) {
1256		− maxOutputSamples = _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
1257		− AudioConstants::STEREO;
1258		− }
	1258	+ // in case of a device switch, consider bufferCapacity volatile across iterations
	1259	+ if (_outputPeriod == 0) {
	1260	+ return;
	1261	+ }
1259	1262
1260		− samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);
1261		− if (samplesNeeded < maxOutputSamples) {
1262		− // avoid overwriting the buffer to prevent losing frames
1263		− break;
1264		− }
	1263	+ int bufferCapacity = _localInjectorsStream.getSampleCapacity();
	1264	+ int maxOutputSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * AudioConstants::STEREO;
	1265	+ if (_localToOutputResampler) {
	1266	+ maxOutputSamples = _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
	1267	+ AudioConstants::STEREO;
	1268	+ }
1265	1269
1266		− // get a network frame of local injectors' audio
1267		− if (!mixLocalAudioInjectors(_localMixBuffer)) {
1268		− break;
1269		− }
	1270	+ samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);
	1271	+ if (samplesNeeded < maxOutputSamples) {
	1272	+ // avoid overwriting the buffer to prevent losing frames
	1273	+ break;
	1274	+ }
1270	1275
1271		− // reverb
1272		− if (_reverb) {
1273		− _localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1274		− }
	1276	+ // get a network frame of local injectors' audio
	1277	+ if (!mixLocalAudioInjectors(_localMixBuffer)) {
	1278	+ break;
	1279	+ }
1275	1280
1276		− int samples;
1277		− if (_localToOutputResampler) {
1278		− // resample to output sample rate
1279		− int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer,
1280		− AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
	1281	+ // reverb
	1282	+ if (_reverb) {
	1283	+ _localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
	1284	+ }
1281	1285
1282		− // write to local injectors' ring buffer
1283		− samples = frames * AudioConstants::STEREO;
1284		− _localInjectorsStream.writeSamples(_localOutputMixBuffer, samples);
	1286	+ int samples;
	1287	+ if (_localToOutputResampler) {
	1288	+ // resample to output sample rate
	1289	+ int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer,
	1290	+ AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1285	1291
1286		− } else {
1287		− // write to local injectors' ring buffer
1288		− samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
1289		− _localInjectorsStream.writeSamples(_localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
1290		− }
	1292	+ // write to local injectors' ring buffer
	1293	+ samples = frames * AudioConstants::STEREO;
	1294	+ _localInjectorsStream.writeSamples(_localOutputMixBuffer, samples);
1291	1295
1292		− _localSamplesAvailable.fetch_add(samples, std::memory_order_release);
1293		− samplesNeeded -= samples;
1294		− }
1295		−}
	1296	+ } else {
	1297	+ // write to local injectors' ring buffer
	1298	+ samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
	1299	+ _localInjectorsStream.writeSamples(_localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
	1300	+ }
1296	1301
1297		−bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
1298		− // check the flag for injectors before attempting to lock
1299		− if (!_localInjectorsAvailable.load(std::memory_order_acquire)) {
1300		− return false;
	1302	+ _localSamplesAvailable.fetch_add(samples, std::memory_order_release);
	1303	+ samplesNeeded -= samples;
	1304	+ }
1301	1305	}
1302	1306
1303		− // lock the injectors
1304		− Lock lock(_injectorsMutex);
1305		−
1306		− QVector<AudioInjectorPointer> injectorsToRemove;
	1307	+ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
	1308	+ // check the flag for injectors before attempting to lock
	1309	+ if (!_localInjectorsAvailable.load(std::memory_order_acquire)) {
	1310	+ return false;
	1311	+ }
1307	1312
1308		− memset(mixBuffer, 0, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO * sizeof(float));
	1313	+ // lock the injectors
	1314	+ Lock lock(_injectorsMutex);
1309	1315
1310		− for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) {
1311		− // the lock guarantees that injectorBuffer, if found, is invariant
1312		− AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
1313		− if (injectorBuffer) {
1314		− static const int HRTF_DATASET_INDEX = 1;
1315		−
1316		− int numChannels = injector->isAmbisonic() ? AudioConstants::AMBISONIC
1317		− : (injector->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
1318		− size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
1319		−
1320		− // get one frame from the injector
1321		− memset(_localScratchBuffer, 0, bytesToRead);
1322		− if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) {
1323		− if (injector->isAmbisonic()) {
1324		− // no distance attenuation
1325		− float gain = injector->getVolume();
1326		−
1327		− //
1328		− // Calculate the soundfield orientation relative to the listener.
1329		− // Injector orientation can be used to align a recording to our world coordinates.
1330		− //
1331		− glm::quat relativeOrientation = injector->getOrientation() * glm::inverse(_orientationGetter());
1332		−
1333		− // convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system
1334		− float qw = relativeOrientation.w;
1335		− float qx = -relativeOrientation.z;
1336		− float qy = -relativeOrientation.x;
1337		− float qz = relativeOrientation.y;
1338		−
1339		− // Ambisonic gets spatialized into mixBuffer
1340		− injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, qw, qx, qy, qz, gain,
1341		− AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1342		−
1343		− } else if (injector->isStereo()) {
1344		− // stereo gets directly mixed into mixBuffer
1345		− float gain = injector->getVolume();
1346		− for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
1347		− mixBuffer[i] += convertToFloat(_localScratchBuffer[i]) * gain;
	1316	+ QVector<AudioInjectorPointer> injectorsToRemove;
	1317	+
	1318	+ memset(mixBuffer, 0, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO * sizeof(float));
	1319	+
	1320	+ for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) {
	1321	+ // the lock guarantees that injectorBuffer, if found, is invariant
	1322	+ AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
	1323	+ if (injectorBuffer) {
	1324	+ static const int HRTF_DATASET_INDEX = 1;
	1325	+
	1326	+ int numChannels = injector->isAmbisonic()
	1327	+ ? AudioConstants::AMBISONIC
	1328	+ : (injector->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
	1329	+ size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
	1330	+
	1331	+ // get one frame from the injector
	1332	+ memset(_localScratchBuffer, 0, bytesToRead);
	1333	+ if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) {
	1334	+ if (injector->isAmbisonic()) {
	1335	+ // no distance attenuation
	1336	+ float gain = injector->getVolume();
	1337	+
	1338	+ //
	1339	+ // Calculate the soundfield orientation relative to the listener.
	1340	+ // Injector orientation can be used to align a recording to our world coordinates.
	1341	+ //
	1342	+ glm::quat relativeOrientation = injector->getOrientation() * glm::inverse(_orientationGetter());
	1343	+
	1344	+ // convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system
	1345	+ float qw = relativeOrientation.w;
	1346	+ float qx = -relativeOrientation.z;
	1347	+ float qy = -relativeOrientation.x;
	1348	+ float qz = relativeOrientation.y;
	1349	+
	1350	+ // Ambisonic gets spatialized into mixBuffer
	1351	+ injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, qw, qx, qy, qz, gain,
	1352	+ AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
	1353	+
	1354	+ } else if (injector->isStereo()) {
	1355	+ // stereo gets directly mixed into mixBuffer
	1356	+ float gain = injector->getVolume();
	1357	+ for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
	1358	+ mixBuffer[i] += convertToFloat(_localScratchBuffer[i]) * gain;
	1359	+ }
	1360	+
	1361	+ } else {
	1362	+ // calculate distance, gain and azimuth for hrtf
	1363	+ glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
	1364	+ float distance = glm::max(glm::length(relativePosition), EPSILON);
	1365	+ float gain = gainForSource(distance, injector->getVolume());
	1366	+ float azimuth = azimuthForSource(relativePosition);
	1367	+
	1368	+ // mono gets spatialized into mixBuffer
	1369	+ injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, azimuth, distance,
	1370	+ gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1348	1371	}
1349	1372
1350	1373	} else {
1351		− // calculate distance, gain and azimuth for hrtf
1352		− glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
1353		− float distance = glm::max(glm::length(relativePosition), EPSILON);
1354		− float gain = gainForSource(distance, injector->getVolume());
1355		− float azimuth = azimuthForSource(relativePosition);
1356		−
1357		− // mono gets spatialized into mixBuffer
1358		− injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, azimuth, distance, gain,
1359		− AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
	1374	+ qCDebug(audioclient) << "injector has no more data, marking finished for removal";
	1375	+ injector->finishLocalInjection();
	1376	+ injectorsToRemove.append(injector);
1360	1377	}
1361	1378
1362	1379	} else {
1363		− qCDebug(audioclient) << "injector has no more data, marking finished for removal";
	1380	+ qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal";
1364	1381	injector->finishLocalInjection();
1365	1382	injectorsToRemove.append(injector);
1366	1383	}
1367		−
1368		− } else {
1369		− qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal";
1370		− injector->finishLocalInjection();
1371		− injectorsToRemove.append(injector);
1372	1384	}
1373		− }
1374	1385
1375		− for (const AudioInjectorPointer& injector : injectorsToRemove) {
1376		− qCDebug(audioclient) << "removing injector";
1377		− _activeLocalAudioInjectors.removeOne(injector);
1378		− }
	1386	+ for (const AudioInjectorPointer& injector : injectorsToRemove) {
	1387	+ qCDebug(audioclient) << "removing injector";
	1388	+ _activeLocalAudioInjectors.removeOne(injector);
	1389	+ }
1379	1390
1380		− // update the flag
1381		− _localInjectorsAvailable.exchange(!_activeLocalAudioInjectors.empty(), std::memory_order_release);
	1391	+ // update the flag
	1392	+ _localInjectorsAvailable.exchange(!_activeLocalAudioInjectors.empty(), std::memory_order_release);
1382	1393
1383		− return true;
1384		−}
	1394	+ return true;
	1395	+ }
1385	1396
1386		−void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
1387		− const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
1388		− assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
	1397	+ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
	1398	+ const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
	1399	+ assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
1389	1400
1390		− outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE);
1391		− int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
	1401	+ outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE);
	1402	+ int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
1392	1403
1393		− bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();
	1404	+ bool hasReverb = _reverb \|\| _receivedAudioStream.hasReverb();
1394	1405
1395		− // apply stereo reverb
1396		− if (hasReverb) {
1397		− updateReverbOptions();
1398		− int16_t* reverbSamples = _networkToOutputResampler ? _networkScratchBuffer : outputSamples;
1399		− _listenerReverb.render(decodedSamples, reverbSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1400		− }
	1406	+ // apply stereo reverb
	1407	+ if (hasReverb) {
	1408	+ updateReverbOptions();
	1409	+ int16_t* reverbSamples = _networkToOutputResampler ? _networkScratchBuffer : outputSamples;
	1410	+ _listenerReverb.render(decodedSamples, reverbSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
	1411	+ }
1401	1412
1402		− // resample to output sample rate
1403		− if (_networkToOutputResampler) {
1404		− const int16_t* inputSamples = hasReverb ? _networkScratchBuffer : decodedSamples;
1405		− _networkToOutputResampler->render(inputSamples, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
1406		− }
	1413	+ // resample to output sample rate
	1414	+ if (_networkToOutputResampler) {
	1415	+ const int16_t* inputSamples = hasReverb ? _networkScratchBuffer : decodedSamples;
	1416	+ _networkToOutputResampler->render(inputSamples, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
	1417	+ }
1407	1418
1408		− // if no transformations were applied, we still need to copy the buffer
1409		− if (!hasReverb && !_networkToOutputResampler) {
1410		− memcpy(outputSamples, decodedSamples, decodedBuffer.size());
	1419	+ // if no transformations were applied, we still need to copy the buffer
	1420	+ if (!hasReverb && !_networkToOutputResampler) {
	1421	+ memcpy(outputSamples, decodedSamples, decodedBuffer.size());
	1422	+ }
1411	1423	}
1412		−}
1413	1424
1414		−void AudioClient::sendMuteEnvironmentPacket() {
1415		− auto nodeList = DependencyManager::get<NodeList>();
	1425	+ void AudioClient::sendMuteEnvironmentPacket() {
	1426	+ auto nodeList = DependencyManager::get<NodeList>();
1416	1427
1417		− int dataSize = sizeof(glm::vec3) + sizeof(float);
	1428	+ int dataSize = sizeof(glm::vec3) + sizeof(float);
1418	1429
1419		− auto mutePacket = NLPacket::create(PacketType::MuteEnvironment, dataSize);
	1430	+ auto mutePacket = NLPacket::create(PacketType::MuteEnvironment, dataSize);
1420	1431
1421		− const float MUTE_RADIUS = 50;
	1432	+ const float MUTE_RADIUS = 50;
1422	1433
1423		− glm::vec3 currentSourcePosition = _positionGetter();
	1434	+ glm::vec3 currentSourcePosition = _positionGetter();
1424	1435
1425		− mutePacket->writePrimitive(currentSourcePosition);
1426		− mutePacket->writePrimitive(MUTE_RADIUS);
	1436	+ mutePacket->writePrimitive(currentSourcePosition);
	1437	+ mutePacket->writePrimitive(MUTE_RADIUS);
1427	1438
1428		− // grab our audio mixer from the NodeList, if it exists
1429		− SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
	1439	+ // grab our audio mixer from the NodeList, if it exists
	1440	+ SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
1430	1441
1431		− if (audioMixer) {
1432		− // send off this mute packet
1433		− nodeList->sendPacket(std::move(mutePacket), *audioMixer);
	1442	+ if (audioMixer) {
	1443	+ // send off this mute packet
	1444	+ nodeList->sendPacket(std::move(mutePacket), *audioMixer);
	1445	+ }
1434	1446	}
1435		−}
1436	1447
1437		−void AudioClient::setMuted(bool muted, bool emitSignal) {
1438		− if (_muted != muted) {
1439		− _muted = muted;
1440		− if (emitSignal) {
1441		− emit muteToggled(_muted);
	1448	+ void AudioClient::setMuted(bool muted, bool emitSignal) {
	1449	+ if (_muted != muted) {
	1450	+ _muted = muted;
	1451	+ if (emitSignal) {
	1452	+ emit muteToggled(_muted);
	1453	+ }
1442	1454	}
1443	1455	}
1444		−}
1445	1456
1446		−void AudioClient::setNoiseReduction(bool enable, bool emitSignal) {
1447		− if (_isNoiseGateEnabled != enable) {
1448		− _isNoiseGateEnabled = enable;
1449		− if (emitSignal) {
1450		− emit noiseReductionChanged(_isNoiseGateEnabled);
	1457	+ void AudioClient::setNoiseReduction(bool enable, bool emitSignal) {
	1458	+ if (_isNoiseGateEnabled != enable) {
	1459	+ _isNoiseGateEnabled = enable;
	1460	+ if (emitSignal) {
	1461	+ emit noiseReductionChanged(_isNoiseGateEnabled);
	1462	+ }
1451	1463	}
1452	1464	}
1453		−}
1454	1465
1455		−bool AudioClient::setIsStereoInput(bool isStereoInput) {
1456		− bool stereoInputChanged = false;
1457		− if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) {
1458		− _isStereoInput = isStereoInput;
1459		− stereoInputChanged = true;
	1466	+ bool AudioClient::setIsStereoInput(bool isStereoInput) {
	1467	+ bool stereoInputChanged = false;
	1468	+ if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) {
	1469	+ _isStereoInput = isStereoInput;
	1470	+ stereoInputChanged = true;
1460	1471
1461		− if (_isStereoInput) {
1462		− _desiredInputFormat.setChannelCount(2);
1463		− } else {
1464		− _desiredInputFormat.setChannelCount(1);
1465		− }
	1472	+ if (_isStereoInput) {
	1473	+ _desiredInputFormat.setChannelCount(2);
	1474	+ } else {
	1475	+ _desiredInputFormat.setChannelCount(1);
	1476	+ }
1466	1477
1467		− // restart the codec
1468		− if (_codec) {
1469		− if (_encoder) {
1470		− _codec->releaseEncoder(_encoder);
	1478	+ // restart the codec
	1479	+ if (_codec) {
	1480	+ if (_encoder) {
	1481	+ _codec->releaseEncoder(_encoder);
	1482	+ }
	1483	+ _encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE,
	1484	+ _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO);
1471	1485	}
1472		− _encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE,
1473		− _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO);
1474		− }
1475		− qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput;
	1486	+ qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput;
	1487	+
	1488	+ // restart the input device
	1489	+ switchInputToAudioDevice(_inputDeviceInfo);
1476	1490
1477		− // restart the input device
1478		− switchInputToAudioDevice(_inputDeviceInfo);
	1491	+ emit isStereoInputChanged(_isStereoInput);
	1492	+ }
1479	1493
1480		− emit isStereoInputChanged(_isStereoInput);
	1494	+ return stereoInputChanged;
1481	1495	}
1482	1496
1483		− return stereoInputChanged;
1484		−}
	1497	+ bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) {
	1498	+ AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
	1499	+ if (injectorBuffer) {
	1500	+ // local injectors are on the AudioInjectorsThread, so we must guard access
	1501	+ Lock lock(_injectorsMutex);
	1502	+ if (!_activeLocalAudioInjectors.contains(injector)) {
	1503	+ qCDebug(audioclient) << "adding new injector";
	1504	+ _activeLocalAudioInjectors.append(injector);
	1505	+ // move local buffer to the LocalAudioThread to avoid dataraces with AudioInjector (like stop())
	1506	+ injectorBuffer->setParent(nullptr);
	1507	+
	1508	+ // update the flag
	1509	+ _localInjectorsAvailable.exchange(true, std::memory_order_release);
	1510	+ } else {
	1511	+ qCDebug(audioclient) << "injector exists in active list already";
	1512	+ }
	1513	+
	1514	+ return true;
1485	1515
1486		−bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) {
1487		− AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer();
1488		− if (injectorBuffer) {
1489		− // local injectors are on the AudioInjectorsThread, so we must guard access
1490		− Lock lock(_injectorsMutex);
1491		− if (!_activeLocalAudioInjectors.contains(injector)) {
1492		− qCDebug(audioclient) << "adding new injector";
1493		− _activeLocalAudioInjectors.append(injector);
1494		− // move local buffer to the LocalAudioThread to avoid dataraces with AudioInjector (like stop())
1495		− injectorBuffer->setParent(nullptr);
1496		−
1497		− // update the flag
1498		− _localInjectorsAvailable.exchange(true, std::memory_order_release);
1499	1516	} else {
1500		− qCDebug(audioclient) << "injector exists in active list already";
	1517	+ // no local buffer
	1518	+ return false;
1501	1519	}
1502		−
1503		− return true;
1504		−
1505		− } else {
1506		− // no local buffer
1507		− return false;
1508	1520	}
1509		−}
1510	1521
1511		−void AudioClient::outputFormatChanged() {
1512		− _outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) /
1513		− _desiredOutputFormat.sampleRate();
1514		− _receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
1515		−}
	1522	+ void AudioClient::outputFormatChanged() {
	1523	+ _outputFrameSize =
	1524	+ (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) /
	1525	+ _desiredOutputFormat.sampleRate();
	1526	+ _receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
	1527	+ }
1516	1528
1517		−bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) {
1518		− Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");
	1529	+ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) {
	1530	+ Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");
1519	1531
1520		− qCDebug(audioclient) << __FUNCTION__ << "inputDeviceInfo: [" << inputDeviceInfo.deviceName() << "]";
1521		− bool supportedFormat = false;
	1532	+ qCDebug(audioclient) << __FUNCTION__ << "inputDeviceInfo: [" << inputDeviceInfo.deviceName() << "]";
	1533	+ bool supportedFormat = false;
1522	1534
1523		− // NOTE: device start() uses the Qt internal device list
1524		− Lock lock(_deviceMutex);
	1535	+ // NOTE: device start() uses the Qt internal device list
	1536	+ Lock lock(_deviceMutex);
1525	1537
1526	1538	#if defined(Q_OS_ANDROID)
1527		− _shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged
	1539	+ _shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged
1528	1540	#endif
1529	1541
1530		− // cleanup any previously initialized device
1531		− if (_audioInput) {
1532		− // The call to stop() causes _inputDevice to be destructed.
1533		− // That in turn causes it to be disconnected (see for example
1534		− // http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect).
1535		− _audioInput->stop();
1536		− _inputDevice = NULL;
	1542	+ // cleanup any previously initialized device
	1543	+ if (_audioInput) {
	1544	+ // The call to stop() causes _inputDevice to be destructed.
	1545	+ // That in turn causes it to be disconnected (see for example
	1546	+ // http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect).
	1547	+ _audioInput->stop();
	1548	+ _inputDevice = NULL;
1537	1549
1538		− _audioInput->deleteLater();
1539		− _audioInput = NULL;
1540		− _numInputCallbackBytes = 0;
	1550	+ _audioInput->deleteLater();
	1551	+ _audioInput = NULL;
	1552	+ _numInputCallbackBytes = 0;
1541	1553
1542		− _inputDeviceInfo = QAudioDeviceInfo();
1543		− }
	1554	+ _inputDeviceInfo = QAudioDeviceInfo();
	1555	+ }
1544	1556
1545		− if (_dummyAudioInput) {
1546		− _dummyAudioInput->stop();
	1557	+ if (_dummyAudioInput) {
	1558	+ _dummyAudioInput->stop();
1547	1559
1548		− _dummyAudioInput->deleteLater();
1549		− _dummyAudioInput = NULL;
1550		− }
	1560	+ _dummyAudioInput->deleteLater();
	1561	+ _dummyAudioInput = NULL;
	1562	+ }
1551	1563
1552		− if (_inputToNetworkResampler) {
1553		− // if we were using an input to network resampler, delete it here
1554		− delete _inputToNetworkResampler;
1555		− _inputToNetworkResampler = NULL;
1556		− }
	1564	+ if (_inputToNetworkResampler) {
	1565	+ // if we were using an input to network resampler, delete it here
	1566	+ delete _inputToNetworkResampler;
	1567	+ _inputToNetworkResampler = NULL;
	1568	+ }
1557	1569
1558		− if (_audioGate) {
1559		− delete _audioGate;
1560		− _audioGate = nullptr;
1561		− }
	1570	+ if (_audioGate) {
	1571	+ delete _audioGate;
	1572	+ _audioGate = nullptr;
	1573	+ }
1562	1574
1563		− if (isShutdownRequest) {
1564		− qCDebug(audioclient) << "The audio input device has shut down.";
1565		− return true;
1566		− }
	1575	+ if (isShutdownRequest) {
	1576	+ qCDebug(audioclient) << "The audio input device has shut down.";
	1577	+ return true;
	1578	+ }
1567	1579
1568		− if (!inputDeviceInfo.isNull()) {
1569		− qCDebug(audioclient) << "The audio input device " << inputDeviceInfo.deviceName() << "is available.";
1570		− _inputDeviceInfo = inputDeviceInfo;
1571		− emit deviceChanged(QAudio::AudioInput, inputDeviceInfo);
	1580	+ if (!inputDeviceInfo.isNull()) {
	1581	+ qCDebug(audioclient) << "The audio input device " << inputDeviceInfo.deviceName() << "is available.";
	1582	+ _inputDeviceInfo = inputDeviceInfo;
	1583	+ emit deviceChanged(QAudio::AudioInput, inputDeviceInfo);
1572	1584
1573		− if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {
1574		− qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
	1585	+ if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {
	1586	+ qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
1575	1587
1576		− // we've got the best we can get for input
1577		− // if required, setup a resampler for this input to our desired network format
1578		− if (_inputFormat != _desiredInputFormat && _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) {
1579		− qCDebug(audioclient) << "Attemping to create a resampler for input format to network format.";
	1588	+ // we've got the best we can get for input
	1589	+ // if required, setup a resampler for this input to our desired network format
	1590	+ if (_inputFormat != _desiredInputFormat && _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) {
	1591	+ qCDebug(audioclient) << "Attemping to create a resampler for input format to network format.";
1580	1592
1581		− assert(_inputFormat.sampleSize() == 16);
1582		− assert(_desiredInputFormat.sampleSize() == 16);
1583		− int channelCount = (_inputFormat.channelCount() == 2 && _desiredInputFormat.channelCount() == 2) ? 2 : 1;
	1593	+ assert(_inputFormat.sampleSize() == 16);
	1594	+ assert(_desiredInputFormat.sampleSize() == 16);
	1595	+ int channelCount = (_inputFormat.channelCount() == 2 && _desiredInputFormat.channelCount() == 2) ? 2 : 1;
1584	1596
1585		− _inputToNetworkResampler =
1586		− new AudioSRC(_inputFormat.sampleRate(), _desiredInputFormat.sampleRate(), channelCount);
	1597	+ _inputToNetworkResampler =
	1598	+ new AudioSRC(_inputFormat.sampleRate(), _desiredInputFormat.sampleRate(), channelCount);
1587	1599
1588		− } else {
1589		− qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
1590		− }
	1600	+ } else {
	1601	+ qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
	1602	+ }
1591	1603
1592		− // the audio gate runs after the resampler
1593		− _audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());
1594		− qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";
	1604	+ // the audio gate runs after the resampler
	1605	+ _audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());
	1606	+ qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";
1595	1607
1596		− // if the user wants stereo but this device can't provide then bail
1597		− if (!_isStereoInput \|\| _inputFormat.channelCount() == 2) {
1598		− _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);
1599		− _numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat);
1600		− _audioInput->setBufferSize(_numInputCallbackBytes);
1601		− // different audio input devices may have different volumes
1602		− emit inputVolumeChanged(_audioInput->volume());
	1608	+ // if the user wants stereo but this device can't provide then bail
	1609	+ if (!_isStereoInput \|\| _inputFormat.channelCount() == 2) {
	1610	+ _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);
	1611	+ _numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat);
	1612	+ _audioInput->setBufferSize(_numInputCallbackBytes);
	1613	+ // different audio input devices may have different volumes
	1614	+ emit inputVolumeChanged(_audioInput->volume());
1603	1615
1604		− // how do we want to handle input working, but output not working?
1605		− int numFrameSamples = calculateNumberOfFrameSamples(_numInputCallbackBytes);
1606		− _inputRingBuffer.resizeForFrameSize(numFrameSamples);
	1616	+ // how do we want to handle input working, but output not working?
	1617	+ int numFrameSamples = calculateNumberOfFrameSamples(_numInputCallbackBytes);
	1618	+ _inputRingBuffer.resizeForFrameSize(numFrameSamples);
1607	1619
1608	1620	#if defined(Q_OS_ANDROID)
1609		− if (_audioInput) {
1610		− _shouldRestartInputSetup = true;
1611		− connect(_audioInput, &QAudioInput::stateChanged, this, &AudioClient::audioInputStateChanged);
1612		− }
	1621	+ if (_audioInput) {
	1622	+ _shouldRestartInputSetup = true;
	1623	+ connect(_audioInput, &QAudioInput::stateChanged, this, &AudioClient::audioInputStateChanged);
	1624	+ }
1613	1625	#endif
1614		− _inputDevice = _audioInput->start();
1615		−
1616		− if (_inputDevice) {
1617		− connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
1618		− supportedFormat = true;
1619		− } else {
1620		− qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error();
1621		− _audioInput->deleteLater();
1622		− _audioInput = NULL;
	1626	+ _inputDevice = _audioInput->start();
	1627	+
	1628	+ if (_inputDevice) {
	1629	+ connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
	1630	+ supportedFormat = true;
	1631	+ } else {
	1632	+ qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error();
	1633	+ _audioInput->deleteLater();
	1634	+ _audioInput = NULL;
	1635	+ }
1623	1636	}
1624	1637	}
1625	1638	}
1626		− }
1627	1639
1628		− // If there is no working input device, use the dummy input device.
1629		− // It generates audio callbacks on a timer to simulate a mic stream of silent packets.
1630		− // This enables clients without a mic to still receive an audio stream from the mixer.
1631		− if (!_audioInput) {
1632		− qCDebug(audioclient) << "Audio input device is not available, using dummy input.";
1633		− _inputDeviceInfo = QAudioDeviceInfo();
1634		− emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo);
	1640	+ // If there is no working input device, use the dummy input device.
	1641	+ // It generates audio callbacks on a timer to simulate a mic stream of silent packets.
	1642	+ // This enables clients without a mic to still receive an audio stream from the mixer.
	1643	+ if (!_audioInput) {
	1644	+ qCDebug(audioclient) << "Audio input device is not available, using dummy input.";
	1645	+ _inputDeviceInfo = QAudioDeviceInfo();
	1646	+ emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo);
1635	1647
1636		− _inputFormat = _desiredInputFormat;
1637		− qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
1638		− qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
	1648	+ _inputFormat = _desiredInputFormat;
	1649	+ qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
	1650	+ qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
1639	1651
1640		− _audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());
1641		− qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";
	1652	+ _audioGate = new AudioGate(_desiredInputFormat.sampleRate(), _desiredInputFormat.channelCount());
	1653	+ qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount() << "channels.";
1642	1654
1643		− // generate audio callbacks at the network sample rate
1644		− _dummyAudioInput = new QTimer(this);
1645		− connect(_dummyAudioInput, SIGNAL(timeout()), this, SLOT(handleDummyAudioInput()));
1646		− _dummyAudioInput->start((int)(AudioConstants::NETWORK_FRAME_MSECS + 0.5f));
1647		− }
	1655	+ // generate audio callbacks at the network sample rate
	1656	+ _dummyAudioInput = new QTimer(this);
	1657	+ connect(_dummyAudioInput, SIGNAL(timeout()), this, SLOT(handleDummyAudioInput()));
	1658	+ _dummyAudioInput->start((int)(AudioConstants::NETWORK_FRAME_MSECS + 0.5f));
	1659	+ }
1648	1660
1649		− return supportedFormat;
1650		−}
	1661	+ return supportedFormat;
	1662	+ }
1651	1663
1652		−void AudioClient::audioInputStateChanged(QAudio::State state) {
	1664	+ void AudioClient::audioInputStateChanged(QAudio::State state) {
1653	1665	#if defined(Q_OS_ANDROID)
1654		− switch (state) {
1655		− case QAudio::StoppedState:
1656		− if (!_audioInput) {
1657		− break;
1658		− }
1659		− // Stopped on purpose
1660		− if (_shouldRestartInputSetup) {
1661		− Lock lock(_deviceMutex);
1662		− _inputDevice = _audioInput->start();
1663		− lock.unlock();
1664		− if (_inputDevice) {
1665		− connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
	1666	+ switch (state) {
	1667	+ case QAudio::StoppedState:
	1668	+ if (!_audioInput) {
	1669	+ break;
1666	1670	}
1667		− }
1668		− break;
1669		− case QAudio::ActiveState:
1670		− break;
1671		− default:
1672		− break;
1673		− }
	1671	+ // Stopped on purpose
	1672	+ if (_shouldRestartInputSetup) {
	1673	+ Lock lock(_deviceMutex);
	1674	+ _inputDevice = _audioInput->start();
	1675	+ lock.unlock();
	1676	+ if (_inputDevice) {
	1677	+ connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput()));
	1678	+ }
	1679	+ }
	1680	+ break;
	1681	+ case QAudio::ActiveState:
	1682	+ break;
	1683	+ default:
	1684	+ break;
	1685	+ }
1674	1686	#endif
1675		−}
	1687	+ }
1676	1688
1677		−void AudioClient::checkInputTimeout() {
	1689	+ void AudioClient::checkInputTimeout() {
1678	1690	#if defined(Q_OS_ANDROID)
1679		− if (_audioInput && _inputReadsSinceLastCheck < MIN_READS_TO_CONSIDER_INPUT_ALIVE) {
1680		− _audioInput->stop();
1681		− } else {
1682		− _inputReadsSinceLastCheck = 0;
1683		− }
	1691	+ if (_audioInput && _inputReadsSinceLastCheck < MIN_READS_TO_CONSIDER_INPUT_ALIVE) {
	1692	+ _audioInput->stop();
	1693	+ } else {
	1694	+ _inputReadsSinceLastCheck = 0;
	1695	+ }
1684	1696	#endif
1685		−}
	1697	+ }
1686	1698
1687		−void AudioClient::setHeadsetPluggedIn(bool pluggedIn) {
	1699	+ void AudioClient::setHeadsetPluggedIn(bool pluggedIn) {
1688	1700	#if defined(Q_OS_ANDROID)
1689		− if (pluggedIn == !_isHeadsetPluggedIn && !_inputDeviceInfo.isNull()) {
1690		− QAndroidJniObject brand = QAndroidJniObject::getStaticObjectField<jstring>("android/os/Build", "BRAND");
1691		− // some samsung phones needs more time to shutdown the previous input device
1692		− if (brand.toString().contains("samsung", Qt::CaseInsensitive)) {
1693		− switchInputToAudioDevice(QAudioDeviceInfo(), true);
1694		− QThread::msleep(200);
1695		− }
	1701	+ if (pluggedIn == !_isHeadsetPluggedIn && !_inputDeviceInfo.isNull()) {
	1702	+ QAndroidJniObject brand = QAndroidJniObject::getStaticObjectField<jstring>("android/os/Build", "BRAND");
	1703	+ // some samsung phones needs more time to shutdown the previous input device
	1704	+ if (brand.toString().contains("samsung", Qt::CaseInsensitive)) {
	1705	+ switchInputToAudioDevice(QAudioDeviceInfo(), true);
	1706	+ QThread::msleep(200);
	1707	+ }
1696	1708
1697		− Setting::Handle<bool> enableAEC(SETTING_AEC_KEY, false);
1698		− bool aecEnabled = enableAEC.get();
	1709	+ Setting::Handle<bool> enableAEC(SETTING_AEC_KEY, false);
	1710	+ bool aecEnabled = enableAEC.get();
1699	1711
1700		− if ((pluggedIn \|\| !aecEnabled) && _inputDeviceInfo.deviceName() != VOICE_RECOGNITION) {
1701		− switchAudioDevice(QAudio::AudioInput, VOICE_RECOGNITION);
1702		− } else if (!pluggedIn && aecEnabled && _inputDeviceInfo.deviceName() != VOICE_COMMUNICATION) {
1703		− switchAudioDevice(QAudio::AudioInput, VOICE_COMMUNICATION);
	1712	+ if ((pluggedIn \|\| !aecEnabled) && _inputDeviceInfo.deviceName() != VOICE_RECOGNITION) {
	1713	+ switchAudioDevice(QAudio::AudioInput, VOICE_RECOGNITION);
	1714	+ } else if (!pluggedIn && aecEnabled && _inputDeviceInfo.deviceName() != VOICE_COMMUNICATION) {
	1715	+ switchAudioDevice(QAudio::AudioInput, VOICE_COMMUNICATION);
	1716	+ }
1704	1717	}
1705		− }
1706		− _isHeadsetPluggedIn = pluggedIn;
	1718	+ _isHeadsetPluggedIn = pluggedIn;
1707	1719	#endif
1708		−}
	1720	+ }
1709	1721
1710		−void AudioClient::outputNotify() {
1711		− int recentUnfulfilled = _audioOutputIODevice.getRecentUnfulfilledReads();
1712		− if (recentUnfulfilled > 0) {
1713		− qCDebug(audioclient, "Starve detected, %d new unfulfilled reads", recentUnfulfilled);
1714		−
1715		− if (_outputStarveDetectionEnabled.get()) {
1716		− quint64 now = usecTimestampNow() / 1000;
1717		− int dt = (int)(now - _outputStarveDetectionStartTimeMsec);
1718		− if (dt > STARVE_DETECTION_PERIOD) {
1719		− _outputStarveDetectionStartTimeMsec = now;
1720		− _outputStarveDetectionCount = 0;
1721		− } else {
1722		− _outputStarveDetectionCount += recentUnfulfilled;
1723		− if (_outputStarveDetectionCount > STARVE_DETECTION_THRESHOLD) {
1724		− int oldOutputBufferSizeFrames = _sessionOutputBufferSizeFrames;
1725		− int newOutputBufferSizeFrames = setOutputBufferSize(oldOutputBufferSizeFrames + 1, false);
1726		−
1727		− if (newOutputBufferSizeFrames > oldOutputBufferSizeFrames) {
1728		− qCDebug(audioclient, "Starve threshold surpassed (%d starves in %d ms)", _outputStarveDetectionCount,
1729		− dt);
1730		− }
	1722	+ void AudioClient::outputNotify() {
	1723	+ int recentUnfulfilled = _audioOutputIODevice.getRecentUnfulfilledReads();
	1724	+ if (recentUnfulfilled > 0) {
	1725	+ qCDebug(audioclient, "Starve detected, %d new unfulfilled reads", recentUnfulfilled);
1731	1726
	1727	+ if (_outputStarveDetectionEnabled.get()) {
	1728	+ quint64 now = usecTimestampNow() / 1000;
	1729	+ int dt = (int)(now - _outputStarveDetectionStartTimeMsec);
	1730	+ if (dt > STARVE_DETECTION_PERIOD) {
1732	1731	_outputStarveDetectionStartTimeMsec = now;
1733	1732	_outputStarveDetectionCount = 0;
	1733	+ } else {
	1734	+ _outputStarveDetectionCount += recentUnfulfilled;
	1735	+ if (_outputStarveDetectionCount > STARVE_DETECTION_THRESHOLD) {
	1736	+ int oldOutputBufferSizeFrames = _sessionOutputBufferSizeFrames;
	1737	+ int newOutputBufferSizeFrames = setOutputBufferSize(oldOutputBufferSizeFrames + 1, false);
	1738	+
	1739	+ if (newOutputBufferSizeFrames > oldOutputBufferSizeFrames) {
	1740	+ qCDebug(audioclient, "Starve threshold surpassed (%d starves in %d ms)",
	1741	+ _outputStarveDetectionCount, dt);
	1742	+ }
	1743	+
	1744	+ _outputStarveDetectionStartTimeMsec = now;
	1745	+ _outputStarveDetectionCount = 0;
	1746	+ }
1734	1747	}
1735	1748	}
1736	1749	}
1737	1750	}
1738		−}
1739	1751
1740		−bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) {
1741		− Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");
	1752	+ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) {
	1753	+ Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread");
1742	1754
1743		− qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName()
1744		− << "]";
1745		− bool supportedFormat = false;
1746		−
1747		− // NOTE: device start() uses the Qt internal device list
1748		− Lock lock(_deviceMutex);
	1755	+ qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName()
	1756	+ << "]";
	1757	+ bool supportedFormat = false;
1749	1758
1750		− Lock localAudioLock(_localAudioMutex);
1751		− _localSamplesAvailable.exchange(0, std::memory_order_release);
	1759	+ // NOTE: device start() uses the Qt internal device list
	1760	+ Lock lock(_deviceMutex);
1752	1761
1753		− // cleanup any previously initialized device
1754		− if (_audioOutput) {
1755		− _audioOutputIODevice.close();
1756		− _audioOutput->stop();
	1762	+ Lock localAudioLock(_localAudioMutex);
	1763	+ _localSamplesAvailable.exchange(0, std::memory_order_release);
1757	1764
1758		− //must be deleted in next eventloop cycle when its called from notify()
1759		− _audioOutput->deleteLater();
1760		− _audioOutput = NULL;
	1765	+ // cleanup any previously initialized device
	1766	+ if (_audioOutput) {
	1767	+ _audioOutputIODevice.close();
	1768	+ _audioOutput->stop();
1761	1769
1762		− _loopbackOutputDevice = NULL;
1763		− //must be deleted in next eventloop cycle when its called from notify()
1764		− _loopbackAudioOutput->deleteLater();
1765		− _loopbackAudioOutput = NULL;
	1770	+ //must be deleted in next eventloop cycle when its called from notify()
	1771	+ _audioOutput->deleteLater();
	1772	+ _audioOutput = NULL;
1766	1773
1767		− delete[] _outputMixBuffer;
1768		− _outputMixBuffer = NULL;
	1774	+ _loopbackOutputDevice = NULL;
	1775	+ //must be deleted in next eventloop cycle when its called from notify()
	1776	+ _loopbackAudioOutput->deleteLater();
	1777	+ _loopbackAudioOutput = NULL;
1769	1778
1770		− delete[] _outputScratchBuffer;
1771		− _outputScratchBuffer = NULL;
	1779	+ delete[] _outputMixBuffer;
	1780	+ _outputMixBuffer = NULL;
1772	1781
1773		− delete[] _localOutputMixBuffer;
1774		− _localOutputMixBuffer = NULL;
	1782	+ delete[] _outputScratchBuffer;
	1783	+ _outputScratchBuffer = NULL;
1775	1784
1776		− _outputDeviceInfo = QAudioDeviceInfo();
1777		− }
	1785	+ delete[] _localOutputMixBuffer;
	1786	+ _localOutputMixBuffer = NULL;
1778	1787
1779		− if (_networkToOutputResampler) {
1780		− // if we were using an input to network resampler, delete it here
1781		− delete _networkToOutputResampler;
1782		− _networkToOutputResampler = NULL;
	1788	+ _outputDeviceInfo = QAudioDeviceInfo();
	1789	+ }
1783	1790
1784		− delete _localToOutputResampler;
1785		− _localToOutputResampler = NULL;
1786		− }
	1791	+ if (_networkToOutputResampler) {
	1792	+ // if we were using an input to network resampler, delete it here
	1793	+ delete _networkToOutputResampler;
	1794	+ _networkToOutputResampler = NULL;
1787	1795
1788		− if (isShutdownRequest) {
1789		− qCDebug(audioclient) << "The audio output device has shut down.";
1790		− return true;
1791		− }
	1796	+ delete _localToOutputResampler;
	1797	+ _localToOutputResampler = NULL;
	1798	+ }
1792	1799
1793		− if (!outputDeviceInfo.isNull()) {
1794		− qCDebug(audioclient) << "The audio output device " << outputDeviceInfo.deviceName() << "is available.";
1795		− _outputDeviceInfo = outputDeviceInfo;
1796		− emit deviceChanged(QAudio::AudioOutput, outputDeviceInfo);
	1800	+ if (isShutdownRequest) {
	1801	+ qCDebug(audioclient) << "The audio output device has shut down.";
	1802	+ return true;
	1803	+ }
1797	1804
1798		− if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {
1799		− qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat;
	1805	+ if (!outputDeviceInfo.isNull()) {
	1806	+ qCDebug(audioclient) << "The audio output device " << outputDeviceInfo.deviceName() << "is available.";
	1807	+ _outputDeviceInfo = outputDeviceInfo;
	1808	+ emit deviceChanged(QAudio::AudioOutput, outputDeviceInfo);
1800	1809
1801		− // we've got the best we can get for input
1802		− // if required, setup a resampler for this input to our desired network format
1803		− if (_desiredOutputFormat != _outputFormat && _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) {
1804		− qCDebug(audioclient) << "Attemping to create a resampler for network format to output format.";
	1810	+ if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {
	1811	+ qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat;
1805	1812
1806		− assert(_desiredOutputFormat.sampleSize() == 16);
1807		− assert(_outputFormat.sampleSize() == 16);
	1813	+ // we've got the best we can get for input
	1814	+ // if required, setup a resampler for this input to our desired network format
	1815	+ if (_desiredOutputFormat != _outputFormat && _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) {
	1816	+ qCDebug(audioclient) << "Attemping to create a resampler for network format to output format.";
1808	1817
1809		− _networkToOutputResampler =
1810		− new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
1811		− _localToOutputResampler =
1812		− new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
	1818	+ assert(_desiredOutputFormat.sampleSize() == 16);
	1819	+ assert(_outputFormat.sampleSize() == 16);
1813	1820
1814		− } else {
1815		− qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
1816		− }
	1821	+ _networkToOutputResampler =
	1822	+ new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
	1823	+ _localToOutputResampler =
	1824	+ new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
1817	1825
1818		− outputFormatChanged();
1819		−
1820		− // setup our general output device for audio-mixer audio
1821		− _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
1822		−
1823		− int deviceChannelCount = _outputFormat.channelCount();
1824		− int frameSize =
1825		− (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * deviceChannelCount * _outputFormat.sampleRate()) /
1826		− _desiredOutputFormat.sampleRate();
1827		− int requestedSize = _sessionOutputBufferSizeFrames * frameSize * AudioConstants::SAMPLE_SIZE;
1828		− _audioOutput->setBufferSize(requestedSize);
1829		−
1830		− // initialize mix buffers on the _audioOutput thread to avoid races
1831		− connect(_audioOutput, &QAudioOutput::stateChanged, [&, frameSize, requestedSize](QAudio::State state) {
1832		− if (state == QAudio::ActiveState) {
1833		− // restrict device callback to _outputPeriod samples
1834		− _outputPeriod = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE;
1835		− // device callback may exceed reported period, so double it to avoid stutter
1836		− _outputPeriod *= 2;
1837		−
1838		− _outputMixBuffer = new float[_outputPeriod];
1839		− _outputScratchBuffer = new int16_t[_outputPeriod];
1840		−
1841		− // size local output mix buffer based on resampled network frame size
1842		− int networkPeriod =
1843		− _localToOutputResampler
1844		− ? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO)
1845		− : AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
1846		− _localOutputMixBuffer = new float[networkPeriod];
1847		−
1848		− // local period should be at least twice the output period,
1849		− // in case two device reads happen before more data can be read (worst case)
1850		− int localPeriod = _outputPeriod * 2;
1851		− // round up to an exact multiple of networkPeriod
1852		− localPeriod = ((localPeriod + networkPeriod - 1) / networkPeriod) * networkPeriod;
1853		− // this ensures lowest latency without stutter from underrun
1854		− _localInjectorsStream.resizeForFrameSize(localPeriod);
1855		−
1856		− int bufferSize = _audioOutput->bufferSize();
1857		− int bufferSamples = bufferSize / AudioConstants::SAMPLE_SIZE;
1858		− int bufferFrames = bufferSamples / (float)frameSize;
1859		− qCDebug(audioclient) << "frame (samples):" << frameSize;
1860		− qCDebug(audioclient) << "buffer (frames):" << bufferFrames;
1861		− qCDebug(audioclient) << "buffer (samples):" << bufferSamples;
1862		− qCDebug(audioclient) << "buffer (bytes):" << bufferSize;
1863		− qCDebug(audioclient) << "requested (bytes):" << requestedSize;
1864		− qCDebug(audioclient) << "period (samples):" << _outputPeriod;
1865		− qCDebug(audioclient) << "local buffer (samples):" << localPeriod;
1866		−
1867		− disconnect(_audioOutput, &QAudioOutput::stateChanged, 0, 0);
1868		−
1869		− // unlock to avoid a deadlock with the device callback (which always succeeds this initialization)
1870		− localAudioLock.unlock();
	1826	+ } else {
	1827	+ qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
1871	1828	}
1872		− });
1873		− connect(_audioOutput, &QAudioOutput::notify, this, &AudioClient::outputNotify);
1874	1829
1875		− _audioOutputIODevice.start();
	1830	+ outputFormatChanged();
	1831	+
	1832	+ // setup our general output device for audio-mixer audio
	1833	+ _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
	1834	+
	1835	+ int deviceChannelCount = _outputFormat.channelCount();
	1836	+ int frameSize =
	1837	+ (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * deviceChannelCount * _outputFormat.sampleRate()) /
	1838	+ _desiredOutputFormat.sampleRate();
	1839	+ int requestedSize = _sessionOutputBufferSizeFrames * frameSize * AudioConstants::SAMPLE_SIZE;
	1840	+ _audioOutput->setBufferSize(requestedSize);
	1841	+
	1842	+ // initialize mix buffers on the _audioOutput thread to avoid races
	1843	+ connect(_audioOutput, &QAudioOutput::stateChanged, [&, frameSize, requestedSize](QAudio::State state) {
	1844	+ if (state == QAudio::ActiveState) {
	1845	+ // restrict device callback to _outputPeriod samples
	1846	+ _outputPeriod = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE;
	1847	+ // device callback may exceed reported period, so double it to avoid stutter
	1848	+ _outputPeriod *= 2;
	1849	+
	1850	+ _outputMixBuffer = new float[_outputPeriod];
	1851	+ _outputScratchBuffer = new int16_t[_outputPeriod];
	1852	+
	1853	+ // size local output mix buffer based on resampled network frame size
	1854	+ int networkPeriod =
	1855	+ _localToOutputResampler
	1856	+ ? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO)
	1857	+ : AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
	1858	+ _localOutputMixBuffer = new float[networkPeriod];
	1859	+
	1860	+ // local period should be at least twice the output period,
	1861	+ // in case two device reads happen before more data can be read (worst case)
	1862	+ int localPeriod = _outputPeriod * 2;
	1863	+ // round up to an exact multiple of networkPeriod
	1864	+ localPeriod = ((localPeriod + networkPeriod - 1) / networkPeriod) * networkPeriod;
	1865	+ // this ensures lowest latency without stutter from underrun
	1866	+ _localInjectorsStream.resizeForFrameSize(localPeriod);
	1867	+
	1868	+ int bufferSize = _audioOutput->bufferSize();
	1869	+ int bufferSamples = bufferSize / AudioConstants::SAMPLE_SIZE;
	1870	+ int bufferFrames = bufferSamples / (float)frameSize;
	1871	+ qCDebug(audioclient) << "frame (samples):" << frameSize;
	1872	+ qCDebug(audioclient) << "buffer (frames):" << bufferFrames;
	1873	+ qCDebug(audioclient) << "buffer (samples):" << bufferSamples;
	1874	+ qCDebug(audioclient) << "buffer (bytes):" << bufferSize;
	1875	+ qCDebug(audioclient) << "requested (bytes):" << requestedSize;
	1876	+ qCDebug(audioclient) << "period (samples):" << _outputPeriod;
	1877	+ qCDebug(audioclient) << "local buffer (samples):" << localPeriod;
	1878	+
	1879	+ disconnect(_audioOutput, &QAudioOutput::stateChanged, 0, 0);
	1880	+
	1881	+ // unlock to avoid a deadlock with the device callback (which always succeeds this initialization)
	1882	+ localAudioLock.unlock();
	1883	+ }
	1884	+ });
	1885	+ connect(_audioOutput, &QAudioOutput::notify, this, &AudioClient::outputNotify);
	1886	+
	1887	+ _audioOutputIODevice.start();
1876	1888
1877		− _audioOutput->start(&_audioOutputIODevice);
	1889	+ _audioOutput->start(&_audioOutputIODevice);
1878	1890
1879		− // setup a loopback audio output device
1880		− _loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
	1891	+ // setup a loopback audio output device
	1892	+ _loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
1881	1893
1882		− _timeSinceLastReceived.start();
	1894	+ _timeSinceLastReceived.start();
1883	1895
1884		− supportedFormat = true;
	1896	+ supportedFormat = true;
	1897	+ }
1885	1898	}
1886		− }
1887	1899
1888		− return supportedFormat;
1889		−}
	1900	+ return supportedFormat;
	1901	+ }
1890	1902
1891		−int AudioClient::setOutputBufferSize(int numFrames, bool persist) {
1892		− qCDebug(audioclient) << __FUNCTION__ << "numFrames:" << numFrames << "persist:" << persist;
	1903	+ int AudioClient::setOutputBufferSize(int numFrames, bool persist) {
	1904	+ qCDebug(audioclient) << __FUNCTION__ << "numFrames:" << numFrames << "persist:" << persist;
1893	1905
1894		− numFrames = std::min(std::max(numFrames, MIN_BUFFER_FRAMES), MAX_BUFFER_FRAMES);
1895		− qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames
1896		− << "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames;
	1906	+ numFrames = std::min(std::max(numFrames, MIN_BUFFER_FRAMES), MAX_BUFFER_FRAMES);
	1907	+ qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames
	1908	+ << "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames;
1897	1909
1898		− if (numFrames != _sessionOutputBufferSizeFrames) {
1899		− qCInfo(audioclient, "Audio output buffer set to %d frames", numFrames);
1900		− _sessionOutputBufferSizeFrames = numFrames;
1901		− if (persist) {
1902		− _outputBufferSizeFrames.set(numFrames);
	1910	+ if (numFrames != _sessionOutputBufferSizeFrames) {
	1911	+ qCInfo(audioclient, "Audio output buffer set to %d frames", numFrames);
	1912	+ _sessionOutputBufferSizeFrames = numFrames;
	1913	+ if (persist) {
	1914	+ _outputBufferSizeFrames.set(numFrames);
	1915	+ }
1903	1916	}
	1917	+ return numFrames;
1904	1918	}
1905		− return numFrames;
1906		−}
1907	1919
1908		−// The following constant is operating system dependent due to differences in
1909		−// the way input audio is handled. The audio input buffer size is inversely
1910		−// proportional to the accelerator ratio.
	1920	+ // The following constant is operating system dependent due to differences in
	1921	+ // the way input audio is handled. The audio input buffer size is inversely
	1922	+ // proportional to the accelerator ratio.
1911	1923
1912	1924	#ifdef Q_OS_WIN
1913		−const float AudioClient::CALLBACK_ACCELERATOR_RATIO = IsWindows8OrGreater() ? 1.0f : 0.25f;
	1925	+ const float AudioClient::CALLBACK_ACCELERATOR_RATIO = IsWindows8OrGreater() ? 1.0f : 0.25f;
1914	1926	#endif
1915	1927
1916	1928	#ifdef Q_OS_MAC
1917		−const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;
	1929	+ const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;
1918	1930	#endif
1919	1931
1920	1932	#ifdef Q_OS_ANDROID
1921		−const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 0.5f;
	1933	+ const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 0.5f;
1922	1934	#elif defined(Q_OS_LINUX)
1923		−const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;
	1935	+ const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f;
1924	1936	#endif
1925	1937
1926		−int AudioClient::calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const {
1927		− int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * format.channelCount() *
1928		− ((float)format.sampleRate() / AudioConstants::SAMPLE_RATE)) /
1929		− CALLBACK_ACCELERATOR_RATIO) +
1930		− 0.5f);
	1938	+ int AudioClient::calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const {
	1939	+ int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * format.channelCount() *
	1940	+ ((float)format.sampleRate() / AudioConstants::SAMPLE_RATE)) /
	1941	+ CALLBACK_ACCELERATOR_RATIO) +
	1942	+ 0.5f);
1931	1943
1932		− return numInputCallbackBytes;
1933		−}
	1944	+ return numInputCallbackBytes;
	1945	+ }
1934	1946
1935		−int AudioClient::calculateNumberOfFrameSamples(int numBytes) const {
1936		− int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / AudioConstants::SAMPLE_SIZE;
1937		− return frameSamples;
1938		−}
	1947	+ int AudioClient::calculateNumberOfFrameSamples(int numBytes) const {
	1948	+ int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / AudioConstants::SAMPLE_SIZE;
	1949	+ return frameSamples;
	1950	+ }
1939	1951
1940		−float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
1941		− glm::quat inverseOrientation = glm::inverse(_orientationGetter());
	1952	+ float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
	1953	+ glm::quat inverseOrientation = glm::inverse(_orientationGetter());
1942	1954
1943		− glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;
	1955	+ glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;
1944	1956
1945		− // project the rotated source position vector onto the XZ plane
1946		− rotatedSourcePosition.y = 0.0f;
	1957	+ // project the rotated source position vector onto the XZ plane
	1958	+ rotatedSourcePosition.y = 0.0f;
1947	1959
1948		− static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
	1960	+ static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
1949	1961
1950		− float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition);
1951		− if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) {
1952		− // produce an oriented angle about the y-axis
1953		− glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2));
1954		− float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"
1955		− return (direction.x < 0.0f) ? -angle : angle;
	1962	+ float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition);
	1963	+ if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) {
	1964	+ // produce an oriented angle about the y-axis
	1965	+ glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2));
	1966	+ float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"
	1967	+ return (direction.x < 0.0f) ? -angle : angle;
1956	1968
1957		− } else {
1958		− // no azimuth if they are in same spot
1959		− return 0.0f;
	1969	+ } else {
	1970	+ // no azimuth if they are in same spot
	1971	+ return 0.0f;
	1972	+ }
1960	1973	}
1961		−}
1962	1974
1963		−float AudioClient::gainForSource(float distance, float volume) {
1964		− // attenuation = -6dB * log2(distance)
1965		− // reference attenuation of 0dB at distance = 1.0m
1966		− float gain = volume / std::max(distance, HRTF_NEARFIELD_MIN);
	1975	+ float AudioClient::gainForSource(float distance, float volume) {
	1976	+ // attenuation = -6dB * log2(distance)
	1977	+ // reference attenuation of 0dB at distance = 1.0m
	1978	+ float gain = volume / std::max(distance, HRTF_NEARFIELD_MIN);
1967	1979
1968		− return gain;
1969		−}
	1980	+ return gain;
	1981	+ }
1970	1982
1971		−qint64 AudioClient::AudioOutputIODevice::readData(char* data, qint64 maxSize) {
1972		− // samples requested from OUTPUT_CHANNEL_COUNT
1973		− int deviceChannelCount = _audio->_outputFormat.channelCount();
1974		− int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
1975		− // restrict samplesRequested to the size of our mix/scratch buffers
1976		− samplesRequested = std::min(samplesRequested, _audio->_outputPeriod);
	1983	+ qint64 AudioClient::AudioOutputIODevice::readData(char* data, qint64 maxSize) {
	1984	+ // samples requested from OUTPUT_CHANNEL_COUNT
	1985	+ int deviceChannelCount = _audio->_outputFormat.channelCount();
	1986	+ int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
	1987	+ // restrict samplesRequested to the size of our mix/scratch buffers
	1988	+ samplesRequested = std::min(samplesRequested, _audio->_outputPeriod);
1977	1989
1978		− int16_t* scratchBuffer = _audio->_outputScratchBuffer;
1979		− float* mixBuffer = _audio->_outputMixBuffer;
	1990	+ int16_t* scratchBuffer = _audio->_outputScratchBuffer;
	1991	+ float* mixBuffer = _audio->_outputMixBuffer;
1980	1992
1981		− int networkSamplesPopped;
1982		− if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
1983		− qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped,
1984		− _receivedAudioStream.getSamplesAvailable(), samplesRequested);
1985		− AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
1986		− lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped);
1987		− for (int i = 0; i < networkSamplesPopped; i++) {
1988		− mixBuffer[i] = convertToFloat(scratchBuffer[i]);
	1993	+ int networkSamplesPopped;
	1994	+ if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
	1995	+ qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped,
	1996	+ _receivedAudioStream.getSamplesAvailable(), samplesRequested);
	1997	+ AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
	1998	+ lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped);
	1999	+ for (int i = 0; i < networkSamplesPopped; i++) {
	2000	+ mixBuffer[i] = convertToFloat(scratchBuffer[i]);
	2001	+ }
	2002	+ samplesRequested = networkSamplesPopped;
1989	2003	}
1990		− samplesRequested = networkSamplesPopped;
1991		− }
1992	2004
1993		− int injectorSamplesPopped = 0;
1994		− {
1995		− bool append = networkSamplesPopped > 0;
1996		− // check the samples we have available locklessly; this is possible because only two functions add to the count:
1997		− // - prepareLocalAudioInjectors will only increase samples count
1998		− // - switchOutputToAudioDevice will zero samples count,
1999		− // stop the device - so that readData will exhaust the existing buffer or see a zeroed samples count,
2000		− // and start the device - which can then only see a zeroed samples count
2001		− int samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);
2002		−
2003		− // if we do not have enough samples buffered despite having injectors, buffer them synchronously
2004		− if (samplesAvailable < samplesRequested && _audio->_localInjectorsAvailable.load(std::memory_order_acquire)) {
2005		− // try_to_lock, in case the device is being shut down already
2006		− std::unique_ptr<Lock> localAudioLock(new Lock(_audio->_localAudioMutex, std::try_to_lock));
2007		− if (localAudioLock->owns_lock()) {
2008		− _audio->prepareLocalAudioInjectors(std::move(localAudioLock));
2009		− samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);
	2005	+ int injectorSamplesPopped = 0;
	2006	+ {
	2007	+ bool append = networkSamplesPopped > 0;
	2008	+ // check the samples we have available locklessly; this is possible because only two functions add to the count:
	2009	+ // - prepareLocalAudioInjectors will only increase samples count
	2010	+ // - switchOutputToAudioDevice will zero samples count,
	2011	+ // stop the device - so that readData will exhaust the existing buffer or see a zeroed samples count,
	2012	+ // and start the device - which can then only see a zeroed samples count
	2013	+ int samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);
	2014	+
	2015	+ // if we do not have enough samples buffered despite having injectors, buffer them synchronously
	2016	+ if (samplesAvailable < samplesRequested && _audio->_localInjectorsAvailable.load(std::memory_order_acquire)) {
	2017	+ // try_to_lock, in case the device is being shut down already
	2018	+ std::unique_ptr<Lock> localAudioLock(new Lock(_audio->_localAudioMutex, std::try_to_lock));
	2019	+ if (localAudioLock->owns_lock()) {
	2020	+ _audio->prepareLocalAudioInjectors(std::move(localAudioLock));
	2021	+ samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire);
	2022	+ }
2010	2023	}
2011		− }
2012	2024
2013		− samplesRequested = std::min(samplesRequested, samplesAvailable);
2014		− if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {
2015		− _audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release);
2016		− qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped,
2017		− _localInjectorsStream.samplesAvailable(), samplesRequested);
	2025	+ samplesRequested = std::min(samplesRequested, samplesAvailable);
	2026	+ if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {
	2027	+ _audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release);
	2028	+ qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped,
	2029	+ _localInjectorsStream.samplesAvailable(), samplesRequested);
	2030	+ }
2018	2031	}
2019		− }
2020	2032
2021		− // prepare injectors for the next callback
2022		− QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); });
2023		−
2024		− int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
2025		− int framesPopped = samplesPopped / AudioConstants::STEREO;
2026		− int bytesWritten;
2027		− if (samplesPopped > 0) {
2028		− if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
2029		− // limit the audio
2030		− _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped);
2031		− } else {
2032		− _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
	2033	+ // prepare injectors for the next callback
	2034	+ QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); });
2033	2035
2034		− // upmix or downmix to deviceChannelCount
2035		− if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
2036		− int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;
2037		− channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);
	2036	+ int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
	2037	+ int framesPopped = samplesPopped / AudioConstants::STEREO;
	2038	+ int bytesWritten;
	2039	+ if (samplesPopped > 0) {
	2040	+ if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
	2041	+ // limit the audio
	2042	+ _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped);
2038	2043	} else {
2039		− channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);
	2044	+ _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
	2045	+
	2046	+ // upmix or downmix to deviceChannelCount
	2047	+ if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
	2048	+ int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;
	2049	+ channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);
	2050	+ } else {
	2051	+ channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);
	2052	+ }
2040	2053	}
	2054	+
	2055	+ bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
	2056	+ } else {
	2057	+ // nothing on network, don't grab anything from injectors, and just return 0s
	2058	+ memset(data, 0, maxSize);
	2059	+ bytesWritten = maxSize;
2041	2060	}
2042	2061
2043		− bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
2044		− } else {
2045		− // nothing on network, don't grab anything from injectors, and just return 0s
2046		− memset(data, 0, maxSize);
2047		− bytesWritten = maxSize;
2048		− }
	2062	+ // send output buffer for recording
	2063	+ if (_audio->_isRecording) {
	2064	+ Lock lock(_recordMutex);
	2065	+ _audio->_audioFileWav.addRawAudioChunk(reinterpret_cast<char*>(scratchBuffer), bytesWritten);
	2066	+ }
2049	2067
2050		− // send output buffer for recording
2051		− if (_audio->_isRecording) {
2052		− Lock lock(_recordMutex);
2053		− _audio->_audioFileWav.addRawAudioChunk(reinterpret_cast<char*>(scratchBuffer), bytesWritten);
2054		− }
	2068	+ int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
	2069	+ float msecsAudioOutputUnplayed =
	2070	+ bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC);
	2071	+ _audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed);
2055	2072
2056		− int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
2057		− float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC);
2058		− _audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed);
	2073	+ if (bytesAudioOutputUnplayed == 0) {
	2074	+ _unfulfilledReads++;
	2075	+ }
2059	2076
2060		− if (bytesAudioOutputUnplayed == 0) {
2061		− _unfulfilledReads++;
	2077	+ return bytesWritten;
2062	2078	}
2063	2079
2064		− return bytesWritten;
2065		−}
2066		−
2067		−bool AudioClient::startRecording(const QString& filepath) {
2068		− if (!_audioFileWav.create(_outputFormat, filepath)) {
2069		− qDebug() << "Error creating audio file: " + filepath;
2070		− return false;
	2080	+ bool AudioClient::startRecording(const QString& filepath) {
	2081	+ if (!_audioFileWav.create(_outputFormat, filepath)) {
	2082	+ qDebug() << "Error creating audio file: " + filepath;
	2083	+ return false;
	2084	+ }
	2085	+ _isRecording = true;
	2086	+ return true;
2071	2087	}
2072		− _isRecording = true;
2073		− return true;
2074		−}
2075	2088
2076		−void AudioClient::stopRecording() {
2077		− if (_isRecording) {
2078		− _isRecording = false;
2079		− _audioFileWav.close();
	2089	+ void AudioClient::stopRecording() {
	2090	+ if (_isRecording) {
	2091	+ _isRecording = false;
	2092	+ _audioFileWav.close();
	2093	+ }
2080	2094	}
2081		−}
2082	2095
2083		−void AudioClient::loadSettings() {
2084		− _receivedAudioStream.setDynamicJitterBufferEnabled(dynamicJitterBufferEnabled.get());
2085		− _receivedAudioStream.setStaticJitterBufferFrames(staticJitterBufferFrames.get());
	2096	+ void AudioClient::loadSettings() {
	2097	+ _receivedAudioStream.setDynamicJitterBufferEnabled(dynamicJitterBufferEnabled.get());
	2098	+ _receivedAudioStream.setStaticJitterBufferFrames(staticJitterBufferFrames.get());
2086	2099
2087		− qCDebug(audioclient) << "---- Initializing Audio Client ----";
2088		− auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
2089		− for (auto& plugin : codecPlugins) {
2090		− qCDebug(audioclient) << "Codec available:" << plugin->getName();
	2100	+ qCDebug(audioclient) << "---- Initializing Audio Client ----";
	2101	+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
	2102	+ for (auto& plugin : codecPlugins) {
	2103	+ qCDebug(audioclient) << "Codec available:" << plugin->getName();
	2104	+ }
2091	2105	}
2092		−}
2093	2106
2094		−void AudioClient::saveSettings() {
2095		− dynamicJitterBufferEnabled.set(_receivedAudioStream.dynamicJitterBufferEnabled());
2096		− staticJitterBufferFrames.set(_receivedAudioStream.getStaticJitterBufferFrames());
2097		−}
	2107	+ void AudioClient::saveSettings() {
	2108	+ dynamicJitterBufferEnabled.set(_receivedAudioStream.dynamicJitterBufferEnabled());
	2109	+ staticJitterBufferFrames.set(_receivedAudioStream.getStaticJitterBufferFrames());
	2110	+ }
2098	2111
2099		−void AudioClient::setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale) {
2100		− avatarBoundingBoxCorner = corner;
2101		− avatarBoundingBoxScale = scale;
2102		−}
	2112	+ void AudioClient::setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale) {
	2113	+ avatarBoundingBoxCorner = corner;
	2114	+ avatarBoundingBoxScale = scale;
	2115	+ }
2103	2116
2104		−void AudioClient::startThread() {
2105		− moveToNewNamedThread(this, "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority);
2106		−}
	2117	+ void AudioClient::startThread() {
	2118	+ moveToNewNamedThread(this, "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority);
	2119	+ }
2107	2120
2108		−void AudioClient::setInputVolume(float volume, bool emitSignal) {
2109		− if (_audioInput && volume != (float)_audioInput->volume()) {
2110		− _audioInput->setVolume(volume);
2111		− if (emitSignal) {
2112		− emit inputVolumeChanged(_audioInput->volume());
	2121	+ void AudioClient::setInputVolume(float volume, bool emitSignal) {
	2122	+ if (_audioInput && volume != (float)_audioInput->volume()) {
	2123	+ _audioInput->setVolume(volume);
	2124	+ if (emitSignal) {
	2125	+ emit inputVolumeChanged(_audioInput->volume());
	2126	+ }
2113	2127	}
2114	2128	}
2115		−}

libraries/audio-client/src/AudioClient.h+12−7

@@ -197,7 +197,11 @@ public slots: · 196 unmodified lines
197	197	void checkInputTimeout();
198	198	void handleDummyAudioInput();
199	199	void handleRecordedAudioInput(const QByteArray& audio);
200		− void handleTTSAudioInput(const QByteArray& audio);
	200	+ void handleTTSAudioInput(const QByteArray& audio,
	201	+ const int& newChunkSize,
	202	+ const int& timerInterval);
	203	+ void clearTTSBuffer();
	204	+ void processTTSBuffer();
201	205	void reset();
202	206	void audioMixerKilled();
203	207
@@ -289,11 +293,12 @@ private: · 85 unmodified lines
289	293	bool mixLocalAudioInjectors(float* mixBuffer);
290	294	float azimuthForSource(const glm::vec3& relativePosition);
291	295	float gainForSource(float distance, float volume);
292		−
293		− void processAudioAndAddToRingBuffer(QByteArray& inputByteArray,
294		− const uchar& channelCount,
295		− const qint32& bytesForDuration,
296		− QByteArray& rollingBuffer);
	296	+
	297	+ Mutex _TTSMutex;
	298	+ QTimer _TTSTimer;
	299	+ bool _isProcessingTTS {false};
	300	+ QByteArray _TTSAudioBuffer;
	301	+ int _TTSChunkSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * 50;
297	302
298	303	#ifdef Q_OS_ANDROID
299	304	QTimer _checkInputTimer;
@@ -401,7 +406,7 @@ private: · 101 unmodified lines
401	406	void configureReverb();
402	407	void updateReverbOptions();
403	408
404		− void handleLocalEchoAndReverb(QByteArray& inputByteArray);
	409	+ void handleLocalEchoAndReverb(QByteArray& inputByteArray, const int& sampleRate, const int& channelCount);
405	410
406	411	bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);
407	412	bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);