//// AudioClient.cpp// libraries/audio-client/src//// Created by Stephen Birarda on 1/22/13.// Copyright 2013 High Fidelity, Inc.// Copyright 2021 Vircadia contributors.// Copyright 2023-2025 Overte e.V.//// Distributed under the Apache License, Version 2.0.// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// SPDX-License-Identifier: Apache-2.0// #include "AudioClient.h" #include <cmath>#include <cstring>#include <math.h>#include <sys/stat.h> #include <glm/glm.hpp>#include <glm/gtx/norm.hpp>#include <glm/gtx/vector_angle.hpp> #ifdef __APPLE__#include <CoreAudio/AudioHardware.h>#endif #ifdef _WIN32#ifndef WIN32_LEAN_AND_MEAN#define WIN32_LEAN_AND_MEAN 1#endif#include <windows.h>#include <Mmsystem.h>#include <mmdeviceapi.h>#include <devicetopology.h>#include <Functiondiscoverykeys_devpkey.h>#include <VersionHelpers.h>#endif #include <QtConcurrent/QtConcurrent>#include <QtCore/QThreadPool>#include <QtCore/QBuffer>#include <QtMultimedia/QAudioInput>#include <QtMultimedia/QAudioOutput> #include <shared/QtHelpers.h>#include <ThreadHelpers.h>#include <NodeList.h>#include <plugins/CodecPlugin.h>#include <plugins/PluginManager.h>#include <udt/PacketHeaders.h>#include <SettingHandle.h>#include <SharedUtil.h>#include <Transform.h> #include "AudioClientLogging.h"#include "AudioLogging.h"#include "AudioHelpers.h" #if defined(Q_OS_ANDROID)#include <QtAndroidExtras/QAndroidJniObject>#endif const int AudioClient::MIN_BUFFER_FRAMES = 1; const int AudioClient::MAX_BUFFER_FRAMES = 20; #if defined(Q_OS_ANDROID)static const int CHECK_INPUT_READS_MSECS = 2000;static const int MIN_READS_TO_CONSIDER_INPUT_ALIVE = 10;#endif const AudioClient::AudioPositionGetter AudioClient::DEFAULT_POSITION_GETTER = []{ return Vectors::ZERO; };const AudioClient::AudioOrientationGetter AudioClient::DEFAULT_ORIENTATION_GETTER = [] { return Quaternions::IDENTITY; }; Setting::Handle<bool> dynamicJitterBufferEnabled("dynamicJitterBuffersEnabled", InboundAudioStream::DEFAULT_DYNAMIC_JITTER_BUFFER_ENABLED);Setting::Handle<int> staticJitterBufferFrames("staticJitterBufferFrames", InboundAudioStream::DEFAULT_STATIC_JITTER_FRAMES); // protect the Qt internal device listusing Mutex = std::mutex;using Lock = std::unique_lock<Mutex>;Mutex _deviceMutex;Mutex _recordMutex; QString defaultAudioDeviceName(QAudio::Mode mode); void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { QWriteLocker lock(&_hmdNameLock); if (mode == QAudio::AudioInput) { _hmdInputName = name; } else { _hmdOutputName = name; }} // thread-safeQList<HifiAudioDeviceInfo> getAvailableDevices(QAudio::Mode mode, const QString& hmdName) { //get hmd device name prior to locking device mutex. in case of shutdown, this thread will be locked and audio client //cannot properly shut down. QString defDeviceName = defaultAudioDeviceName(mode); // NOTE: availableDevices() clobbers the Qt internal device list Lock lock(_deviceMutex); auto devices = QAudioDeviceInfo::availableDevices(mode); HifiAudioDeviceInfo defaultDesktopDevice; QList<HifiAudioDeviceInfo> newDevices; for (auto& device : devices) { newDevices.push_back(HifiAudioDeviceInfo(device, false, mode)); if (device.deviceName() == defDeviceName.trimmed()) { defaultDesktopDevice = HifiAudioDeviceInfo(device, true, mode, HifiAudioDeviceInfo::both); } } if (defaultDesktopDevice.getDevice().isNull()) { if (devices.size() > 0) { qCDebug(audioclient) << __FUNCTION__ << "Default device not found in list:" << defDeviceName << "Setting Default to: " << devices.first().deviceName(); newDevices.push_front(HifiAudioDeviceInfo(devices.first(), true, mode, HifiAudioDeviceInfo::both)); } else { //current audio list is empty for some reason. qCWarning(audioclient) << __FUNCTION__ << "Default device not found in list and no alternative selection available"; } } else { newDevices.push_front(defaultDesktopDevice); } if (!hmdName.isNull()) { HifiAudioDeviceInfo hmdDevice; foreach(auto device, newDevices) { if (device.getDevice().deviceName() == hmdName) { hmdDevice = HifiAudioDeviceInfo(device.getDevice(), true, mode, HifiAudioDeviceInfo::hmd); break; } } if (!hmdDevice.getDevice().isNull()) { newDevices.push_front(hmdDevice); } } return newDevices;} // now called from a background thread, to keep blocking operations off the audio threadvoid AudioClient::checkDevices() { // Make sure we're not shutting down Lock timerMutex(_checkDevicesMutex); // If we HAVE shut down after we were queued, but prior to execution, early exit if (nullptr == _checkDevicesTimer) { return; } QString hmdInputName; QString hmdOutputName; { QReadLocker readLock(&_hmdNameLock); hmdInputName = _hmdInputName; hmdOutputName = _hmdOutputName; } auto inputDevices = getAvailableDevices(QAudio::AudioInput, hmdInputName); auto outputDevices = getAvailableDevices(QAudio::AudioOutput, hmdOutputName); static const QMetaMethod devicesChangedSig= QMetaMethod::fromSignal(&AudioClient::devicesChanged); //only emit once the scripting interface has connected to the signal if (isSignalConnected(devicesChangedSig)) { Lock lock(_deviceMutex); if (inputDevices != _inputDevices) { _inputDevices.swap(inputDevices); emit devicesChanged(QAudio::AudioInput, _inputDevices); } if (outputDevices != _outputDevices) { _outputDevices.swap(outputDevices); emit devicesChanged(QAudio::AudioOutput, _outputDevices); } } } HifiAudioDeviceInfo AudioClient::getActiveAudioDevice(QAudio::Mode mode) const { Lock lock(_deviceMutex); if (mode == QAudio::AudioInput) { return _inputDeviceInfo; } else { return _outputDeviceInfo; }} QList<HifiAudioDeviceInfo> AudioClient::getAudioDevices(QAudio::Mode mode) const { Lock lock(_deviceMutex); if (mode == QAudio::AudioInput) { return _inputDevices; } else { return _outputDevices; }} static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { for (int i = 0; i < numSamples/2; i++) { // read 2 samples int16_t left = *source++; int16_t right = *source++; // write 2 + N samples *dest++ = left; *dest++ = right; for (int n = 0; n < numExtraChannels; n++) { *dest++ = 0; } }} static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { for (int i = 0; i < numSamples/2; i++) { // read 2 samples int16_t left = *source++; int16_t right = *source++; // write 1 sample *dest++ = (int16_t)((left + right) / 2); }} static bool detectClipping(int16_t* samples, int numSamples, int numChannels) { const int32_t CLIPPING_THRESHOLD = 32392; // -0.1 dBFS const int CLIPPING_DETECTION = 3; // consecutive samples over threshold bool isClipping = false; if (numChannels == 2) { int oversLeft = 0; int oversRight = 0; for (int i = 0; i < numSamples/2; i++) { int32_t left = std::abs((int32_t)samples[2*i+0]); int32_t right = std::abs((int32_t)samples[2*i+1]); if (left > CLIPPING_THRESHOLD) { isClipping |= (++oversLeft >= CLIPPING_DETECTION); } else { oversLeft = 0; } if (right > CLIPPING_THRESHOLD) { isClipping |= (++oversRight >= CLIPPING_DETECTION); } else { oversRight = 0; } } } else { int overs = 0; for (int i = 0; i < numSamples; i++) { int32_t sample = std::abs((int32_t)samples[i]); if (sample > CLIPPING_THRESHOLD) { isClipping |= (++overs >= CLIPPING_DETECTION); } else { overs = 0; } } } return isClipping;} static float computeLoudness(int16_t* samples, int numSamples) { float scale = numSamples ? 1.0f / numSamples : 0.0f; int32_t loudness = 0; for (int i = 0; i < numSamples; i++) { loudness += std::abs((int32_t)samples[i]); } return (float)loudness * scale;} template <int NUM_CHANNELS>static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float gain1) { // fast path for unity gain if (gain0 == 1.0f && gain1 == 1.0f) { return; } // cubic poly from gain0 to gain1 float c3 = -2.0f * (gain1 - gain0); float c2 = 3.0f * (gain1 - gain0); float c0 = gain0; float t = 0.0f; float tStep = 1.0f / numFrames; for (int i = 0; i < numFrames; i++) { // evaluate poly over t=[0,1) float gain = (c3 * t + c2) * t * t + c0; t += tStep; // apply gain to all channels for (int ch = 0; ch < NUM_CHANNELS; ch++) { buffer[NUM_CHANNELS*i + ch] *= gain; } }} static inline float convertToFloat(int16_t sample) { return (float)sample * (1 / 32768.0f);} AudioClient::AudioClient() { // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); // Set up the desired audio format, since scripting API expects it to be set and audio scripting API // is initialized before audio thread starts. _desiredInputFormat.setSampleRate(AudioConstants::SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); _desiredInputFormat.setCodec("audio/pcm"); _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); _desiredInputFormat.setChannelCount(1); _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); // deprecate legacy settings { Setting::Handle<int>::Deprecated("maxFramesOverDesired", InboundAudioStream::MAX_FRAMES_OVER_DESIRED); Setting::Handle<int>::Deprecated("windowStarveThreshold", InboundAudioStream::WINDOW_STARVE_THRESHOLD); Setting::Handle<int>::Deprecated("windowSecondsForDesiredCalcOnTooManyStarves", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES); Setting::Handle<int>::Deprecated("windowSecondsForDesiredReduction", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_REDUCTION); Setting::Handle<bool>::Deprecated("useStDevForJitterCalc", InboundAudioStream::USE_STDEV_FOR_JITTER); Setting::Handle<bool>::Deprecated("repetitionWithFade", InboundAudioStream::REPETITION_WITH_FADE); } connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &AudioClient::processReceivedSamples, Qt::DirectConnection); connect(this, &AudioClient::changeDevice, this, [=, this](const HifiAudioDeviceInfo& outputDeviceInfo) { qCDebug(audioclient)<< "got AudioClient::changeDevice signal, about to call switchOutputToAudioDevice() outputDeviceInfo: ["<< outputDeviceInfo.deviceName() << "]"; switchOutputToAudioDevice(outputDeviceInfo); }); connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash defaultAudioDeviceName(QAudio::AudioInput); defaultAudioDeviceName(QAudio::AudioOutput); checkDevices(); // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000; connect(_checkDevicesTimer, &QTimer::timeout, this, [=, this] { QtConcurrent::run(QThreadPool::globalInstance(), [=, this] { checkDevices(); // On some systems (Ubuntu) checking all the audio devices can take more than 2 seconds. To // avoid consuming all of the thread pool, don't start the check interval until the previous // check has completed. QMetaObject::invokeMethod(_checkDevicesTimer, "start", Q_ARG(int, DEVICE_CHECK_INTERVAL_MSECS)); }); }); _checkDevicesTimer->setSingleShot(true); _checkDevicesTimer->start(DEVICE_CHECK_INTERVAL_MSECS); // start a thread to detect peak value changes _checkPeakValuesTimer = new QTimer(this); connect(_checkPeakValuesTimer, &QTimer::timeout, this, [this] { QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); }); }); const unsigned long PEAK_VALUES_CHECK_INTERVAL_MSECS = 50; _checkPeakValuesTimer->start(PEAK_VALUES_CHECK_INTERVAL_MSECS); configureReverb(); #if defined(WEBRTC_AUDIO) configureWebrtc();#endif auto nodeList = DependencyManager::get<NodeList>(); auto& packetReceiver = nodeList->getPacketReceiver(); packetReceiver.registerListener(PacketType::AudioStreamStats, PacketReceiver::makeSourcedListenerReference<AudioIOStats>(&_stats, &AudioIOStats::processStreamStatsPacket)); packetReceiver.registerListener(PacketType::AudioEnvironment, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleAudioEnvironmentDataPacket)); packetReceiver.registerListener(PacketType::SilentAudioFrame, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleAudioDataPacket)); packetReceiver.registerListener(PacketType::MixedAudio, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleAudioDataPacket)); packetReceiver.registerListener(PacketType::NoisyMute, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleNoisyMutePacket)); packetReceiver.registerListener(PacketType::MuteEnvironment, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleMuteEnvironmentPacket)); packetReceiver.registerListener(PacketType::SelectedAudioFormat, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleSelectedAudioFormat)); auto& domainHandler = nodeList->getDomainHandler(); connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { _solo.reset(); }); connect(nodeList.data(), &NodeList::nodeActivated, this, [this](SharedNodePointer node) { if (node->getType() == NodeType::AudioMixer) { _solo.resend(); } });} AudioClient::~AudioClient() { stop(); if (_codec && _encoder) { _codec->releaseEncoder(_encoder); _encoder = nullptr; }} void AudioClient::customDeleter() {#if defined(Q_OS_ANDROID) _shouldRestartInputSetup = false;#endif deleteLater();} void AudioClient::handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec) { qCDebug(audioclient) << __FUNCTION__ << "sendingNode:" << *node << "currentCodec:" << currentCodec << "recievedCodec:" << recievedCodec; selectAudioFormat(recievedCodec);} void AudioClient::reset() { _receivedAudioStream.reset(); _stats.reset(); _sourceReverb.reset(); _listenerReverb.reset(); _localReverb.reset();} void AudioClient::audioMixerKilled() { _hasReceivedFirstPacket = false; _outgoingAvatarAudioSequenceNumber = 0; _stats.reset(); emit disconnected();} void AudioClient::setAudioPaused(bool pause) { if (_audioPaused != pause) { _audioPaused = pause; if (!_audioPaused) { negotiateAudioFormat(); } }} HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { if ((!isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::hmd) || (isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::desktop)) { result = audioDevice; break; } } } return result;} #ifdef Q_OS_WINQString getWinDeviceName(IMMDevice* pEndpoint) { QString deviceName; IPropertyStore* pPropertyStore; pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore); PROPVARIANT pv; PropVariantInit(&pv); HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); pPropertyStore->Release(); pPropertyStore = nullptr; deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal); if (!IsWindows8OrGreater()) { // Windows 7 provides only the 31 first characters of the device name. const DWORD QT_WIN7_MAX_AUDIO_DEVICENAME_LEN = 31; deviceName = deviceName.left(QT_WIN7_MAX_AUDIO_DEVICENAME_LEN); } PropVariantClear(&pv); return deviceName;} QString AudioClient::getWinDeviceName(wchar_t* guid) { QString deviceName; HRESULT hr = S_OK;Lines 1–500 of 2529. Load more
// // AudioClient.cpp // libraries/audio-client/src // // Created by Stephen Birarda on 1/22/13. // Copyright 2013 High Fidelity, Inc. // Copyright 2021 Vircadia contributors. // Copyright 2023-2025 Overte e.V. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // SPDX-License-Identifier: Apache-2.0 // #include "AudioClient.h" #include <cmath> #include <cstring> #include <math.h> #include <sys/stat.h> #include <glm/glm.hpp> #include <glm/gtx/norm.hpp> #include <glm/gtx/vector_angle.hpp> #ifdef __APPLE__ #include <CoreAudio/AudioHardware.h> #endif #ifdef _WIN32 #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN 1 #endif #include <windows.h> #include <Mmsystem.h> #include <mmdeviceapi.h> #include <devicetopology.h> #include <Functiondiscoverykeys_devpkey.h> #include <VersionHelpers.h> #endif #include <QtConcurrent/QtConcurrent> #include <QtCore/QThreadPool> #include <QtCore/QBuffer> #include <QtMultimedia/QAudioInput> #include <QtMultimedia/QAudioOutput> #include <shared/QtHelpers.h> #include <ThreadHelpers.h> #include <NodeList.h> #include <plugins/CodecPlugin.h> #include <plugins/PluginManager.h> #include <udt/PacketHeaders.h> #include <SettingHandle.h> #include <SharedUtil.h> #include <Transform.h> #include "AudioClientLogging.h" #include "AudioLogging.h" #include "AudioHelpers.h" #if defined(Q_OS_ANDROID) #include <QtAndroidExtras/QAndroidJniObject> #endif const int AudioClient::MIN_BUFFER_FRAMES = 1; const int AudioClient::MAX_BUFFER_FRAMES = 20; #if defined(Q_OS_ANDROID) static const int CHECK_INPUT_READS_MSECS = 2000; static const int MIN_READS_TO_CONSIDER_INPUT_ALIVE = 10; #endif const AudioClient::AudioPositionGetter AudioClient::DEFAULT_POSITION_GETTER = []{ return Vectors::ZERO; }; const AudioClient::AudioOrientationGetter AudioClient::DEFAULT_ORIENTATION_GETTER = [] { return Quaternions::IDENTITY; }; Setting::Handle<bool> dynamicJitterBufferEnabled("dynamicJitterBuffersEnabled", InboundAudioStream::DEFAULT_DYNAMIC_JITTER_BUFFER_ENABLED); Setting::Handle<int> staticJitterBufferFrames("staticJitterBufferFrames", InboundAudioStream::DEFAULT_STATIC_JITTER_FRAMES); // protect the Qt internal device list using Mutex = std::mutex; using Lock = std::unique_lock<Mutex>; Mutex _deviceMutex; Mutex _recordMutex; QString defaultAudioDeviceName(QAudio::Mode mode); void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { QWriteLocker lock(&_hmdNameLock); if (mode == QAudio::AudioInput) { _hmdInputName = name; } else { _hmdOutputName = name; } } // thread-safe QList<HifiAudioDeviceInfo> getAvailableDevices(QAudio::Mode mode, const QString& hmdName) { //get hmd device name prior to locking device mutex. in case of shutdown, this thread will be locked and audio client //cannot properly shut down. QString defDeviceName = defaultAudioDeviceName(mode); // NOTE: availableDevices() clobbers the Qt internal device list Lock lock(_deviceMutex); auto devices = QAudioDeviceInfo::availableDevices(mode); HifiAudioDeviceInfo defaultDesktopDevice; QList<HifiAudioDeviceInfo> newDevices; for (auto& device : devices) { newDevices.push_back(HifiAudioDeviceInfo(device, false, mode)); if (device.deviceName() == defDeviceName.trimmed()) { defaultDesktopDevice = HifiAudioDeviceInfo(device, true, mode, HifiAudioDeviceInfo::both); } } if (defaultDesktopDevice.getDevice().isNull()) { if (devices.size() > 0) { qCDebug(audioclient) << __FUNCTION__ << "Default device not found in list:" << defDeviceName << "Setting Default to: " << devices.first().deviceName(); newDevices.push_front(HifiAudioDeviceInfo(devices.first(), true, mode, HifiAudioDeviceInfo::both)); } else { //current audio list is empty for some reason. qCWarning(audioclient) << __FUNCTION__ << "Default device not found in list and no alternative selection available"; } } else { newDevices.push_front(defaultDesktopDevice); } if (!hmdName.isNull()) { HifiAudioDeviceInfo hmdDevice; foreach(auto device, newDevices) { if (device.getDevice().deviceName() == hmdName) { hmdDevice = HifiAudioDeviceInfo(device.getDevice(), true, mode, HifiAudioDeviceInfo::hmd); break; } } if (!hmdDevice.getDevice().isNull()) { newDevices.push_front(hmdDevice); } } return newDevices; } // now called from a background thread, to keep blocking operations off the audio thread void AudioClient::checkDevices() { // Make sure we're not shutting down Lock timerMutex(_checkDevicesMutex); // If we HAVE shut down after we were queued, but prior to execution, early exit if (nullptr == _checkDevicesTimer) { return; } QString hmdInputName; QString hmdOutputName; { QReadLocker readLock(&_hmdNameLock); hmdInputName = _hmdInputName; hmdOutputName = _hmdOutputName; } auto inputDevices = getAvailableDevices(QAudio::AudioInput, hmdInputName); auto outputDevices = getAvailableDevices(QAudio::AudioOutput, hmdOutputName); static const QMetaMethod devicesChangedSig= QMetaMethod::fromSignal(&AudioClient::devicesChanged); //only emit once the scripting interface has connected to the signal if (isSignalConnected(devicesChangedSig)) { Lock lock(_deviceMutex); if (inputDevices != _inputDevices) { _inputDevices.swap(inputDevices); emit devicesChanged(QAudio::AudioInput, _inputDevices); } if (outputDevices != _outputDevices) { _outputDevices.swap(outputDevices); emit devicesChanged(QAudio::AudioOutput, _outputDevices); } } } HifiAudioDeviceInfo AudioClient::getActiveAudioDevice(QAudio::Mode mode) const { Lock lock(_deviceMutex); if (mode == QAudio::AudioInput) { return _inputDeviceInfo; } else { return _outputDeviceInfo; } } QList<HifiAudioDeviceInfo> AudioClient::getAudioDevices(QAudio::Mode mode) const { Lock lock(_deviceMutex); if (mode == QAudio::AudioInput) { return _inputDevices; } else { return _outputDevices; } } static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { for (int i = 0; i < numSamples/2; i++) { // read 2 samples int16_t left = *source++; int16_t right = *source++; // write 2 + N samples *dest++ = left; *dest++ = right; for (int n = 0; n < numExtraChannels; n++) { *dest++ = 0; } } } static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { for (int i = 0; i < numSamples/2; i++) { // read 2 samples int16_t left = *source++; int16_t right = *source++; // write 1 sample *dest++ = (int16_t)((left + right) / 2); } } static bool detectClipping(int16_t* samples, int numSamples, int numChannels) { const int32_t CLIPPING_THRESHOLD = 32392; // -0.1 dBFS const int CLIPPING_DETECTION = 3; // consecutive samples over threshold bool isClipping = false; if (numChannels == 2) { int oversLeft = 0; int oversRight = 0; for (int i = 0; i < numSamples/2; i++) { int32_t left = std::abs((int32_t)samples[2*i+0]); int32_t right = std::abs((int32_t)samples[2*i+1]); if (left > CLIPPING_THRESHOLD) { isClipping |= (++oversLeft >= CLIPPING_DETECTION); } else { oversLeft = 0; } if (right > CLIPPING_THRESHOLD) { isClipping |= (++oversRight >= CLIPPING_DETECTION); } else { oversRight = 0; } } } else { int overs = 0; for (int i = 0; i < numSamples; i++) { int32_t sample = std::abs((int32_t)samples[i]); if (sample > CLIPPING_THRESHOLD) { isClipping |= (++overs >= CLIPPING_DETECTION); } else { overs = 0; } } } return isClipping; } static float computeLoudness(int16_t* samples, int numSamples) { float scale = numSamples ? 1.0f / numSamples : 0.0f; int32_t loudness = 0; for (int i = 0; i < numSamples; i++) { loudness += std::abs((int32_t)samples[i]); } return (float)loudness * scale; } template <int NUM_CHANNELS> static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float gain1) { // fast path for unity gain if (gain0 == 1.0f && gain1 == 1.0f) { return; } // cubic poly from gain0 to gain1 float c3 = -2.0f * (gain1 - gain0); float c2 = 3.0f * (gain1 - gain0); float c0 = gain0; float t = 0.0f; float tStep = 1.0f / numFrames; for (int i = 0; i < numFrames; i++) { // evaluate poly over t=[0,1) float gain = (c3 * t + c2) * t * t + c0; t += tStep; // apply gain to all channels for (int ch = 0; ch < NUM_CHANNELS; ch++) { buffer[NUM_CHANNELS*i + ch] *= gain; } } } static inline float convertToFloat(int16_t sample) { return (float)sample * (1 / 32768.0f); } AudioClient::AudioClient() { // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); // Set up the desired audio format, since scripting API expects it to be set and audio scripting API // is initialized before audio thread starts. _desiredInputFormat.setSampleRate(AudioConstants::SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); _desiredInputFormat.setCodec("audio/pcm"); _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); _desiredInputFormat.setChannelCount(1); _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); // deprecate legacy settings { Setting::Handle<int>::Deprecated("maxFramesOverDesired", InboundAudioStream::MAX_FRAMES_OVER_DESIRED); Setting::Handle<int>::Deprecated("windowStarveThreshold", InboundAudioStream::WINDOW_STARVE_THRESHOLD); Setting::Handle<int>::Deprecated("windowSecondsForDesiredCalcOnTooManyStarves", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES); Setting::Handle<int>::Deprecated("windowSecondsForDesiredReduction", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_REDUCTION); Setting::Handle<bool>::Deprecated("useStDevForJitterCalc", InboundAudioStream::USE_STDEV_FOR_JITTER); Setting::Handle<bool>::Deprecated("repetitionWithFade", InboundAudioStream::REPETITION_WITH_FADE); } connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &AudioClient::processReceivedSamples, Qt::DirectConnection); connect(this, &AudioClient::changeDevice, this, [=, this](const HifiAudioDeviceInfo& outputDeviceInfo) { qCDebug(audioclient)<< "got AudioClient::changeDevice signal, about to call switchOutputToAudioDevice() outputDeviceInfo: ["<< outputDeviceInfo.deviceName() << "]"; switchOutputToAudioDevice(outputDeviceInfo); }); connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash defaultAudioDeviceName(QAudio::AudioInput); defaultAudioDeviceName(QAudio::AudioOutput); checkDevices(); // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000; connect(_checkDevicesTimer, &QTimer::timeout, this, [=, this] { QtConcurrent::run(QThreadPool::globalInstance(), [=, this] { checkDevices(); // On some systems (Ubuntu) checking all the audio devices can take more than 2 seconds. To // avoid consuming all of the thread pool, don't start the check interval until the previous // check has completed. QMetaObject::invokeMethod(_checkDevicesTimer, "start", Q_ARG(int, DEVICE_CHECK_INTERVAL_MSECS)); }); }); _checkDevicesTimer->setSingleShot(true); _checkDevicesTimer->start(DEVICE_CHECK_INTERVAL_MSECS); // start a thread to detect peak value changes _checkPeakValuesTimer = new QTimer(this); connect(_checkPeakValuesTimer, &QTimer::timeout, this, [this] { QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); }); }); const unsigned long PEAK_VALUES_CHECK_INTERVAL_MSECS = 50; _checkPeakValuesTimer->start(PEAK_VALUES_CHECK_INTERVAL_MSECS); configureReverb(); #if defined(WEBRTC_AUDIO) configureWebrtc(); #endif auto nodeList = DependencyManager::get<NodeList>(); auto& packetReceiver = nodeList->getPacketReceiver(); packetReceiver.registerListener(PacketType::AudioStreamStats, PacketReceiver::makeSourcedListenerReference<AudioIOStats>(&_stats, &AudioIOStats::processStreamStatsPacket)); packetReceiver.registerListener(PacketType::AudioEnvironment, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleAudioEnvironmentDataPacket)); packetReceiver.registerListener(PacketType::SilentAudioFrame, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleAudioDataPacket)); packetReceiver.registerListener(PacketType::MixedAudio, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleAudioDataPacket)); packetReceiver.registerListener(PacketType::NoisyMute, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleNoisyMutePacket)); packetReceiver.registerListener(PacketType::MuteEnvironment, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleMuteEnvironmentPacket)); packetReceiver.registerListener(PacketType::SelectedAudioFormat, PacketReceiver::makeUnsourcedListenerReference<AudioClient>(this, &AudioClient::handleSelectedAudioFormat)); auto& domainHandler = nodeList->getDomainHandler(); connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { _solo.reset(); }); connect(nodeList.data(), &NodeList::nodeActivated, this, [this](SharedNodePointer node) { if (node->getType() == NodeType::AudioMixer) { _solo.resend(); } }); } AudioClient::~AudioClient() { stop(); if (_codec && _encoder) { _codec->releaseEncoder(_encoder); _encoder = nullptr; } } void AudioClient::customDeleter() { #if defined(Q_OS_ANDROID) _shouldRestartInputSetup = false; #endif deleteLater(); } void AudioClient::handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec) { qCDebug(audioclient) << __FUNCTION__ << "sendingNode:" << *node << "currentCodec:" << currentCodec << "recievedCodec:" << recievedCodec; selectAudioFormat(recievedCodec); } void AudioClient::reset() { _receivedAudioStream.reset(); _stats.reset(); _sourceReverb.reset(); _listenerReverb.reset(); _localReverb.reset(); } void AudioClient::audioMixerKilled() { _hasReceivedFirstPacket = false; _outgoingAvatarAudioSequenceNumber = 0; _stats.reset(); emit disconnected(); } void AudioClient::setAudioPaused(bool pause) { if (_audioPaused != pause) { _audioPaused = pause; if (!_audioPaused) { negotiateAudioFormat(); } } } HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { if ((!isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::hmd) || (isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::desktop)) { result = audioDevice; break; } } } return result; } #ifdef Q_OS_WIN QString getWinDeviceName(IMMDevice* pEndpoint) { QString deviceName; IPropertyStore* pPropertyStore; pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore); PROPVARIANT pv; PropVariantInit(&pv); HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); pPropertyStore->Release(); pPropertyStore = nullptr; deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal); if (!IsWindows8OrGreater()) { // Windows 7 provides only the 31 first characters of the device name. const DWORD QT_WIN7_MAX_AUDIO_DEVICENAME_LEN = 31; deviceName = deviceName.left(QT_WIN7_MAX_AUDIO_DEVICENAME_LEN); } PropVariantClear(&pv); return deviceName; } QString AudioClient::getWinDeviceName(wchar_t* guid) { QString deviceName; HRESULT hr = S_OK;