//// Audio.cpp// interface/src//// Created by Stephen Birarda on 1/22/13.// Copyright 2013 High Fidelity, Inc.//// Distributed under the Apache License, Version 2.0.// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include <cstring>#include <sys/stat.h> #include <math.h> #ifdef __APPLE__#include <CoreAudio/AudioHardware.h>#endif #ifdef WIN32#define WIN32_LEAN_AND_MEAN#include <windows.h>#include <Mmsystem.h>#include <mmdeviceapi.h>#include <devicetopology.h>#include <Functiondiscoverykeys_devpkey.h>#endif #include <QtCore/QBuffer>#include <QtMultimedia/QAudioInput>#include <QtMultimedia/QAudioOutput>#include <QSvgRenderer> #include <NodeList.h>#include <PacketHeaders.h>#include <SharedUtil.h>#include <StdDev.h>#include <UUID.h>#include <glm/glm.hpp> #include "Application.h"#include "Audio.h"#include "Menu.h"#include "Util.h" static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; // Mute icon configrationstatic const int MUTE_ICON_SIZE = 24; Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : AbstractAudioInterface(parent), _audioInput(NULL), _desiredInputFormat(), _inputFormat(), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputDevice(NULL), _numOutputCallbackBytes(0), _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _proceduralAudioOutput(NULL), _proceduralOutputDevice(NULL), _inputRingBuffer(0), _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL), _isStereoInput(false), _averagedLatency(0.0), _measuredJitter(0), _jitterBufferSamples(initialJitterBufferSamples), _lastInputLoudness(0), _timeSinceLastClip(-1.0), _dcOffset(0), _noiseGateMeasuredFloor(0), _noiseGateSampleCounter(0), _noiseGateOpen(false), _noiseGateEnabled(true), _toneInjectionEnabled(false), _noiseGateFramesToClose(0), _totalPacketsReceived(0), _totalInputAudioSamples(0), _collisionSoundMagnitude(0.0f), _collisionSoundFrequency(0.0f), _collisionSoundNoise(0.0f), _collisionSoundDuration(0.0f), _proceduralEffectSample(0), _numFramesDisplayStarve(0), _muted(false), _processSpatialAudio(false), _spatialAudioStart(0), _spatialAudioFinish(0), _spatialAudioRingBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, true), // random access mode _scopeEnabled(false), _scopeEnabledPause(false), _scopeInputOffset(0), _scopeOutputOffset(0), _framesPerScope(DEFAULT_FRAMES_PER_SCOPE), _samplesPerScope(NETWORK_SAMPLES_PER_FRAME * _framesPerScope), _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0){ // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // Create the noise sample array _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES];} void Audio::init(QGLWidget *parent) { _micTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic.svg")); _muteTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic-mute.svg")); _boxTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/audio-box.svg"));} void Audio::reset() { _ringBuffer.reset();} QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { QAudioDeviceInfo result; foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { qDebug() << audioDevice.deviceName() << " " << deviceName; if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { result = audioDevice; } } return result;} QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {#ifdef __APPLE__ if (QAudioDeviceInfo::availableDevices(mode).size() > 1) { AudioDeviceID defaultDeviceID = 0; uint32_t propertySize = sizeof(AudioDeviceID); AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDefaultInputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; if (mode == QAudio::AudioOutput) { propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; } OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject, &propertyAddress, 0, NULL, &propertySize, &defaultDeviceID); if (!getPropertyError && propertySize) { CFStringRef deviceName = NULL; propertySize = sizeof(deviceName); propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString; getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0, NULL, &propertySize, &deviceName); if (!getPropertyError && propertySize) { // find a device in the list that matches the name we have and return it foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) { return audioDevice; } } } } }#endif#ifdef WIN32 QString deviceName; //Check for Windows Vista or higher, IMMDeviceEnumerator doesn't work below that. OSVERSIONINFO osvi; ZeroMemory(&osvi, sizeof(OSVERSIONINFO)); osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&osvi); const DWORD VISTA_MAJOR_VERSION = 6; if (osvi.dwMajorVersion < VISTA_MAJOR_VERSION) {// lower then vista if (mode == QAudio::AudioInput) { WAVEINCAPS wic; // first use WAVE_MAPPER to get the default devices manufacturer ID waveInGetDevCaps(WAVE_MAPPER, &wic, sizeof(wic)); //Use the received manufacturer id to get the device's real name waveInGetDevCaps(wic.wMid, &wic, sizeof(wic)); qDebug() << "input device:" << wic.szPname; deviceName = wic.szPname; } else { WAVEOUTCAPS woc; // first use WAVE_MAPPER to get the default devices manufacturer ID waveOutGetDevCaps(WAVE_MAPPER, &woc, sizeof(woc)); //Use the received manufacturer id to get the device's real name waveOutGetDevCaps(woc.wMid, &woc, sizeof(woc)); qDebug() << "output device:" << woc.szPname; deviceName = woc.szPname; } } else { HRESULT hr = S_OK; CoInitialize(NULL); IMMDeviceEnumerator* pMMDeviceEnumerator = NULL; CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, &pEndpoint); if (hr == E_NOTFOUND) { printf("Audio Error: device not found\n"); deviceName = QString("NONE"); } else { IPropertyStore* pPropertyStore; pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore); pEndpoint->Release(); pEndpoint = NULL; PROPVARIANT pv; PropVariantInit(&pv); hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); pPropertyStore->Release(); pPropertyStore = NULL; deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal); const DWORD WINDOWS7_MAJOR_VERSION = 6; const DWORD WINDOWS7_MINOR_VERSION = 1; if (osvi.dwMajorVersion <= WINDOWS7_MAJOR_VERSION && osvi.dwMinorVersion <= WINDOWS7_MINOR_VERSION) { // Windows 7 provides only the 31 first characters of the device name. const DWORD QT_WIN7_MAX_AUDIO_DEVICENAME_LEN = 31; deviceName = deviceName.left(QT_WIN7_MAX_AUDIO_DEVICENAME_LEN); } qDebug() << (mode == QAudio::AudioOutput ? "output" : "input") << " device:" << deviceName; PropVariantClear(&pv); } pMMDeviceEnumerator->Release(); pMMDeviceEnumerator = NULL; CoUninitialize(); } qDebug() << "DEBUG [" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]"; return getNamedAudioDeviceForMode(mode, deviceName);#endif // fallback for failed lookup is the default device return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice();} bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, const QAudioFormat& desiredAudioFormat, QAudioFormat& adjustedAudioFormat) { if (!audioDevice.isFormatSupported(desiredAudioFormat)) { qDebug() << "The desired format for audio I/O is" << desiredAudioFormat; qDebug("The desired audio format is not supported by this device"); if (desiredAudioFormat.channelCount() == 1) { adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setChannelCount(2); if (audioDevice.isFormatSupported(adjustedAudioFormat)) { return true; } else { adjustedAudioFormat.setChannelCount(1); } } if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) { // use 48, which is a sample downsample, upsample adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2); // return the nearest in case it needs 2 channels adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); return true; } return false; } else { // set the adjustedAudioFormat to the desiredAudioFormat, since it will work adjustedAudioFormat = desiredAudioFormat; return true; }} void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples, unsigned int numDestinationSamples, const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount()); // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels // this is the case with our HDMI capture if (sourceToDestinationFactor >= 2) { // we need to downsample from 48 to 24 // for now this only supports a mono output - this would be the case for audio input if (destinationAudioFormat.channelCount() == 1) { for (unsigned int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) { if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 2) + (sourceSamples[i] / 2); } else { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 4) + (sourceSamples[i] / 2) + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4); } } } else { // this is a 48 to 24 resampling but both source and destination are two channels // squish two samples into one in each channel for (unsigned int i = 0; i < numSourceSamples; i += 4) { destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 2] / 2); destinationSamples[(i / 2) + 1] = (sourceSamples[i + 1] / 2) + (sourceSamples[i + 3] / 2); } } } else { if (sourceAudioFormat.sampleRate() == destinationAudioFormat.sampleRate()) { // mono to stereo, same sample rate if (!(sourceAudioFormat.channelCount() == 1 && destinationAudioFormat.channelCount() == 2)) { qWarning() << "Unsupported format conversion" << sourceAudioFormat << destinationAudioFormat; return; } for (const int16_t* sourceEnd = sourceSamples + numSourceSamples; sourceSamples != sourceEnd; sourceSamples++) { *destinationSamples++ = *sourceSamples; *destinationSamples++ = *sourceSamples; } return; } // upsample from 24 to 48 // for now this only supports a stereo to stereo conversion - this is our case for network audio to output int sourceIndex = 0; int dtsSampleRateFactor = (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()); int sampleShift = destinationAudioFormat.channelCount() * dtsSampleRateFactor; int destinationToSourceFactor = (1 / sourceToDestinationFactor); for (unsigned int i = 0; i < numDestinationSamples; i += sampleShift) { sourceIndex = (i / destinationToSourceFactor); // fill the L/R channels and make the rest silent for (unsigned int j = i; j < i + sampleShift; j++) { if (j % destinationAudioFormat.channelCount() == 0) { // left channel destinationSamples[j] = sourceSamples[sourceIndex]; } else if (j % destinationAudioFormat.channelCount() == 1) { // right channel destinationSamples[j] = sourceSamples[sourceIndex + (sourceAudioFormat.channelCount() > 1 ? 1 : 0)]; } else { // channels above 2, fill with silence destinationSamples[j] = 0; } } } } }} void Audio::start() { // set up the desired audio format _desiredInputFormat.setSampleRate(SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); _desiredInputFormat.setCodec("audio/pcm"); _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); _desiredInputFormat.setChannelCount(1); _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(2); QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); qDebug() << "The default audio input device is" << inputDeviceInfo.deviceName(); bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); qDebug() << "The default audio output device is" << outputDeviceInfo.deviceName(); bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); if (!inputFormatSupported) { qDebug() << "Unable to set up audio input because of a problem with input format."; } if (!outputFormatSupported) { qDebug() << "Unable to set up audio output because of a problem with output format."; }} void Audio::stop() { // "switch" to invalid devices in order to shut down the state switchInputToAudioDevice(QAudioDeviceInfo()); switchOutputToAudioDevice(QAudioDeviceInfo());} QString Audio::getDefaultDeviceName(QAudio::Mode mode) { QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode); return deviceInfo.deviceName();} QVector<QString> Audio::getDeviceNames(QAudio::Mode mode) { QVector<QString> deviceNames; foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { deviceNames << audioDevice.deviceName().trimmed(); } return deviceNames;} bool Audio::switchInputToAudioDevice(const QString& inputDeviceName) { qDebug() << "DEBUG [" << inputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName).deviceName() << "]"; return switchInputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName));} bool Audio::switchOutputToAudioDevice(const QString& outputDeviceName) { qDebug() << "DEBUG [" << outputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName).deviceName() << "]"; return switchOutputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName));} void Audio::handleAudioInput() { static char audioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMicrophoneAudioNoEcho); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); static int16_t* networkAudioSamples = (int16_t*) (audioDataPacket + leadingBytes); float inputToNetworkInputRatio = calculateDeviceToNetworkInputRatio(_numInputCallbackBytes); unsigned int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; QByteArray inputByteArray = _inputDevice->readAll(); if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted && _audioOutput) { // if this person wants local loopback add that to the locally injected audio if (!_loopbackOutputDevice && _loopbackAudioOutput) { // we didn't have the loopback output device going so set that up now _loopbackOutputDevice = _loopbackAudioOutput->start(); } if (_inputFormat == _outputFormat) { if (_loopbackOutputDevice) { _loopbackOutputDevice->write(inputByteArray); } } else { float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) * (_outputFormat.channelCount() / _inputFormat.channelCount()); QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0); linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(), inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat); if (_loopbackOutputDevice) { _loopbackOutputDevice->write(loopBackByteArray); } } } _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); const int numNetworkBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; const int numNetworkSamples = _isStereoInput ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // zero out the monoAudioSamples array and the locally injected audio memset(networkAudioSamples, 0, numNetworkBytes); if (!_muted) { // we aren't muted, downsample the input audio linearResampling((int16_t*) inputAudioSamples, networkAudioSamples, inputSamplesRequired, numNetworkSamples, _inputFormat, _desiredInputFormat); // only impose the noise gate and perform tone injection if we sending mono audio if (!_isStereoInput) { // // Impose Noise Gate // // The Noise Gate is used to reject constant background noise by measuring the noise // floor observed at the microphone and then opening the 'gate' to allow microphone // signals to be transmitted when the microphone samples average level exceeds a multiple // of the noise floor. // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded // to open the gate. // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames // will we wait before closing the gate. // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor?Lines 1–500 of 1456. Load more
// // Audio.cpp // interface/src // // Created by Stephen Birarda on 1/22/13. // Copyright 2013 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include <cstring> #include <sys/stat.h> #include <math.h> #ifdef __APPLE__ #include <CoreAudio/AudioHardware.h> #endif #ifdef WIN32 #define WIN32_LEAN_AND_MEAN #include <windows.h> #include <Mmsystem.h> #include <mmdeviceapi.h> #include <devicetopology.h> #include <Functiondiscoverykeys_devpkey.h> #endif #include <QtCore/QBuffer> #include <QtMultimedia/QAudioInput> #include <QtMultimedia/QAudioOutput> #include <QSvgRenderer> #include <NodeList.h> #include <PacketHeaders.h> #include <SharedUtil.h> #include <StdDev.h> #include <UUID.h> #include <glm/glm.hpp> #include "Application.h" #include "Audio.h" #include "Menu.h" #include "Util.h" static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; // Mute icon configration static const int MUTE_ICON_SIZE = 24; Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : AbstractAudioInterface(parent), _audioInput(NULL), _desiredInputFormat(), _inputFormat(), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputDevice(NULL), _numOutputCallbackBytes(0), _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _proceduralAudioOutput(NULL), _proceduralOutputDevice(NULL), _inputRingBuffer(0), _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL), _isStereoInput(false), _averagedLatency(0.0), _measuredJitter(0), _jitterBufferSamples(initialJitterBufferSamples), _lastInputLoudness(0), _timeSinceLastClip(-1.0), _dcOffset(0), _noiseGateMeasuredFloor(0), _noiseGateSampleCounter(0), _noiseGateOpen(false), _noiseGateEnabled(true), _toneInjectionEnabled(false), _noiseGateFramesToClose(0), _totalPacketsReceived(0), _totalInputAudioSamples(0), _collisionSoundMagnitude(0.0f), _collisionSoundFrequency(0.0f), _collisionSoundNoise(0.0f), _collisionSoundDuration(0.0f), _proceduralEffectSample(0), _numFramesDisplayStarve(0), _muted(false), _processSpatialAudio(false), _spatialAudioStart(0), _spatialAudioFinish(0), _spatialAudioRingBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, true), // random access mode _scopeEnabled(false), _scopeEnabledPause(false), _scopeInputOffset(0), _scopeOutputOffset(0), _framesPerScope(DEFAULT_FRAMES_PER_SCOPE), _samplesPerScope(NETWORK_SAMPLES_PER_FRAME * _framesPerScope), _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // Create the noise sample array _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; } void Audio::init(QGLWidget *parent) { _micTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic.svg")); _muteTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic-mute.svg")); _boxTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/audio-box.svg")); } void Audio::reset() { _ringBuffer.reset(); } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { QAudioDeviceInfo result; foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { qDebug() << audioDevice.deviceName() << " " << deviceName; if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { result = audioDevice; } } return result; } QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { #ifdef __APPLE__ if (QAudioDeviceInfo::availableDevices(mode).size() > 1) { AudioDeviceID defaultDeviceID = 0; uint32_t propertySize = sizeof(AudioDeviceID); AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDefaultInputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; if (mode == QAudio::AudioOutput) { propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; } OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject, &propertyAddress, 0, NULL, &propertySize, &defaultDeviceID); if (!getPropertyError && propertySize) { CFStringRef deviceName = NULL; propertySize = sizeof(deviceName); propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString; getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0, NULL, &propertySize, &deviceName); if (!getPropertyError && propertySize) { // find a device in the list that matches the name we have and return it foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) { return audioDevice; } } } } } #endif #ifdef WIN32 QString deviceName; //Check for Windows Vista or higher, IMMDeviceEnumerator doesn't work below that. OSVERSIONINFO osvi; ZeroMemory(&osvi, sizeof(OSVERSIONINFO)); osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&osvi); const DWORD VISTA_MAJOR_VERSION = 6; if (osvi.dwMajorVersion < VISTA_MAJOR_VERSION) {// lower then vista if (mode == QAudio::AudioInput) { WAVEINCAPS wic; // first use WAVE_MAPPER to get the default devices manufacturer ID waveInGetDevCaps(WAVE_MAPPER, &wic, sizeof(wic)); //Use the received manufacturer id to get the device's real name waveInGetDevCaps(wic.wMid, &wic, sizeof(wic)); qDebug() << "input device:" << wic.szPname; deviceName = wic.szPname; } else { WAVEOUTCAPS woc; // first use WAVE_MAPPER to get the default devices manufacturer ID waveOutGetDevCaps(WAVE_MAPPER, &woc, sizeof(woc)); //Use the received manufacturer id to get the device's real name waveOutGetDevCaps(woc.wMid, &woc, sizeof(woc)); qDebug() << "output device:" << woc.szPname; deviceName = woc.szPname; } } else { HRESULT hr = S_OK; CoInitialize(NULL); IMMDeviceEnumerator* pMMDeviceEnumerator = NULL; CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, &pEndpoint); if (hr == E_NOTFOUND) { printf("Audio Error: device not found\n"); deviceName = QString("NONE"); } else { IPropertyStore* pPropertyStore; pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore); pEndpoint->Release(); pEndpoint = NULL; PROPVARIANT pv; PropVariantInit(&pv); hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); pPropertyStore->Release(); pPropertyStore = NULL; deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal); const DWORD WINDOWS7_MAJOR_VERSION = 6; const DWORD WINDOWS7_MINOR_VERSION = 1; if (osvi.dwMajorVersion <= WINDOWS7_MAJOR_VERSION && osvi.dwMinorVersion <= WINDOWS7_MINOR_VERSION) { // Windows 7 provides only the 31 first characters of the device name. const DWORD QT_WIN7_MAX_AUDIO_DEVICENAME_LEN = 31; deviceName = deviceName.left(QT_WIN7_MAX_AUDIO_DEVICENAME_LEN); } qDebug() << (mode == QAudio::AudioOutput ? "output" : "input") << " device:" << deviceName; PropVariantClear(&pv); } pMMDeviceEnumerator->Release(); pMMDeviceEnumerator = NULL; CoUninitialize(); } qDebug() << "DEBUG [" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]"; return getNamedAudioDeviceForMode(mode, deviceName); #endif // fallback for failed lookup is the default device return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice(); } bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, const QAudioFormat& desiredAudioFormat, QAudioFormat& adjustedAudioFormat) { if (!audioDevice.isFormatSupported(desiredAudioFormat)) { qDebug() << "The desired format for audio I/O is" << desiredAudioFormat; qDebug("The desired audio format is not supported by this device"); if (desiredAudioFormat.channelCount() == 1) { adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setChannelCount(2); if (audioDevice.isFormatSupported(adjustedAudioFormat)) { return true; } else { adjustedAudioFormat.setChannelCount(1); } } if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) { // use 48, which is a sample downsample, upsample adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2); // return the nearest in case it needs 2 channels adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); return true; } return false; } else { // set the adjustedAudioFormat to the desiredAudioFormat, since it will work adjustedAudioFormat = desiredAudioFormat; return true; } } void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples, unsigned int numDestinationSamples, const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount()); // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels // this is the case with our HDMI capture if (sourceToDestinationFactor >= 2) { // we need to downsample from 48 to 24 // for now this only supports a mono output - this would be the case for audio input if (destinationAudioFormat.channelCount() == 1) { for (unsigned int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) { if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 2) + (sourceSamples[i] / 2); } else { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 4) + (sourceSamples[i] / 2) + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4); } } } else { // this is a 48 to 24 resampling but both source and destination are two channels // squish two samples into one in each channel for (unsigned int i = 0; i < numSourceSamples; i += 4) { destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 2] / 2); destinationSamples[(i / 2) + 1] = (sourceSamples[i + 1] / 2) + (sourceSamples[i + 3] / 2); } } } else { if (sourceAudioFormat.sampleRate() == destinationAudioFormat.sampleRate()) { // mono to stereo, same sample rate if (!(sourceAudioFormat.channelCount() == 1 && destinationAudioFormat.channelCount() == 2)) { qWarning() << "Unsupported format conversion" << sourceAudioFormat << destinationAudioFormat; return; } for (const int16_t* sourceEnd = sourceSamples + numSourceSamples; sourceSamples != sourceEnd; sourceSamples++) { *destinationSamples++ = *sourceSamples; *destinationSamples++ = *sourceSamples; } return; } // upsample from 24 to 48 // for now this only supports a stereo to stereo conversion - this is our case for network audio to output int sourceIndex = 0; int dtsSampleRateFactor = (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()); int sampleShift = destinationAudioFormat.channelCount() * dtsSampleRateFactor; int destinationToSourceFactor = (1 / sourceToDestinationFactor); for (unsigned int i = 0; i < numDestinationSamples; i += sampleShift) { sourceIndex = (i / destinationToSourceFactor); // fill the L/R channels and make the rest silent for (unsigned int j = i; j < i + sampleShift; j++) { if (j % destinationAudioFormat.channelCount() == 0) { // left channel destinationSamples[j] = sourceSamples[sourceIndex]; } else if (j % destinationAudioFormat.channelCount() == 1) { // right channel destinationSamples[j] = sourceSamples[sourceIndex + (sourceAudioFormat.channelCount() > 1 ? 1 : 0)]; } else { // channels above 2, fill with silence destinationSamples[j] = 0; } } } } } } void Audio::start() { // set up the desired audio format _desiredInputFormat.setSampleRate(SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); _desiredInputFormat.setCodec("audio/pcm"); _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); _desiredInputFormat.setChannelCount(1); _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(2); QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); qDebug() << "The default audio input device is" << inputDeviceInfo.deviceName(); bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); qDebug() << "The default audio output device is" << outputDeviceInfo.deviceName(); bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); if (!inputFormatSupported) { qDebug() << "Unable to set up audio input because of a problem with input format."; } if (!outputFormatSupported) { qDebug() << "Unable to set up audio output because of a problem with output format."; } } void Audio::stop() { // "switch" to invalid devices in order to shut down the state switchInputToAudioDevice(QAudioDeviceInfo()); switchOutputToAudioDevice(QAudioDeviceInfo()); } QString Audio::getDefaultDeviceName(QAudio::Mode mode) { QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode); return deviceInfo.deviceName(); } QVector<QString> Audio::getDeviceNames(QAudio::Mode mode) { QVector<QString> deviceNames; foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { deviceNames << audioDevice.deviceName().trimmed(); } return deviceNames; } bool Audio::switchInputToAudioDevice(const QString& inputDeviceName) { qDebug() << "DEBUG [" << inputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName).deviceName() << "]"; return switchInputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName)); } bool Audio::switchOutputToAudioDevice(const QString& outputDeviceName) { qDebug() << "DEBUG [" << outputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName).deviceName() << "]"; return switchOutputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName)); } void Audio::handleAudioInput() { static char audioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMicrophoneAudioNoEcho); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); static int16_t* networkAudioSamples = (int16_t*) (audioDataPacket + leadingBytes); float inputToNetworkInputRatio = calculateDeviceToNetworkInputRatio(_numInputCallbackBytes); unsigned int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; QByteArray inputByteArray = _inputDevice->readAll(); if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted && _audioOutput) { // if this person wants local loopback add that to the locally injected audio if (!_loopbackOutputDevice && _loopbackAudioOutput) { // we didn't have the loopback output device going so set that up now _loopbackOutputDevice = _loopbackAudioOutput->start(); } if (_inputFormat == _outputFormat) { if (_loopbackOutputDevice) { _loopbackOutputDevice->write(inputByteArray); } } else { float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) * (_outputFormat.channelCount() / _inputFormat.channelCount()); QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0); linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(), inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat); if (_loopbackOutputDevice) { _loopbackOutputDevice->write(loopBackByteArray); } } } _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); const int numNetworkBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; const int numNetworkSamples = _isStereoInput ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // zero out the monoAudioSamples array and the locally injected audio memset(networkAudioSamples, 0, numNetworkBytes); if (!_muted) { // we aren't muted, downsample the input audio linearResampling((int16_t*) inputAudioSamples, networkAudioSamples, inputSamplesRequired, numNetworkSamples, _inputFormat, _desiredInputFormat); // only impose the noise gate and perform tone injection if we sending mono audio if (!_isStereoInput) { // // Impose Noise Gate // // The Noise Gate is used to reject constant background noise by measuring the noise // floor observed at the microphone and then opening the 'gate' to allow microphone // signals to be transmitted when the microphone samples average level exceeds a multiple // of the noise floor. // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded // to open the gate. // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames // will we wait before closing the gate. // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor?