//// Audio.cpp// interface//// Created by Stephen Birarda on 1/22/13.// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.// #include <cstring>#include <sys/stat.h> #include <math.h> #ifdef __APPLE__#include <CoreAudio/AudioHardware.h>#endif #include <QtCore/QBuffer>#include <QtMultimedia/QAudioInput>#include <QtMultimedia/QAudioOutput>#include <QSvgRenderer> #include <AngleUtil.h>#include <NodeList.h>#include <PacketHeaders.h>#include <SharedUtil.h>#include <StdDev.h>#include <UUID.h> #include "Application.h"#include "Audio.h"#include "Menu.h"#include "Util.h" static const float JITTER_BUFFER_LENGTH_MSECS = 12;static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; // Mute icon configrationstatic const int ICON_SIZE = 24;static const int ICON_LEFT = 0;static const int ICON_TOP = 115;static const int ICON_TOP_MIRROR = 220; Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent) : AbstractAudioInterface(parent), _audioInput(NULL), _desiredInputFormat(), _inputFormat(), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputDevice(NULL), _numOutputCallbackBytes(0), _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _proceduralAudioOutput(NULL), _proceduralOutputDevice(NULL), _inputRingBuffer(0), _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), _jitterBufferSamples(initialJitterBufferSamples), _lastInputLoudness(0), _dcOffset(0), _noiseGateMeasuredFloor(0), _noiseGateSampleCounter(0), _noiseGateOpen(false), _noiseGateEnabled(true), _noiseGateFramesToClose(0), _lastVelocity(0), _lastAcceleration(0), _totalPacketsReceived(0), _collisionSoundMagnitude(0.0f), _collisionSoundFrequency(0.0f), _collisionSoundNoise(0.0f), _collisionSoundDuration(0.0f), _proceduralEffectSample(0), _numFramesDisplayStarve(0), _muted(false){ // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // Create the noise sample array _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES];} void Audio::init(QGLWidget *parent) { switchToResourcesParentIfRequired(); _micTextureId = parent->bindTexture(QImage("./resources/images/mic.svg")); _muteTextureId = parent->bindTexture(QImage("./resources/images/mute.svg"));} void Audio::reset() { _ringBuffer.reset();} QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {#ifdef __APPLE__ if (QAudioDeviceInfo::availableDevices(mode).size() > 1) { AudioDeviceID defaultDeviceID = 0; uint32_t propertySize = sizeof(AudioDeviceID); AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDefaultInputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; if (mode == QAudio::AudioOutput) { propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; } OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject, &propertyAddress, 0, NULL, &propertySize, &defaultDeviceID); if (!getPropertyError && propertySize) { CFStringRef deviceName = NULL; propertySize = sizeof(deviceName); propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString; getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0, NULL, &propertySize, &deviceName); if (!getPropertyError && propertySize) { // find a device in the list that matches the name we have and return it foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) { return audioDevice; } } } } }#endif // fallback for failed lookup is the default device return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice();} bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, const QAudioFormat& desiredAudioFormat, QAudioFormat& adjustedAudioFormat) { if (!audioDevice.isFormatSupported(desiredAudioFormat)) { qDebug() << "The desired format for audio I/O is" << desiredAudioFormat; qDebug("The desired audio format is not supported by this device"); if (desiredAudioFormat.channelCount() == 1) { adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setChannelCount(2); if (audioDevice.isFormatSupported(adjustedAudioFormat)) { return true; } else { adjustedAudioFormat.setChannelCount(1); } } if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) { // use 48, which is a sample downsample, upsample adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2); // return the nearest in case it needs 2 channels adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); return true; } return false; } else { // set the adjustedAudioFormat to the desiredAudioFormat, since it will work adjustedAudioFormat = desiredAudioFormat; return true; }} void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples, unsigned int numDestinationSamples, const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount()); // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels // this is the case with our HDMI capture if (sourceToDestinationFactor >= 2) { // we need to downsample from 48 to 24 // for now this only supports a mono output - this would be the case for audio input for (unsigned int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) { if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 2) + (sourceSamples[i] / 2); } else { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 4) + (sourceSamples[i] / 2) + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4); } } } else { // upsample from 24 to 48 // for now this only supports a stereo to stereo conversion - this is our case for network audio to output int sourceIndex = 0; int dtsSampleRateFactor = (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()); int sampleShift = destinationAudioFormat.channelCount() * dtsSampleRateFactor; int destinationToSourceFactor = (1 / sourceToDestinationFactor); for (unsigned int i = 0; i < numDestinationSamples; i += sampleShift) { sourceIndex = (i / destinationToSourceFactor); // fill the L/R channels and make the rest silent for (unsigned int j = i; j < i + sampleShift; j++) { if (j % destinationAudioFormat.channelCount() == 0) { // left channel destinationSamples[j] = sourceSamples[sourceIndex]; } else if (j % destinationAudioFormat.channelCount() == 1) { // right channel destinationSamples[j] = sourceSamples[sourceIndex + (sourceAudioFormat.channelCount() > 1 ? 1 : 0)]; } else { // channels above 2, fill with silence destinationSamples[j] = 0; } } } } }} const int CALLBACK_ACCELERATOR_RATIO = 2; void Audio::start() { // set up the desired audio format _desiredInputFormat.setSampleRate(SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); _desiredInputFormat.setCodec("audio/pcm"); _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); _desiredInputFormat.setChannelCount(1); _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(2); QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); qDebug() << "The audio input device is" << inputDeviceInfo.deviceName(); if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) { qDebug() << "The format to be used for audio input is" << _inputFormat; _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this); _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() * (_inputFormat.sampleRate() / SAMPLE_RATE) / CALLBACK_ACCELERATOR_RATIO; _audioInput->setBufferSize(_numInputCallbackBytes); QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); qDebug() << "The audio output device is" << outputDeviceInfo.deviceName(); if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { qDebug() << "The format to be used for audio output is" << _outputFormat; _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t)); _inputDevice = _audioInput->start(); connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput())); // setup our general output device for audio-mixer audio _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); _outputDevice = _audioOutput->start(); // setup a loopback audio output device _loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); // setup a procedural audio output device _proceduralAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); gettimeofday(&_lastReceiveTime, NULL); } return; } qDebug() << "Unable to set up audio I/O because of a problem with input or output formats.";} void Audio::handleAudioInput() { static char monoAudioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMicrophoneAudioNoEcho); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat); static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; static unsigned int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; QByteArray inputByteArray = _inputDevice->readAll(); if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { // if this person wants local loopback add that to the locally injected audio if (!_loopbackOutputDevice) { // we didn't have the loopback output device going so set that up now _loopbackOutputDevice = _loopbackAudioOutput->start(); } if (_inputFormat == _outputFormat) { _loopbackOutputDevice->write(inputByteArray); } else { static float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) * (_outputFormat.channelCount() / _inputFormat.channelCount()); QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0); linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(), inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat); _loopbackOutputDevice->write(loopBackByteArray); } } _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); // zero out the monoAudioSamples array and the locally injected audio memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // zero out the locally injected audio in preparation for audio procedural sounds memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); if (!_muted) { // we aren't muted, downsample the input audio linearResampling((int16_t*) inputAudioSamples, monoAudioSamples, inputSamplesRequired, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, _inputFormat, _desiredInputFormat); // // Impose Noise Gate // // The Noise Gate is used to reject constant background noise by measuring the noise // floor observed at the microphone and then opening the 'gate' to allow microphone // signals to be transmitted when the microphone samples average level exceeds a multiple // of the noise floor. // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded // to open the gate. // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames // will we wait before closing the gate. // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? float loudness = 0; float thisSample = 0; int samplesOverNoiseGate = 0; const float NOISE_GATE_HEIGHT = 7.f; const int NOISE_GATE_WIDTH = 5; const int NOISE_GATE_CLOSE_FRAME_DELAY = 5; const int NOISE_GATE_FRAMES_TO_AVERAGE = 5; const float DC_OFFSET_AVERAGING = 0.99f; float measuredDcOffset = 0.f; for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { measuredDcOffset += monoAudioSamples[i]; monoAudioSamples[i] -= (int16_t) _dcOffset; thisSample = fabsf(monoAudioSamples[i]); loudness += thisSample; // Noise Reduction: Count peaks above the average loudness if (thisSample > (_noiseGateMeasuredFloor * NOISE_GATE_HEIGHT)) { samplesOverNoiseGate++; } } measuredDcOffset /= NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; if (_dcOffset == 0.f) { // On first frame, copy over measured offset _dcOffset = measuredDcOffset; } else { _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.f - DC_OFFSET_AVERAGING) * measuredDcOffset; } // _lastInputLoudness = fabs(loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); float averageOfAllSampleFrames = 0.f; _noiseSampleFrames[_noiseGateSampleCounter++] = _lastInputLoudness; if (_noiseGateSampleCounter == NUMBER_OF_NOISE_SAMPLE_FRAMES) { float smallestSample = FLT_MAX; for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_FRAMES - NOISE_GATE_FRAMES_TO_AVERAGE; i+= NOISE_GATE_FRAMES_TO_AVERAGE) { float thisAverage = 0.0f; for (int j = i; j < i + NOISE_GATE_FRAMES_TO_AVERAGE; j++) { thisAverage += _noiseSampleFrames[j]; averageOfAllSampleFrames += _noiseSampleFrames[j]; } thisAverage /= NOISE_GATE_FRAMES_TO_AVERAGE; if (thisAverage < smallestSample) { smallestSample = thisAverage; } } averageOfAllSampleFrames /= NUMBER_OF_NOISE_SAMPLE_FRAMES; _noiseGateMeasuredFloor = smallestSample; _noiseGateSampleCounter = 0; } if (_noiseGateEnabled) { if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { _noiseGateOpen = true; _noiseGateFramesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; } else { if (--_noiseGateFramesToClose == 0) { _noiseGateOpen = false; } } if (!_noiseGateOpen) { for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { monoAudioSamples[i] = 0; } } } // add input data just written to the scope QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), Q_ARG(bool, false), Q_ARG(bool, true)); } else { // our input loudness is 0, since we're muted _lastInputLoudness = 0; } // add procedural effects to the appropriate input samples addProceduralSounds(monoAudioSamples, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); if (!_proceduralOutputDevice) { _proceduralOutputDevice = _proceduralAudioOutput->start(); } // send whatever procedural sounds we want to locally loop back to the _proceduralOutputDevice QByteArray proceduralOutput; proceduralOutput.resize(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 4 * sizeof(int16_t)); linearResampling(_localProceduralSamples, reinterpret_cast<int16_t*>(proceduralOutput.data()), NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 4, _desiredInputFormat, _outputFormat); _proceduralOutputDevice->write(proceduralOutput); NodeList* nodeList = NodeList::getInstance(); SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer && audioMixer->getActiveSocket()) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getOrientation(); // we need the amount of bytes in the buffer + 1 for type // + 12 for 3 floats for position + float for bearing + 1 attenuation byte PacketType packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) ? PacketTypeMicrophoneAudioWithEcho : PacketTypeMicrophoneAudioNoEcho; char* currentPacketPtr = monoAudioDataPacket + populatePacketHeader(monoAudioDataPacket, packetType); // memcpy the three float positions memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); currentPacketPtr += (sizeof(headPosition)); Lines 1–500 of 862. Load more
// // Audio.cpp // interface // // Created by Stephen Birarda on 1/22/13. // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // #include <cstring> #include <sys/stat.h> #include <math.h> #ifdef __APPLE__ #include <CoreAudio/AudioHardware.h> #endif #include <QtCore/QBuffer> #include <QtMultimedia/QAudioInput> #include <QtMultimedia/QAudioOutput> #include <QSvgRenderer> #include <AngleUtil.h> #include <NodeList.h> #include <PacketHeaders.h> #include <SharedUtil.h> #include <StdDev.h> #include <UUID.h> #include "Application.h" #include "Audio.h" #include "Menu.h" #include "Util.h" static const float JITTER_BUFFER_LENGTH_MSECS = 12; static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; // Mute icon configration static const int ICON_SIZE = 24; static const int ICON_LEFT = 0; static const int ICON_TOP = 115; static const int ICON_TOP_MIRROR = 220; Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent) : AbstractAudioInterface(parent), _audioInput(NULL), _desiredInputFormat(), _inputFormat(), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputDevice(NULL), _numOutputCallbackBytes(0), _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _proceduralAudioOutput(NULL), _proceduralOutputDevice(NULL), _inputRingBuffer(0), _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), _jitterBufferSamples(initialJitterBufferSamples), _lastInputLoudness(0), _dcOffset(0), _noiseGateMeasuredFloor(0), _noiseGateSampleCounter(0), _noiseGateOpen(false), _noiseGateEnabled(true), _noiseGateFramesToClose(0), _lastVelocity(0), _lastAcceleration(0), _totalPacketsReceived(0), _collisionSoundMagnitude(0.0f), _collisionSoundFrequency(0.0f), _collisionSoundNoise(0.0f), _collisionSoundDuration(0.0f), _proceduralEffectSample(0), _numFramesDisplayStarve(0), _muted(false) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // Create the noise sample array _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; } void Audio::init(QGLWidget *parent) { switchToResourcesParentIfRequired(); _micTextureId = parent->bindTexture(QImage("./resources/images/mic.svg")); _muteTextureId = parent->bindTexture(QImage("./resources/images/mute.svg")); } void Audio::reset() { _ringBuffer.reset(); } QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { #ifdef __APPLE__ if (QAudioDeviceInfo::availableDevices(mode).size() > 1) { AudioDeviceID defaultDeviceID = 0; uint32_t propertySize = sizeof(AudioDeviceID); AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDefaultInputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; if (mode == QAudio::AudioOutput) { propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; } OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject, &propertyAddress, 0, NULL, &propertySize, &defaultDeviceID); if (!getPropertyError && propertySize) { CFStringRef deviceName = NULL; propertySize = sizeof(deviceName); propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString; getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0, NULL, &propertySize, &deviceName); if (!getPropertyError && propertySize) { // find a device in the list that matches the name we have and return it foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) { return audioDevice; } } } } } #endif // fallback for failed lookup is the default device return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice(); } bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, const QAudioFormat& desiredAudioFormat, QAudioFormat& adjustedAudioFormat) { if (!audioDevice.isFormatSupported(desiredAudioFormat)) { qDebug() << "The desired format for audio I/O is" << desiredAudioFormat; qDebug("The desired audio format is not supported by this device"); if (desiredAudioFormat.channelCount() == 1) { adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setChannelCount(2); if (audioDevice.isFormatSupported(adjustedAudioFormat)) { return true; } else { adjustedAudioFormat.setChannelCount(1); } } if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) { // use 48, which is a sample downsample, upsample adjustedAudioFormat = desiredAudioFormat; adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2); // return the nearest in case it needs 2 channels adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); return true; } return false; } else { // set the adjustedAudioFormat to the desiredAudioFormat, since it will work adjustedAudioFormat = desiredAudioFormat; return true; } } void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples, unsigned int numDestinationSamples, const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount()); // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels // this is the case with our HDMI capture if (sourceToDestinationFactor >= 2) { // we need to downsample from 48 to 24 // for now this only supports a mono output - this would be the case for audio input for (unsigned int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) { if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 2) + (sourceSamples[i] / 2); } else { destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = (sourceSamples[i - sourceAudioFormat.channelCount()] / 4) + (sourceSamples[i] / 2) + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4); } } } else { // upsample from 24 to 48 // for now this only supports a stereo to stereo conversion - this is our case for network audio to output int sourceIndex = 0; int dtsSampleRateFactor = (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()); int sampleShift = destinationAudioFormat.channelCount() * dtsSampleRateFactor; int destinationToSourceFactor = (1 / sourceToDestinationFactor); for (unsigned int i = 0; i < numDestinationSamples; i += sampleShift) { sourceIndex = (i / destinationToSourceFactor); // fill the L/R channels and make the rest silent for (unsigned int j = i; j < i + sampleShift; j++) { if (j % destinationAudioFormat.channelCount() == 0) { // left channel destinationSamples[j] = sourceSamples[sourceIndex]; } else if (j % destinationAudioFormat.channelCount() == 1) { // right channel destinationSamples[j] = sourceSamples[sourceIndex + (sourceAudioFormat.channelCount() > 1 ? 1 : 0)]; } else { // channels above 2, fill with silence destinationSamples[j] = 0; } } } } } } const int CALLBACK_ACCELERATOR_RATIO = 2; void Audio::start() { // set up the desired audio format _desiredInputFormat.setSampleRate(SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); _desiredInputFormat.setCodec("audio/pcm"); _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); _desiredInputFormat.setChannelCount(1); _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(2); QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); qDebug() << "The audio input device is" << inputDeviceInfo.deviceName(); if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) { qDebug() << "The format to be used for audio input is" << _inputFormat; _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this); _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() * (_inputFormat.sampleRate() / SAMPLE_RATE) / CALLBACK_ACCELERATOR_RATIO; _audioInput->setBufferSize(_numInputCallbackBytes); QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); qDebug() << "The audio output device is" << outputDeviceInfo.deviceName(); if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { qDebug() << "The format to be used for audio output is" << _outputFormat; _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t)); _inputDevice = _audioInput->start(); connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput())); // setup our general output device for audio-mixer audio _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); _outputDevice = _audioOutput->start(); // setup a loopback audio output device _loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); // setup a procedural audio output device _proceduralAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); gettimeofday(&_lastReceiveTime, NULL); } return; } qDebug() << "Unable to set up audio I/O because of a problem with input or output formats."; } void Audio::handleAudioInput() { static char monoAudioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMicrophoneAudioNoEcho); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat); static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; static unsigned int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; QByteArray inputByteArray = _inputDevice->readAll(); if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { // if this person wants local loopback add that to the locally injected audio if (!_loopbackOutputDevice) { // we didn't have the loopback output device going so set that up now _loopbackOutputDevice = _loopbackAudioOutput->start(); } if (_inputFormat == _outputFormat) { _loopbackOutputDevice->write(inputByteArray); } else { static float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) * (_outputFormat.channelCount() / _inputFormat.channelCount()); QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0); linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(), inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat); _loopbackOutputDevice->write(loopBackByteArray); } } _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); // zero out the monoAudioSamples array and the locally injected audio memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // zero out the locally injected audio in preparation for audio procedural sounds memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); if (!_muted) { // we aren't muted, downsample the input audio linearResampling((int16_t*) inputAudioSamples, monoAudioSamples, inputSamplesRequired, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, _inputFormat, _desiredInputFormat); // // Impose Noise Gate // // The Noise Gate is used to reject constant background noise by measuring the noise // floor observed at the microphone and then opening the 'gate' to allow microphone // signals to be transmitted when the microphone samples average level exceeds a multiple // of the noise floor. // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded // to open the gate. // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames // will we wait before closing the gate. // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? float loudness = 0; float thisSample = 0; int samplesOverNoiseGate = 0; const float NOISE_GATE_HEIGHT = 7.f; const int NOISE_GATE_WIDTH = 5; const int NOISE_GATE_CLOSE_FRAME_DELAY = 5; const int NOISE_GATE_FRAMES_TO_AVERAGE = 5; const float DC_OFFSET_AVERAGING = 0.99f; float measuredDcOffset = 0.f; for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { measuredDcOffset += monoAudioSamples[i]; monoAudioSamples[i] -= (int16_t) _dcOffset; thisSample = fabsf(monoAudioSamples[i]); loudness += thisSample; // Noise Reduction: Count peaks above the average loudness if (thisSample > (_noiseGateMeasuredFloor * NOISE_GATE_HEIGHT)) { samplesOverNoiseGate++; } } measuredDcOffset /= NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; if (_dcOffset == 0.f) { // On first frame, copy over measured offset _dcOffset = measuredDcOffset; } else { _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.f - DC_OFFSET_AVERAGING) * measuredDcOffset; } // _lastInputLoudness = fabs(loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); float averageOfAllSampleFrames = 0.f; _noiseSampleFrames[_noiseGateSampleCounter++] = _lastInputLoudness; if (_noiseGateSampleCounter == NUMBER_OF_NOISE_SAMPLE_FRAMES) { float smallestSample = FLT_MAX; for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_FRAMES - NOISE_GATE_FRAMES_TO_AVERAGE; i+= NOISE_GATE_FRAMES_TO_AVERAGE) { float thisAverage = 0.0f; for (int j = i; j < i + NOISE_GATE_FRAMES_TO_AVERAGE; j++) { thisAverage += _noiseSampleFrames[j]; averageOfAllSampleFrames += _noiseSampleFrames[j]; } thisAverage /= NOISE_GATE_FRAMES_TO_AVERAGE; if (thisAverage < smallestSample) { smallestSample = thisAverage; } } averageOfAllSampleFrames /= NUMBER_OF_NOISE_SAMPLE_FRAMES; _noiseGateMeasuredFloor = smallestSample; _noiseGateSampleCounter = 0; } if (_noiseGateEnabled) { if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { _noiseGateOpen = true; _noiseGateFramesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; } else { if (--_noiseGateFramesToClose == 0) { _noiseGateOpen = false; } } if (!_noiseGateOpen) { for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { monoAudioSamples[i] = 0; } } } // add input data just written to the scope QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), Q_ARG(bool, false), Q_ARG(bool, true)); } else { // our input loudness is 0, since we're muted _lastInputLoudness = 0; } // add procedural effects to the appropriate input samples addProceduralSounds(monoAudioSamples, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); if (!_proceduralOutputDevice) { _proceduralOutputDevice = _proceduralAudioOutput->start(); } // send whatever procedural sounds we want to locally loop back to the _proceduralOutputDevice QByteArray proceduralOutput; proceduralOutput.resize(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 4 * sizeof(int16_t)); linearResampling(_localProceduralSamples, reinterpret_cast<int16_t*>(proceduralOutput.data()), NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 4, _desiredInputFormat, _outputFormat); _proceduralOutputDevice->write(proceduralOutput); NodeList* nodeList = NodeList::getInstance(); SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer && audioMixer->getActiveSocket()) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getOrientation(); // we need the amount of bytes in the buffer + 1 for type // + 12 for 3 floats for position + float for bearing + 1 attenuation byte PacketType packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) ? PacketTypeMicrophoneAudioWithEcho : PacketTypeMicrophoneAudioNoEcho; char* currentPacketPtr = monoAudioDataPacket + populatePacketHeader(monoAudioDataPacket, packetType); // memcpy the three float positions memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); currentPacketPtr += (sizeof(headPosition));