////  Audio.cpp//  interface/src////  Created by Stephen Birarda on 1/22/13.//  Copyright 2013 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include <cstring>#include <sys/stat.h> #include <math.h> #ifdef __APPLE__#include <CoreAudio/AudioHardware.h>#endif #ifdef WIN32#define WIN32_LEAN_AND_MEAN#include <windows.h>#include <Mmsystem.h>#include <mmdeviceapi.h>#include <devicetopology.h>#include <Functiondiscoverykeys_devpkey.h>#endif #include <QtCore/QBuffer>#include <QtMultimedia/QAudioInput>#include <QtMultimedia/QAudioOutput>#include <QSvgRenderer> #include <NodeList.h>#include <PacketHeaders.h>#include <SharedUtil.h>#include <StdDev.h>#include <UUID.h>#include <glm/glm.hpp> #include "Audio.h" #include "Menu.h"#include "Util.h"#include "PositionalAudioStream.h" static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; static const int FRAMES_AVAILABLE_STATS_WINDOW_SECONDS = 10;static const int APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS = (int)(30.0f * 1000.0f / AUDIO_CALLBACK_MSECS); // Mute icon configrationstatic const int MUTE_ICON_SIZE = 24; static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 100; Audio::Audio(QObject* parent) :    AbstractAudioInterface(parent),    _audioInput(NULL),    _desiredInputFormat(),    _inputFormat(),    _numInputCallbackBytes(0),    _audioOutput(NULL),    _desiredOutputFormat(),    _outputFormat(),    _outputFrameSize(0),    _numOutputCallbackBytes(0),    _loopbackAudioOutput(NULL),    _loopbackOutputDevice(NULL),    _proceduralAudioOutput(NULL),    _proceduralOutputDevice(NULL),    _inputRingBuffer(0),    _receivedAudioStream(0, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, InboundAudioStream::Settings()),    _isStereoInput(false),    _averagedLatency(0.0),    _lastInputLoudness(0),    _timeSinceLastClip(-1.0),    _dcOffset(0),    _noiseGateMeasuredFloor(0),    _noiseGateSampleCounter(0),    _noiseGateOpen(false),    _noiseGateEnabled(true),    _audioSourceInjectEnabled(false),    _noiseGateFramesToClose(0),    _totalInputAudioSamples(0),    _collisionSoundMagnitude(0.0f),    _collisionSoundFrequency(0.0f),    _collisionSoundNoise(0.0f),    _collisionSoundDuration(0.0f),    _proceduralEffectSample(0),    _muted(false),    _reverb(false),    _reverbOptions(&_scriptReverbOptions),    _processSpatialAudio(false),    _spatialAudioStart(0),    _spatialAudioFinish(0),    _spatialAudioRingBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, true), // random access mode    _scopeEnabled(false),    _scopeEnabledPause(false),    _scopeInputOffset(0),    _scopeOutputOffset(0),    _framesPerScope(DEFAULT_FRAMES_PER_SCOPE),    _samplesPerScope(NETWORK_SAMPLES_PER_FRAME * _framesPerScope),    _noiseSourceEnabled(false),    _toneSourceEnabled(true),    _peqEnabled(false),    _scopeInput(0),    _scopeOutputLeft(0),    _scopeOutputRight(0),    _scopeLastFrame(),    _statsEnabled(false),    _statsShowInjectedStreams(false),    _outgoingAvatarAudioSequenceNumber(0),    _audioInputMsecsReadStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS),    _inputRingBufferMsecsAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS),    _audioOutputMsecsUnplayedStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS),    _lastSentAudioPacket(0),    _packetSentTimeGaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS),    _audioOutputIODevice(_receivedAudioStream){    // clear the array of locally injected samples    memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);    // Create the noise sample array    _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES];        connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedSilence, this, &Audio::addStereoSilenceToScope, Qt::DirectConnection);    connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedLastFrameRepeatedWithFade, this, &Audio::addLastFrameRepeatedWithFadeToScope, Qt::DirectConnection);    connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedStereoSamples, this, &Audio::addStereoSamplesToScope, Qt::DirectConnection);    connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedSamples, Qt::DirectConnection);        // Initialize GVerb    initGverb();} void Audio::init(QGLWidget *parent) {    _micTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic.svg"));    _muteTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic-mute.svg"));    _boxTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/audio-box.svg"));} void Audio::reset() {    _receivedAudioStream.reset();    resetStats();    _peq.reset();    _noiseSource.reset();    _toneSource.reset();    _sourceGain.reset();    _inputGain.reset();} void Audio::resetStats() {    _receivedAudioStream.resetStats();     _audioMixerAvatarStreamAudioStats = AudioStreamStats();    _audioMixerInjectedStreamAudioStatsMap.clear();     _audioInputMsecsReadStats.reset();    _inputRingBufferMsecsAvailableStats.reset();     _audioOutputMsecsUnplayedStats.reset();    _packetSentTimeGaps.reset();} void Audio::audioMixerKilled() {    _outgoingAvatarAudioSequenceNumber = 0;    resetStats();}  QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) {    QAudioDeviceInfo result;#ifdef WIN32    // NOTE    // this is a workaround for a windows only QtBug https://bugreports.qt-project.org/browse/QTBUG-16117    // static QAudioDeviceInfo objects get deallocated when QList<QAudioDevieInfo> objects go out of scope    result = (mode == QAudio::AudioInput) ?         QAudioDeviceInfo::defaultInputDevice() :         QAudioDeviceInfo::defaultOutputDevice();#else    foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) {        qDebug() << audioDevice.deviceName() << " " << deviceName;        if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) {            result = audioDevice;        }    }#endif    return result;} QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {#ifdef __APPLE__    if (QAudioDeviceInfo::availableDevices(mode).size() > 1) {        AudioDeviceID defaultDeviceID = 0;        uint32_t propertySize = sizeof(AudioDeviceID);        AudioObjectPropertyAddress propertyAddress = {            kAudioHardwarePropertyDefaultInputDevice,            kAudioObjectPropertyScopeGlobal,            kAudioObjectPropertyElementMaster        };         if (mode == QAudio::AudioOutput) {            propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice;        }          OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject,                                                               &propertyAddress,                                                               0,                                                               NULL,                                                               &propertySize,                                                               &defaultDeviceID);         if (!getPropertyError && propertySize) {            CFStringRef deviceName = NULL;            propertySize = sizeof(deviceName);            propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString;            getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0,                                                          NULL, &propertySize, &deviceName);             if (!getPropertyError && propertySize) {                // find a device in the list that matches the name we have and return it                foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) {                    if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) {                        return audioDevice;                    }                }            }        }    }#endif#ifdef WIN32    QString deviceName;    //Check for Windows Vista or higher, IMMDeviceEnumerator doesn't work below that.    OSVERSIONINFO osvi;    ZeroMemory(&osvi, sizeof(OSVERSIONINFO));    osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);    GetVersionEx(&osvi);    const DWORD VISTA_MAJOR_VERSION = 6;    if (osvi.dwMajorVersion < VISTA_MAJOR_VERSION) {// lower then vista        if (mode == QAudio::AudioInput) {            WAVEINCAPS wic;            // first use WAVE_MAPPER to get the default devices manufacturer ID            waveInGetDevCaps(WAVE_MAPPER, &wic, sizeof(wic));            //Use the received manufacturer id to get the device's real name            waveInGetDevCaps(wic.wMid, &wic, sizeof(wic));            qDebug() << "input device:" << wic.szPname;            deviceName = wic.szPname;        } else {            WAVEOUTCAPS woc;            // first use WAVE_MAPPER to get the default devices manufacturer ID            waveOutGetDevCaps(WAVE_MAPPER, &woc, sizeof(woc));            //Use the received manufacturer id to get the device's real name            waveOutGetDevCaps(woc.wMid, &woc, sizeof(woc));            qDebug() << "output device:" << woc.szPname;            deviceName = woc.szPname;        }    } else {        HRESULT hr = S_OK;        CoInitialize(NULL);        IMMDeviceEnumerator* pMMDeviceEnumerator = NULL;        CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator);        IMMDevice* pEndpoint;        hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, &pEndpoint);        if (hr == E_NOTFOUND) {            printf("Audio Error: device not found\n");            deviceName = QString("NONE");        } else {            IPropertyStore* pPropertyStore;            pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore);            pEndpoint->Release();            pEndpoint = NULL;            PROPVARIANT pv;            PropVariantInit(&pv);            hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv);            pPropertyStore->Release();            pPropertyStore = NULL;            deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal);            const DWORD WINDOWS7_MAJOR_VERSION = 6;            const DWORD WINDOWS7_MINOR_VERSION = 1;            if (osvi.dwMajorVersion <= WINDOWS7_MAJOR_VERSION && osvi.dwMinorVersion <= WINDOWS7_MINOR_VERSION) {                // Windows 7 provides only the 31 first characters of the device name.                const DWORD QT_WIN7_MAX_AUDIO_DEVICENAME_LEN = 31;                deviceName = deviceName.left(QT_WIN7_MAX_AUDIO_DEVICENAME_LEN);            }            qDebug() << (mode == QAudio::AudioOutput ? "output" : "input") << " device:" << deviceName;            PropVariantClear(&pv);        }        pMMDeviceEnumerator->Release();        pMMDeviceEnumerator = NULL;        CoUninitialize();    }     qDebug() << "DEBUG [" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]";        return getNamedAudioDeviceForMode(mode, deviceName);#endif      // fallback for failed lookup is the default device    return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice();} bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,                                  const QAudioFormat& desiredAudioFormat,                                  QAudioFormat& adjustedAudioFormat) {    if (!audioDevice.isFormatSupported(desiredAudioFormat)) {        qDebug() << "The desired format for audio I/O is" << desiredAudioFormat;        qDebug("The desired audio format is not supported by this device");                if (desiredAudioFormat.channelCount() == 1) {            adjustedAudioFormat = desiredAudioFormat;            adjustedAudioFormat.setChannelCount(2);             if (audioDevice.isFormatSupported(adjustedAudioFormat)) {                return true;            } else {                adjustedAudioFormat.setChannelCount(1);            }        }         if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) {            // use 48, which is a sample downsample, upsample            adjustedAudioFormat = desiredAudioFormat;            adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2);             // return the nearest in case it needs 2 channels            adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat);            return true;        }         return false;    } else {        // set the adjustedAudioFormat to the desiredAudioFormat, since it will work        adjustedAudioFormat = desiredAudioFormat;        return true;    }} void linearResampling(const int16_t* sourceSamples, int16_t* destinationSamples,                      unsigned int numSourceSamples, unsigned int numDestinationSamples,                      const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) {    if (sourceAudioFormat == destinationAudioFormat) {        memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t));    } else {        float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate())            * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount());         // take into account the number of channels in source and destination        // accomodate for the case where have an output with > 2 channels        // this is the case with our HDMI capture         if (sourceToDestinationFactor >= 2) {            // we need to downsample from 48 to 24            // for now this only supports a mono output - this would be the case for audio input            if (destinationAudioFormat.channelCount() == 1) {                for (unsigned int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) {                    if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) {                        destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] =                        (sourceSamples[i - sourceAudioFormat.channelCount()] / 2)                        + (sourceSamples[i] / 2);                    } else {                        destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] =                        (sourceSamples[i - sourceAudioFormat.channelCount()] / 4)                        + (sourceSamples[i] / 2)                        + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4);                    }                }            } else {                // this is a 48 to 24 resampling but both source and destination are two channels                // squish two samples into one in each channel                for (unsigned int i = 0; i < numSourceSamples; i += 4) {                    destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 2] / 2);                    destinationSamples[(i / 2) + 1] = (sourceSamples[i + 1] / 2) + (sourceSamples[i + 3] / 2);                }            }        } else {            if (sourceAudioFormat.sampleRate() == destinationAudioFormat.sampleRate()) {                // mono to stereo, same sample rate                if (!(sourceAudioFormat.channelCount() == 1 && destinationAudioFormat.channelCount() == 2)) {                    qWarning() << "Unsupported format conversion" << sourceAudioFormat << destinationAudioFormat;                    return;                }                for (const int16_t* sourceEnd = sourceSamples + numSourceSamples; sourceSamples != sourceEnd;                        sourceSamples++) {                    *destinationSamples++ = *sourceSamples;                    *destinationSamples++ = *sourceSamples;                }                return;            }                    // upsample from 24 to 48            // for now this only supports a stereo to stereo conversion - this is our case for network audio to output            int sourceIndex = 0;            int dtsSampleRateFactor = (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate());            int sampleShift = destinationAudioFormat.channelCount() * dtsSampleRateFactor;            int destinationToSourceFactor = (1 / sourceToDestinationFactor);             for (unsigned int i = 0; i < numDestinationSamples; i += sampleShift) {                sourceIndex = (i / destinationToSourceFactor);                 // fill the L/R channels and make the rest silent                for (unsigned int j = i; j < i + sampleShift; j++) {                    if (j % destinationAudioFormat.channelCount() == 0) {                        // left channel                        destinationSamples[j] = sourceSamples[sourceIndex];                    } else if (j % destinationAudioFormat.channelCount() == 1) {                         // right channel                        destinationSamples[j] = sourceSamples[sourceIndex + (sourceAudioFormat.channelCount() > 1 ? 1 : 0)];                    } else {                        // channels above 2, fill with silence                        destinationSamples[j] = 0;                    }                }            }        }    }} void Audio::start() {     // set up the desired audio format    _desiredInputFormat.setSampleRate(SAMPLE_RATE);    _desiredInputFormat.setSampleSize(16);    _desiredInputFormat.setCodec("audio/pcm");    _desiredInputFormat.setSampleType(QAudioFormat::SignedInt);    _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian);    _desiredInputFormat.setChannelCount(1);     _desiredOutputFormat = _desiredInputFormat;    _desiredOutputFormat.setChannelCount(2);     QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput);    qDebug() << "The default audio input device is" << inputDeviceInfo.deviceName();    bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo);     QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput);    qDebug() << "The default audio output device is" << outputDeviceInfo.deviceName();    bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo);        if (!inputFormatSupported) {        qDebug() << "Unable to set up audio input because of a problem with input format.";    }    if (!outputFormatSupported) {        qDebug() << "Unable to set up audio output because of a problem with output format.";    }     _inputFrameBuffer.initialize( _inputFormat.channelCount(), _audioInput->bufferSize() * 8 );    _peq.initialize( _inputFormat.sampleRate() );     _inputGain.initialize();    _sourceGain.initialize();    _noiseSource.initialize();    _toneSource.initialize();    _sourceGain.setParameters(0.25f,0.0f);     _inputGain.setParameters(1.0f,0.0f);} void Audio::stop() {     _inputFrameBuffer.finalize();    _peq.finalize();    _inputGain.finalize();    _sourceGain.finalize();    _noiseSource.finalize();    _toneSource.finalize();        // "switch" to invalid devices in order to shut down the state    switchInputToAudioDevice(QAudioDeviceInfo());    switchOutputToAudioDevice(QAudioDeviceInfo());} QString Audio::getDefaultDeviceName(QAudio::Mode mode) {    QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode);    return deviceInfo.deviceName();} QVector<QString> Audio::getDeviceNames(QAudio::Mode mode) {    QVector<QString> deviceNames;    foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) {        deviceNames << audioDevice.deviceName().trimmed();    }    return deviceNames;} bool Audio::switchInputToAudioDevice(const QString& inputDeviceName) {    qDebug() << "DEBUG [" << inputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName).deviceName() << "]";    return switchInputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName));} bool Audio::switchOutputToAudioDevice(const QString& outputDeviceName) {    qDebug() << "DEBUG [" << outputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName).deviceName() << "]";    return switchOutputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName));} void Audio::initGverb() {    // Initialize a new gverb instance    _gverb = gverb_new(_outputFormat.sampleRate(), _reverbOptions->getMaxRoomSize(), _reverbOptions->getRoomSize(),                       _reverbOptions->getReverbTime(), _reverbOptions->getDamping(), _reverbOptions->getSpread(),

Lines 1–500 of 2014. Load more