////  Audio.cpp//  interface////  Created by Stephen Birarda on 1/22/13.//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.// #include <cstring>#include <sys/stat.h> #ifdef __APPLE__#include <CoreAudio/AudioHardware.h>#endif #include <QtCore/QBuffer>#include <QtMultimedia/QAudioInput>#include <QtMultimedia/QAudioOutput>#include <QSvgRenderer> #include <AngleUtil.h>#include <NodeList.h>#include <NodeTypes.h>#include <PacketHeaders.h>#include <SharedUtil.h>#include <StdDev.h>#include <UUID.h> #include "Application.h"#include "Audio.h"#include "Menu.h"#include "Util.h" static const float JITTER_BUFFER_LENGTH_MSECS = 12;static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; // Mute icon configrationstatic const int ICON_SIZE = 24;static const int ICON_LEFT = 20;static const int BOTTOM_PADDING = 110; Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent) :    AbstractAudioInterface(parent),    _audioInput(NULL),    _desiredInputFormat(),    _inputFormat(),    _numInputCallbackBytes(0),    _audioOutput(NULL),    _desiredOutputFormat(),    _outputFormat(),    _outputDevice(NULL),    _numOutputCallbackBytes(0),    _loopbackAudioOutput(NULL),    _loopbackOutputDevice(NULL),    _inputRingBuffer(0),    _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL),    _scope(scope),    _averagedLatency(0.0),    _measuredJitter(0),    _jitterBufferSamples(initialJitterBufferSamples),    _lastInputLoudness(0),    _lastVelocity(0),    _lastAcceleration(0),    _totalPacketsReceived(0),    _collisionSoundMagnitude(0.0f),    _collisionSoundFrequency(0.0f),    _collisionSoundNoise(0.0f),    _collisionSoundDuration(0.0f),    _proceduralEffectSample(0),    _numFramesDisplayStarve(0),    _muted(false){    // clear the array of locally injected samples    memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);} void Audio::init(QGLWidget *parent) {    switchToResourcesParentIfRequired();    _micTextureId = parent->bindTexture(QImage("./resources/images/mic.svg"));    _muteTextureId = parent->bindTexture(QImage("./resources/images/mute.svg"));} void Audio::reset() {    _ringBuffer.reset();} QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {#ifdef __APPLE__    if (QAudioDeviceInfo::availableDevices(mode).size() > 1) {        AudioDeviceID defaultDeviceID = 0;        uint32_t propertySize = sizeof(AudioDeviceID);        AudioObjectPropertyAddress propertyAddress = {            kAudioHardwarePropertyDefaultInputDevice,            kAudioObjectPropertyScopeGlobal,            kAudioObjectPropertyElementMaster        };                if (mode == QAudio::AudioOutput) {            propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice;        }                        OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject,                                                               &propertyAddress,                                                               0,                                                               NULL,                                                               &propertySize,                                                               &defaultDeviceID);                if (!getPropertyError && propertySize) {            CFStringRef deviceName = NULL;            propertySize = sizeof(deviceName);            propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString;            getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0,                                                          NULL, &propertySize, &deviceName);                        if (!getPropertyError && propertySize) {                // find a device in the list that matches the name we have and return it                foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) {                    if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) {                        return audioDevice;                    }                }            }        }    }#endif        // fallback for failed lookup is the default device    return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice();} bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,                                  const QAudioFormat& desiredAudioFormat,                                  QAudioFormat& adjustedAudioFormat) {    if (!audioDevice.isFormatSupported(desiredAudioFormat)) {        qDebug() << "The desired format for audio I/O is" << desiredAudioFormat << "\n";        qDebug() << "The desired audio format is not supported by this device.\n";                if (desiredAudioFormat.channelCount() == 1) {            adjustedAudioFormat = desiredAudioFormat;            adjustedAudioFormat.setChannelCount(2);                        if (audioDevice.isFormatSupported(adjustedAudioFormat)) {                return true;            } else {                adjustedAudioFormat.setChannelCount(1);            }        }                if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) {            // use 48, which is a sample downsample, upsample            adjustedAudioFormat = desiredAudioFormat;            adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2);                        // return the nearest in case it needs 2 channels            adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat);            return true;        }                return false;    } else {        // set the adjustedAudioFormat to the desiredAudioFormat, since it will work        adjustedAudioFormat = desiredAudioFormat;        return true;    }} void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples,                      unsigned int numSourceSamples, unsigned int numDestinationSamples,                      const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) {    if (sourceAudioFormat == destinationAudioFormat) {        memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t));    } else {        float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate())            * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount());                // take into account the number of channels in source and destination        // accomodate for the case where have an output with > 2 channels        // this is the case with our HDMI capture                if (sourceToDestinationFactor >= 2) {            // we need to downsample from 48 to 24            // for now this only supports a mono output - this would be the case for audio input                        for (int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) {                if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) {                    destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] =                        (sourceSamples[i - sourceAudioFormat.channelCount()] / 2)                        + (sourceSamples[i] / 2);                } else {                    destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] =                        (sourceSamples[i - sourceAudioFormat.channelCount()] / 4)                        + (sourceSamples[i] / 2)                        + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4);                }            }                    } else {            // upsample from 24 to 48            // for now this only supports a stereo to stereo conversion - this is our case for network audio to output            int sourceIndex = 0;            int dtsSampleRateFactor = (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate());            int sampleShift = destinationAudioFormat.channelCount() * dtsSampleRateFactor;            int destinationToSourceFactor = (1 / sourceToDestinationFactor);                        for (int i = 0; i < numDestinationSamples; i += sampleShift) {                sourceIndex = (i / destinationToSourceFactor);                                // fill the L/R channels and make the rest silent                for (int j = i; j < i + sampleShift; j++) {                    if (j % destinationAudioFormat.channelCount() == 0) {                        // left channel                        destinationSamples[j] = sourceSamples[sourceIndex];                    } else if (j % destinationAudioFormat.channelCount() == 1) {                         // right channel                        destinationSamples[j] = sourceSamples[sourceIndex + (sourceAudioFormat.channelCount() > 1 ? 1 : 0)];                    } else {                        // channels above 2, fill with silence                        destinationSamples[j] = 0;                    }                }            }        }    }} const int CALLBACK_ACCELERATOR_RATIO = 2; void Audio::start() {        // set up the desired audio format    _desiredInputFormat.setSampleRate(SAMPLE_RATE);    _desiredInputFormat.setSampleSize(16);    _desiredInputFormat.setCodec("audio/pcm");    _desiredInputFormat.setSampleType(QAudioFormat::SignedInt);    _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian);    _desiredInputFormat.setChannelCount(1);        _desiredOutputFormat = _desiredInputFormat;    _desiredOutputFormat.setChannelCount(2);        QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput);        qDebug() << "The audio input device is" << inputDeviceInfo.deviceName() << "\n";        if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {        qDebug() << "The format to be used for audio input is" << _inputFormat << "\n";                _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);        _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount()            * (_inputFormat.sampleRate() / SAMPLE_RATE)            / CALLBACK_ACCELERATOR_RATIO;        _audioInput->setBufferSize(_numInputCallbackBytes);                QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput);                qDebug() << "The audio output device is" << outputDeviceInfo.deviceName() << "\n";         if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {            qDebug() << "The format to be used for audio output is" << _outputFormat << "\n";                        _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t));            _inputDevice = _audioInput->start();            connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput()));                        // setup our general output device for audio-mixer audio            _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);            _outputDevice = _audioOutput->start();                        // setup a loopback audio output device            _loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);                        gettimeofday(&_lastReceiveTime, NULL);        }                return;    }        qDebug() << "Unable to set up audio I/O because of a problem with input or output formats.\n";} void Audio::handleAudioInput() {    static char monoAudioDataPacket[MAX_PACKET_SIZE];        static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO);    static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) +  NUM_BYTES_RFC4122_UUID;        static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes);        static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO        / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL;        static int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio;        QByteArray inputByteArray = _inputDevice->readAll();        if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) {        // if this person wants local loopback add that to the locally injected audio                if (!_loopbackOutputDevice) {            // we didn't have the loopback output device going so set that up now            _loopbackOutputDevice = _loopbackAudioOutput->start();        }                if (_inputFormat == _outputFormat) {            _loopbackOutputDevice->write(inputByteArray);        } else {            static float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate())                * (_outputFormat.channelCount() / _inputFormat.channelCount());                        QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0);                        linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(),                             inputByteArray.size() / sizeof(int16_t),                             loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat);                        _loopbackOutputDevice->write(loopBackByteArray);        }    }        _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());        while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) {                int16_t inputAudioSamples[inputSamplesRequired];        _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired);                // zero out the monoAudioSamples array and the locally injected audio        memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);                // zero out the locally injected audio in preparation for audio procedural sounds        memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);                if (!_muted) {            // we aren't muted, downsample the input audio            linearResampling((int16_t*) inputAudioSamples,                             monoAudioSamples,                             inputSamplesRequired,                             NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL,                             _inputFormat, _desiredInputFormat);                        float loudness = 0;                        for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {                loudness += fabsf(monoAudioSamples[i]);            }                        _lastInputLoudness = loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL;                        // add input data just written to the scope            QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection,                                      Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples,                                                                   NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)),                                      Q_ARG(bool, false), Q_ARG(bool, true));        } else {            // our input loudness is 0, since we're muted            _lastInputLoudness = 0;        }                // add procedural effects to the appropriate input samples        addProceduralSounds(monoAudioSamples,                            NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);                NodeList* nodeList = NodeList::getInstance();        Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER);                if (audioMixer && nodeList->getNodeActiveSocketOrPing(audioMixer)) {            MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar();                        glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition();            glm::quat headOrientation = interfaceAvatar->getHead().getOrientation();                        // we need the amount of bytes in the buffer + 1 for type            // + 12 for 3 floats for position + float for bearing + 1 attenuation byte                        PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)                ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO;                        char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket,                                                                                  packetType);                        // pack Source Data            QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122();            memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size());            currentPacketPtr += rfcUUID.size();                        // memcpy the three float positions            memcpy(currentPacketPtr, &headPosition, sizeof(headPosition));            currentPacketPtr += (sizeof(headPosition));                        // memcpy our orientation            memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));            currentPacketPtr += sizeof(headOrientation);                        nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket,                                                    NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes,                                                    audioMixer->getActiveSocket()->getAddress(),                                                    audioMixer->getActiveSocket()->getPort());                        Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO)                .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes);        }    }} void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) {    const int NUM_INITIAL_PACKETS_DISCARD = 3;    const int STANDARD_DEVIATION_SAMPLE_COUNT = 500;        timeval currentReceiveTime;    gettimeofday(&currentReceiveTime, NULL);    _totalPacketsReceived++;        double timeDiff = diffclock(&_lastReceiveTime, &currentReceiveTime);        //  Discard first few received packets for computing jitter (often they pile up on start)    if (_totalPacketsReceived > NUM_INITIAL_PACKETS_DISCARD) {        _stdev.addValue(timeDiff);    }        if (_stdev.getSamples() > STANDARD_DEVIATION_SAMPLE_COUNT) {        _measuredJitter = _stdev.getStDev();        _stdev.reset();        //  Set jitter buffer to be a multiple of the measured standard deviation        const int MAX_JITTER_BUFFER_SAMPLES = _ringBuffer.getSampleCapacity() / 2;        const float NUM_STANDARD_DEVIATIONS = 3.f;        if (Menu::getInstance()->getAudioJitterBufferSamples() == 0) {            float newJitterBufferSamples = (NUM_STANDARD_DEVIATIONS * _measuredJitter) / 1000.f * SAMPLE_RATE;            setJitterBufferSamples(glm::clamp((int)newJitterBufferSamples, 0, MAX_JITTER_BUFFER_SAMPLES));        }    }        _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size());        static float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate())        * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount());        static int numRequiredOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / networkOutputToOutputRatio;        QByteArray outputBuffer;    outputBuffer.resize(numRequiredOutputSamples * sizeof(int16_t));        // if there is anything in the ring buffer, decide what to do    if (_ringBuffer.samplesAvailable() > 0) {        if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO                                                         + (_jitterBufferSamples * 2))) {            // starved and we don't have enough to start, keep waiting            qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n";        } else {            //  We are either already playing back, or we have enough audio to start playing back.            _ringBuffer.setIsStarved(false);                        // copy the samples we'll resample from the ring buffer - this also            // pushes the read pointer of the ring buffer forwards            int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];            _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO);                        // add the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL from each QByteArray            // in our _localInjectionByteArrays QVector to the _localInjectedSamples                        // add to the output samples whatever is in the _localAudioOutput byte array            // that lets this user hear sound effects and loopback (if enabled)                        for (int b = 0; b < _localInjectionByteArrays.size(); b++) {                QByteArray audioByteArray = _localInjectionByteArrays.at(b);                                int16_t* byteArraySamples = (int16_t*) audioByteArray.data();                                int samplesToRead = MIN(audioByteArray.size() / sizeof(int16_t),                                        NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);                                for (int i = 0; i < samplesToRead; i++) {                    _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + byteArraySamples[i],                                                          MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);                }                                if (samplesToRead < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) {                    // there isn't anything left to inject from this byte array, remove it from the vector                    _localInjectionByteArrays.remove(b);                } else {                    // pull out the bytes we just read for outputs                    audioByteArray.remove(0, samplesToRead * sizeof(int16_t));                                        // still data left to read - replace the byte array in the QVector with the smaller one                    _localInjectionByteArrays.replace(b, audioByteArray);                }            }                        for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {                ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _localInjectedSamples[i],                                                      MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);                ringBufferSamples[(i * 2) + 1] = glm::clamp(ringBufferSamples[(i * 2) + 1] + _localInjectedSamples[i],                                                            MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);            }                        // copy the packet from the RB to the output            linearResampling(ringBufferSamples,

Lines 1–500 of 775. Load more