////  AudioMixer.cpp//  assignment-client/src/audio////  Created by Stephen Birarda on 8/22/13.//  Copyright 2013 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include <errno.h>#include <fcntl.h>#include <fstream>#include <iostream>#include <math.h>#include <memory>#include <signal.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <thread> #ifdef _WIN32#include <math.h>#else#include <arpa/inet.h>#include <netinet/in.h>#include <sys/socket.h>#endif //_WIN32 #include <glm/glm.hpp>#include <glm/gtx/norm.hpp>#include <glm/gtx/vector_angle.hpp> #include <QtCore/QCoreApplication>#include <QtCore/QJsonArray>#include <QtCore/QJsonDocument>#include <QtCore/QJsonObject>#include <QtCore/QJsonValue>#include <QtCore/QThread>#include <QtNetwork/QNetworkRequest>#include <QtNetwork/QNetworkReply> #include <LogHandler.h>#include <NetworkAccessManager.h>#include <NodeList.h>#include <Node.h>#include <OctreeConstants.h>#include <udt/PacketHeaders.h>#include <SharedUtil.h>#include <StDev.h>#include <UUID.h> #include "AudioRingBuffer.h"#include "AudioMixerClientData.h"#include "AvatarAudioStream.h"#include "InjectedAudioStream.h" #include "AudioMixer.h" const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f;const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f;const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f;const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer";const QString AUDIO_ENV_GROUP_KEY = "audio_env";const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer"; InboundAudioStream::Settings AudioMixer::_streamSettings; bool AudioMixer::_enableFilter = true; bool AudioMixer::shouldMute(float quietestFrame) {    return (quietestFrame > _noiseMutingThreshold);} AudioMixer::AudioMixer(ReceivedMessage& message) :    ThreadedAssignment(message),    _trailingSleepRatio(1.0f),    _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2.0f),    _performanceThrottlingRatio(0.0f),    _attenuationPerDoublingInDistance(DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE),    _noiseMutingThreshold(DEFAULT_NOISE_MUTING_THRESHOLD){    auto nodeList = DependencyManager::get<NodeList>();    auto& packetReceiver = nodeList->getPacketReceiver();     packetReceiver.registerListenerForTypes({ PacketType::MicrophoneAudioNoEcho, PacketType::MicrophoneAudioWithEcho,                                              PacketType::InjectAudio, PacketType::SilentAudioFrame,                                              PacketType::AudioStreamStats },                                            this, "handleNodeAudioPacket");    packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");     connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled);} const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; float AudioMixer::gainForSource(const PositionalAudioStream& streamToAdd,                                const AvatarAudioStream& listeningNodeStream, const glm::vec3& relativePosition, bool isEcho) {    float gain = 1.0f;     float distanceBetween = glm::length(relativePosition);     if (distanceBetween < EPSILON) {        distanceBetween = EPSILON;    }     if (streamToAdd.getType() == PositionalAudioStream::Injector) {        gain *= reinterpret_cast<const InjectedAudioStream*>(&streamToAdd)->getAttenuationRatio();    }     if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) {        //  source is another avatar, apply fixed off-axis attenuation to make them quieter as they turn away from listener        glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition;         float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f),                                           glm::normalize(rotatedListenerPosition));         const float MAX_OFF_AXIS_ATTENUATION = 0.2f;        const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f;         float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION +        (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / PI_OVER_TWO));         // multiply the current attenuation coefficient by the calculated off axis coefficient        gain *= offAxisCoefficient;    }     float attenuationPerDoublingInDistance = _attenuationPerDoublingInDistance;    for (int i = 0; i < _zonesSettings.length(); ++i) {        if (_audioZones[_zonesSettings[i].source].contains(streamToAdd.getPosition()) &&            _audioZones[_zonesSettings[i].listener].contains(listeningNodeStream.getPosition())) {            attenuationPerDoublingInDistance = _zonesSettings[i].coefficient;            break;        }    }     if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) {        // calculate the distance coefficient using the distance to this node        float distanceCoefficient = 1.0f - (logf(distanceBetween / ATTENUATION_BEGINS_AT_DISTANCE) / logf(2.0f)                                            * attenuationPerDoublingInDistance);         if (distanceCoefficient < 0) {            distanceCoefficient = 0;        }         // multiply the current attenuation coefficient by the distance coefficient        gain *= distanceCoefficient;    }     return gain;} float AudioMixer::azimuthForSource(const PositionalAudioStream& streamToAdd, const AvatarAudioStream& listeningNodeStream,                                   const glm::vec3& relativePosition) {    glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation());     //  Compute sample delay for the two ears to create phase panning    glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;     // project the rotated source position vector onto the XZ plane    rotatedSourcePosition.y = 0.0f;     static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;     if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) {        // produce an oriented angle about the y-axis        return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f));    } else {        // there is no distance between listener and source - return no azimuth        return 0;    }} void AudioMixer::addStreamToMixForListeningNodeWithStream(AudioMixerClientData& listenerNodeData,                                                          const PositionalAudioStream& streamToAdd,                                                          const QUuid& sourceNodeID,                                                          const AvatarAudioStream& listeningNodeStream) {      // to reduce artifacts we calculate the gain and azimuth for every source for this listener    // even if we are not going to end up mixing in this source     ++_totalMixes;     // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct     // check if this is a server echo of a source back to itself    bool isEcho = (&streamToAdd == &listeningNodeStream);     // figure out the gain for this source at the listener    glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition();    float gain = gainForSource(streamToAdd, listeningNodeStream, relativePosition, isEcho);     // figure out the azimuth to this source at the listener    float azimuth = isEcho ? 0.0f : azimuthForSource(streamToAdd, listeningNodeStream, relativePosition);     float repeatedFrameFadeFactor = 1.0f;     static const int HRTF_DATASET_INDEX = 1;     if (!streamToAdd.lastPopSucceeded()) {        bool forceSilentBlock = true;         if (_streamSettings._repetitionWithFade && !streamToAdd.getLastPopOutput().isNull()) {             // reptition with fade is enabled, and we do have a valid previous frame to repeat            // so we mix the previously-mixed block             // this is preferable to not mixing it at all to avoid the harsh jump to silence             // we'll repeat the last block until it has a block to mix            // and we'll gradually fade that repeated block into silence.             // calculate its fade factor, which depends on how many times it's already been repeated.             repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1);            if (repeatedFrameFadeFactor > 0.0f) {                // apply the repeatedFrameFadeFactor to the gain                gain *= repeatedFrameFadeFactor;                 forceSilentBlock = false;            }        }         if (forceSilentBlock) {            // we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled            // in this case we will call renderSilent with a forced silent block            // this ensures the correct tail from the previously mixed block and the correct spatialization of first block            // of any upcoming audio             if (!streamToAdd.isStereo() && !isEcho) {                // get the existing listener-source HRTF object, or create a new one                auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());                 // this is not done for stereo streams since they do not go through the HRTF                static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {};                hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, gain,                                  AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);                 ++_hrtfSilentRenders;;            }             return;        }    }     // grab the stream from the ring buffer    AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput();     if (streamToAdd.isStereo() || isEcho) {        // this is a stereo source or server echo so we do not pass it through the HRTF        // simply apply our calculated gain to each sample        if (streamToAdd.isStereo()) {            for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) {                _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE);            }             ++_manualStereoMixes;        } else {            for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) {                auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE);                _mixedSamples[i] += monoSample;                _mixedSamples[i + 1] += monoSample;            }             ++_manualEchoMixes;        }         return;    }     // get the existing listener-source HRTF object, or create a new one    auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());     static int16_t streamBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL];     streamPopOutput.readSamples(streamBlock, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);     // if the frame we're about to mix is silent, simply call render silent and move on    if (streamToAdd.getLastPopOutputLoudness() == 0.0f) {        // silent frame from source         // we still need to call renderSilent via the HRTF for mono source        hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, gain,                          AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);         ++_hrtfSilentRenders;         return;    }     if (_performanceThrottlingRatio > 0.0f        && streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= _minAudibilityThreshold) {        // the mixer is struggling so we're going to drop off some streams         // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0        hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, 0.0f,                          AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);         ++_hrtfStruggleRenders;         return;    }     ++_hrtfRenders;     // mono stream, call the HRTF with our block and calculated azimuth and gain    hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, gain,                AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);} bool AudioMixer::prepareMixForListeningNode(Node* node) {    AvatarAudioStream* nodeAudioStream = static_cast<AudioMixerClientData*>(node->getLinkedData())->getAvatarAudioStream();    AudioMixerClientData* listenerNodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());     // zero out the client mix for this node    memset(_mixedSamples, 0, sizeof(_mixedSamples));     // loop through all other nodes that have sufficient audio to mix     DependencyManager::get<NodeList>()->eachNode([&](const SharedNodePointer& otherNode){        if (otherNode->getLinkedData()) {            AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData();             // enumerate the ARBs attached to the otherNode and add all that should be added to mix            auto streamsCopy = otherNodeClientData->getAudioStreams();             for (auto& streamPair : streamsCopy) {                 auto otherNodeStream = streamPair.second;                 if (*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) {                    addStreamToMixForListeningNodeWithStream(*listenerNodeData, *otherNodeStream, otherNode->getUUID(),                                                             *nodeAudioStream);                }            }        }    });     // use the per listner AudioLimiter to render the mixed data...    listenerNodeData->audioLimiter.render(_mixedSamples, _clampedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);     // check for silent audio after the peak limitor has converted the samples    bool hasAudio = false;    for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) {        if (_clampedSamples[i] != 0) {            hasAudio = true;            break;        }    }    return hasAudio;} void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) {    // Send stream properties    bool hasReverb = false;    float reverbTime, wetLevel;    // find reverb properties    for (int i = 0; i < _zoneReverbSettings.size(); ++i) {        AudioMixerClientData* data = static_cast<AudioMixerClientData*>(node->getLinkedData());        glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition();        AABox box = _audioZones[_zoneReverbSettings[i].zone];        if (box.contains(streamPosition)) {            hasReverb = true;            reverbTime = _zoneReverbSettings[i].reverbTime;            wetLevel = _zoneReverbSettings[i].wetLevel;             break;        }    }     AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());    AvatarAudioStream* stream = nodeData->getAvatarAudioStream();    bool dataChanged = (stream->hasReverb() != hasReverb) ||    (stream->hasReverb() && (stream->getRevebTime() != reverbTime ||                             stream->getWetLevel() != wetLevel));    if (dataChanged) {        // Update stream        if (hasReverb) {            stream->setReverb(reverbTime, wetLevel);        } else {            stream->clearReverb();        }    }     // Send at change or every so often    float CHANCE_OF_SEND = 0.01f;    bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND);     if (sendData) {        auto nodeList = DependencyManager::get<NodeList>();         unsigned char bitset = 0;         int packetSize = sizeof(bitset);         if (hasReverb) {            packetSize += sizeof(reverbTime) + sizeof(wetLevel);        }         auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize);         if (hasReverb) {            setAtBit(bitset, HAS_REVERB_BIT);        }         envPacket->writePrimitive(bitset);         if (hasReverb) {            envPacket->writePrimitive(reverbTime);            envPacket->writePrimitive(wetLevel);        }        nodeList->sendPacket(std::move(envPacket), *node);    }} void AudioMixer::handleNodeAudioPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {    DependencyManager::get<NodeList>()->updateNodeWithDataFromPacket(message, sendingNode);} void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {    auto nodeList = DependencyManager::get<NodeList>();     if (sendingNode->isAllowedEditor()) {        glm::vec3 position;        float radius;         auto newPacket = NLPacket::create(PacketType::MuteEnvironment, sizeof(position) + sizeof(radius));         // read the position and radius from the sent packet        message->readPrimitive(&position);        message->readPrimitive(&radius);         // write them to our packet        newPacket->writePrimitive(position);        newPacket->writePrimitive(radius);         nodeList->eachNode([&](const SharedNodePointer& node){            if (node->getType() == NodeType::Agent && node->getActiveSocket() &&                node->getLinkedData() && node != sendingNode) {                nodeList->sendUnreliablePacket(*newPacket, *node);            }        });    }} void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {    // enumerate the connected listeners to remove HRTF objects for the disconnected node    auto nodeList = DependencyManager::get<NodeList>();     nodeList->eachNode([](const SharedNodePointer& node) {        auto clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());        if (clientData) {            clientData->removeHRTFsForNode(node->getUUID());        }    });} void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) {    auto injectorClientData = qobject_cast<AudioMixerClientData*>(sender());    if (injectorClientData) {        // enumerate the connected listeners to remove HRTF objects for the disconnected injector        auto nodeList = DependencyManager::get<NodeList>();         nodeList->eachNode([injectorClientData, &streamID](const SharedNodePointer& node){            auto listenerClientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());            if (listenerClientData) {                listenerClientData->removeHRTFForStream(injectorClientData->getNodeID(), streamID);            }        });    }} QString AudioMixer::percentageForMixStats(int counter) {    if (_totalMixes > 0) {        float mixPercentage = (float(counter) / _totalMixes) * 100.0f;        return QString::number(mixPercentage, 'f', 2);    } else {        return QString("0.0");    }} void AudioMixer::sendStatsPacket() {    static QJsonObject statsObject;     statsObject["useDynamicJitterBuffers"] = _streamSettings._dynamicJitterBuffers;    statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f;    statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio;     statsObject["avg_listeners_per_frame"] = (float) _sumListeners / (float) _numStatFrames;     QJsonObject mixStats;    mixStats["%_hrtf_mixes"] = percentageForMixStats(_hrtfRenders);    mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_hrtfSilentRenders);    mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_hrtfStruggleRenders);    mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_manualStereoMixes);    mixStats["%_manual_echo_mixes"] = percentageForMixStats(_manualEchoMixes); 

Lines 1–500 of 934. Load more