////  AudioMixer.cpp//  assignment-client/src/audio////  Created by Stephen Birarda on 8/22/13.//  Copyright 2013 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include "AudioMixer.h" #include <thread> #include <QtCore/QJsonArray>#include <QtCore/QJsonDocument>#include <QtCore/QJsonObject>#include <QtCore/QJsonValue>#include <shared/QtHelpers.h> #include <LogHandler.h>#include <NetworkAccessManager.h>#include <NodeList.h>#include <Node.h>#include <OctreeConstants.h>#include <plugins/PluginManager.h>#include <plugins/CodecPlugin.h>#include <udt/PacketHeaders.h>#include <SharedUtil.h>#include <StDev.h>#include <UUID.h>#include <CPUDetect.h>#include <ZoneEntityItem.h> #include "AudioLogging.h"#include "AudioHelpers.h"#include "AudioRingBuffer.h"#include "AudioMixerClientData.h"#include "AvatarAudioStream.h"#include "InjectedAudioStream.h"#include "crash-handler/CrashHandler.h"#include "../AssignmentDynamicFactory.h"#include "../entities/AssignmentParentFinder.h" using namespace std; static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f;    // attenuation = -6dB * log2(distance)static const int DISABLE_STATIC_JITTER_FRAMES = -1;static const float DEFAULT_NOISE_MUTING_THRESHOLD = 1.0f;static const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer";static const QString AUDIO_ENV_GROUP_KEY = "audio_env";static const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer";static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading";  int AudioMixer::_numStaticJitterFrames{ DISABLE_STATIC_JITTER_FRAMES };float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };map<QString, shared_ptr<CodecPlugin>> AudioMixer::_availableCodecs{ };QStringList AudioMixer::_codecPreferenceOrder{};unordered_map<QString, AudioMixer::ZoneSettings> AudioMixer::_audioZones; AudioMixer::AudioMixer(ReceivedMessage& message) :    ThreadedAssignment(message){     DependencyManager::registerInheritance<EntityDynamicFactoryInterface, AssignmentDynamicFactory>();    DependencyManager::set<AssignmentDynamicFactory>();     // Always clear settings first    // This prevents previous assignment settings from sticking around    clearDomainSettings();     // hash the available codecs (on the mixer)    _availableCodecs.clear(); // Make sure struct is clean    auto pluginManager = DependencyManager::set<PluginManager>();    // Only load codec plugins; for now assume codec plugins have 'codec' in their name.    auto codecPluginFilter = [](const QJsonObject& metaData) {        QJsonValue nameValue = metaData["MetaData"]["name"];        return nameValue.toString().contains("codec", Qt::CaseInsensitive);    };    pluginManager->setPluginFilter(codecPluginFilter);     const auto& codecPlugins = pluginManager->getCodecPlugins();    for(const auto& codec : codecPlugins) {        _availableCodecs[codec->getName()] = codec;    }     auto nodeList = DependencyManager::get<NodeList>();    auto& packetReceiver = nodeList->getPacketReceiver();     // packets whose consequences are limited to their own node can be parallelized    packetReceiver.registerListenerForTypes({            PacketType::MicrophoneAudioNoEcho,            PacketType::MicrophoneAudioWithEcho,            PacketType::InjectAudio,            PacketType::AudioStreamStats,            PacketType::SilentAudioFrame,            PacketType::NegotiateAudioFormat,            PacketType::MuteEnvironment,            PacketType::NodeIgnoreRequest,            PacketType::RadiusIgnoreRequest,            PacketType::RequestsDomainListData,            PacketType::PerAvatarGainSet,            PacketType::InjectorGainSet,            PacketType::AudioSoloRequest,            PacketType::StopInjector },            PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::queueAudioPacket)    );     // packets whose consequences are global should be processed on the main thread    packetReceiver.registerListener(PacketType::MuteEnvironment,        PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleMuteEnvironmentPacket));    packetReceiver.registerListener(PacketType::NodeMuteRequest,        PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleNodeMuteRequestPacket));    packetReceiver.registerListener(PacketType::KillAvatar,        PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleKillAvatarPacket));    packetReceiver.registerListenerForTypes({ PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase },        PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleOctreePacket));     packetReceiver.registerListenerForTypes({        PacketType::ReplicatedMicrophoneAudioNoEcho,        PacketType::ReplicatedMicrophoneAudioWithEcho,        PacketType::ReplicatedInjectAudio,        PacketType::ReplicatedSilentAudioFrame },        PacketReceiver::makeUnsourcedListenerReference<AudioMixer>(this, &AudioMixer::queueReplicatedAudioPacket)    );     connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled);} void AudioMixer::aboutToFinish() {    DependencyManager::destroy<PluginManager>();} void AudioMixer::queueAudioPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {    if (message->getType() == PacketType::SilentAudioFrame) {        _numSilentPackets++;    }     getOrCreateClientData(node.data())->queuePacket(message, node);} void AudioMixer::queueReplicatedAudioPacket(QSharedPointer<ReceivedMessage> message) {    // make sure we have a replicated node for the original sender of the packet    auto nodeList = DependencyManager::get<NodeList>();     // Node ID is now part of user data, since replicated audio packets are non-sourced.    QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID));     auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent,                                                    message->getSenderSockAddr(), message->getSenderSockAddr(),                                                    Node::NULL_LOCAL_ID, true, true);    replicatedNode->setLastHeardMicrostamp(usecTimestampNow());     // construct a "fake" audio received message from the byte array and packet list information    auto audioData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID);     PacketType rewrittenType = PacketTypeEnum::getReplicatedPacketMapping().key(message->getType());     if (rewrittenType == PacketType::Unknown) {        qCDebug(audio) << "Cannot unwrap replicated packet type not present in REPLICATED_PACKET_WRAPPING";    }     auto replicatedMessage = QSharedPointer<ReceivedMessage>::create(audioData, rewrittenType,                                                                     versionForPacketType(rewrittenType),                                                                     message->getSenderSockAddr(), Node::NULL_LOCAL_ID);     getOrCreateClientData(replicatedNode.data())->queuePacket(replicatedMessage, replicatedNode);} void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {    auto nodeList = DependencyManager::get<NodeList>();     if (sendingNode->getCanKick()) {        glm::vec3 position;        float radius;         auto newPacket = NLPacket::create(PacketType::MuteEnvironment, sizeof(position) + sizeof(radius));         // read the position and radius from the sent packet        message->readPrimitive(&position);        message->readPrimitive(&radius);         // write them to our packet        newPacket->writePrimitive(position);        newPacket->writePrimitive(radius);         nodeList->eachNode([&](const SharedNodePointer& node){            if (node->getType() == NodeType::Agent && node->getActiveSocket() &&                node->getLinkedData() && node != sendingNode) {                nodeList->sendUnreliablePacket(*newPacket, *node);            }        });    }} const pair<QString, CodecPluginPointer> AudioMixer::negotiateCodec(vector<QString> codecs) {    QString selectedCodecName;    CodecPluginPointer selectedCodec;     // read the codecs requested (by the client)    int minPreference = numeric_limits<int>::max();    for (auto& codec : codecs) {        if (_availableCodecs.count(codec) > 0) {            int preference = _codecPreferenceOrder.indexOf(codec);             // choose the preferred, available codec            if (preference >= 0 && preference < minPreference) {                minPreference = preference;                selectedCodecName  = codec;            }        }    }     return make_pair(selectedCodecName, _availableCodecs[selectedCodecName]);}  void AudioMixer::handleNodeMuteRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode) {    auto nodeList = DependencyManager::get<NodeList>();    QUuid nodeUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID));    if (sendingNode->getCanKick()) {        auto node = nodeList->nodeWithUUID(nodeUUID);        if (node) {            // we need to set a flag so we send them the appropriate packet to mute them            AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData();            nodeData->setShouldMuteClient(true);        } else {            qWarning() << "Node mute packet received for unknown node " << uuidStringWithoutCurlyBraces(nodeUUID);        }    } else {        qWarning() << "Node mute packet received from node that cannot mute, ignoring";    }} void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {    auto clientData = dynamic_cast<AudioMixerClientData*>(killedNode->getLinkedData());    if (clientData) {        // stage the removal of all streams from this node, workers handle when preparing mixes for listeners        _workerSharedData.removedNodes.emplace_back(killedNode->getLocalID());    }} void AudioMixer::handleKillAvatarPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode) {    auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());    if (clientData) {        clientData->removeAgentAvatarAudioStream();         // stage a removal of the avatar audio stream from this Agent, workers handle when preparing mixes for listeners        _workerSharedData.removedStreams.emplace_back(sendingNode->getUUID(), sendingNode->getLocalID(), QUuid());    }} void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) {    auto injectorClientData = qobject_cast<AudioMixerClientData*>(sender());     if (injectorClientData) {        // stage the removal of this stream, workers handle when preparing mixes for listeners        _workerSharedData.removedStreams.emplace_back(injectorClientData->getNodeID(),                                                      injectorClientData->getNodeLocalID(),                                                      streamID);    }} QString AudioMixer::percentageForMixStats(int counter) {    if (_stats.totalMixes > 0) {        float mixPercentage = (float(counter) / _stats.totalMixes) * 100.0f;        return QString::number(mixPercentage, 'f', 2);    } else {        return QString("0.0");    }} void AudioMixer::sendStatsPacket() {    QJsonObject statsObject;     if (_numStatFrames == 0) {        return;    } #ifdef DEBUG_EVENT_QUEUE    QJsonObject qtStats;     _workerPool.queueStats(qtStats);    statsObject["audio_thread_event_queue"] = qtStats;#endif     // general stats    statsObject["useDynamicJitterBuffers"] = _numStaticJitterFrames == DISABLE_STATIC_JITTER_FRAMES;     statsObject["threads"] = _workerPool.numThreads();     statsObject["trailing_mix_ratio"] = _trailingMixRatio;    statsObject["throttling_ratio"] = _throttlingRatio;     statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames;    statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames;    statsObject["avg_listeners_(silent)_per_frame"] = (float)_stats.sumListenersSilent / (float)_numStatFrames;     statsObject["silent_packets_per_frame"] = (float)_numSilentPackets / (float)_numStatFrames;     // timing stats    QJsonObject timingStats;     auto addTiming = [&](Timer& timer, string name) {        uint64_t timing, trailing;        timer.get(timing, trailing);        timingStats[("us_per_" + name).c_str()] = (qint64)(timing / _numStatFrames);        timingStats[("us_per_" + name + "_trailing").c_str()] = (qint64)(trailing / _numStatFrames);    };     addTiming(_ticTiming, "tic");    addTiming(_checkTimeTiming, "check_time");    addTiming(_sleepTiming, "sleep");    addTiming(_frameTiming, "frame");    addTiming(_packetsTiming, "packets");    addTiming(_mixTiming, "mix");    addTiming(_eventsTiming, "events"); #ifdef HIFI_AUDIO_MIXER_DEBUG    timingStats["ns_per_mix"] = (_stats.totalMixes > 0) ?  (float)(_stats.mixTime / _stats.totalMixes) : 0;#endif     // call it "avg_..." to keep it higher in the display, sorted alphabetically    statsObject["avg_timing_stats"] = timingStats;     // mix stats    QJsonObject mixStats;     mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders);    mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes);    mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes);     mixStats["1_hrtf_renders"] = (int)(_stats.hrtfRenders / (float)_numStatFrames);    mixStats["1_hrtf_resets"] = (int)(_stats.hrtfResets / (float)_numStatFrames);    mixStats["1_hrtf_updates"] = (int)(_stats.hrtfUpdates / (float)_numStatFrames);     mixStats["2_skipped_streams"] = (int)(_stats.skipped / (float)_numStatFrames);    mixStats["2_inactive_streams"] = (int)(_stats.inactive / (float)_numStatFrames);    mixStats["2_active_streams"] = (int)(_stats.active / (float)_numStatFrames);     mixStats["3_skippped_to_active"] = (int)(_stats.skippedToActive / (float)_numStatFrames);    mixStats["3_skippped_to_inactive"] = (int)(_stats.skippedToInactive / (float)_numStatFrames);    mixStats["3_inactive_to_skippped"] = (int)(_stats.inactiveToSkipped / (float)_numStatFrames);    mixStats["3_inactive_to_active"] = (int)(_stats.inactiveToActive / (float)_numStatFrames);    mixStats["3_active_to_skippped"] = (int)(_stats.activeToSkipped / (float)_numStatFrames);    mixStats["3_active_to_inactive"] = (int)(_stats.activeToInactive / (float)_numStatFrames);     mixStats["total_mixes"] = _stats.totalMixes;    mixStats["avg_mixes_per_block"] = _stats.totalMixes / _numStatFrames;     statsObject["mix_stats"] = mixStats;     _numStatFrames = _numSilentPackets = 0;    _stats.reset();     // add stats for each listerner    auto nodeList = DependencyManager::get<NodeList>();    QJsonObject listenerStats;     nodeList->eachNode([&](const SharedNodePointer& node) {        AudioMixerClientData* clientData = static_cast<AudioMixerClientData*>(node->getLinkedData());        if (clientData) {            QJsonObject nodeStats;            QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID());             nodeStats["outbound_kbps"] = node->getOutboundKbps();            nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidString;             nodeStats["jitter"] = clientData->getAudioStreamStats();             listenerStats[uuidString] = nodeStats;        }    });     // add the listeners object to the root object    statsObject["z_listeners"] = listenerStats;     // send off the stats packets    ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject);} void AudioMixer::run() {     qCDebug(audio) << "Waiting for connection to domain to request settings from domain-server.";     // wait until we have the domain-server settings, otherwise we bail    DomainHandler& domainHandler = DependencyManager::get<NodeList>()->getDomainHandler();    connect(&domainHandler, &DomainHandler::settingsReceived, this, &AudioMixer::start);    connect(&domainHandler, &DomainHandler::settingsReceiveFail, this, &AudioMixer::domainSettingsRequestFailed);     ThreadedAssignment::commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer);} AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) {    auto clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());     if (!clientData) {        node->setLinkedData(unique_ptr<NodeData> { new AudioMixerClientData(node->getUUID(), node->getLocalID()) });        clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());        connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector);    }     return clientData;} void AudioMixer::start() {    auto nodeList = DependencyManager::get<NodeList>();     // prepare the NodeList    nodeList->addSetOfNodeTypesToNodeInterestSet({        NodeType::Agent, NodeType::EntityScriptServer, NodeType::EntityServer,        NodeType::UpstreamAudioMixer, NodeType::DownstreamAudioMixer    });    nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); };     // parse out any AudioMixer settings    {        DomainHandler& domainHandler = nodeList->getDomainHandler();        const QJsonObject& settingsObject = domainHandler.getSettingsObject();        parseSettingsObject(settingsObject);    }     setupEntityQuery();     // mix state    unsigned int frame = 1;     while (!_isFinished) {        auto ticTimer = _ticTiming.timer();         // Set our query each frame        {            _entityViewer.queryOctree();        }         if (_startFrameTimestamp.time_since_epoch().count() == 0) {            _startFrameTimestamp = _idealFrameTimestamp = p_high_resolution_clock::now();        } else {            auto timer = _checkTimeTiming.timer();            auto frameDuration = timeFrame();            throttle(frameDuration, frame);        }         auto frameTimer = _frameTiming.timer();         // process (node-isolated) audio packets across worker threads        {            auto packetsTimer = _packetsTiming.timer();             // first clear the concurrent vector of added streams that the workers will add to when they process packets            _workerSharedData.addedStreams.clear();             nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {                _workerPool.processPackets(cbegin, cend);            });        }         // process queued events (networking, global audio packets, &c.)        {            auto eventsTimer = _eventsTiming.timer();             // clear removed nodes and removed streams before we process events that will setup the new set            _workerSharedData.removedNodes.clear();            _workerSharedData.removedStreams.clear();             // since we're a while loop we need to yield to qt's event processing            QCoreApplication::processEvents();        }         int numToRetain = -1;        assert(_throttlingRatio >= 0.0f && _throttlingRatio <= 1.0f);        if (_throttlingRatio > EPSILON) {            numToRetain = nodeList->size() * (1.0f - _throttlingRatio);        }        nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {            // mix across worker threads            auto mixTimer = _mixTiming.timer();            _workerPool.mix(cbegin, cend, frame, numToRetain);        });         // gather stats        _workerPool.each([&](AudioMixerWorker& worker) {            _stats.accumulate(worker.stats);            worker.stats.reset();        });         ++frame;        ++_numStatFrames;          if (_isFinished) {            // alert qt eventing that this is finished            QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete);            break;        }    }}

Lines 1–500 of 942. Load more