////  AudioMixerClientData.cpp//  assignment-client/src/audio////  Created by Stephen Birarda on 10/18/13.//  Copyright 2013 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include "AudioMixerClientData.h" #include <random> #include <glm/common.hpp> #include <QtCore/QDebug>#include <QtCore/QJsonArray> #include <udt/PacketHeaders.h>#include <UUID.h> #include "InjectedAudioStream.h" #include "AudioLogging.h"#include "AudioHelpers.h"#include "AudioMixer.h" AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID, Node::LocalID nodeLocalID) :    NodeData(nodeID, nodeLocalID),    audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO),    _outgoingMixedAudioSequenceNumber(0),    _downstreamAudioStreamStats(){    // of the ~94 blocks in a second of audio sent from the AudioMixer, pick a random one to send out a stats packet on    // this ensures we send out stats to this client around every second    // but do not send all of the stats packets out at the same time    std::random_device randomDevice;    std::mt19937 numberGenerator { randomDevice() };    std::uniform_int_distribution<> distribution { 1, (int) ceil(1.0f / AudioConstants::NETWORK_FRAME_SECS) };     _frameToSendStats = distribution(numberGenerator);} AudioMixerClientData::~AudioMixerClientData() {    if (_codec) {        _codec->releaseDecoder(_decoder);        _codec->releaseEncoder(_encoder);    }} void AudioMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {    if (!_packetQueue.node) {        _packetQueue.node = node;    }    _packetQueue.push(message);} int AudioMixerClientData::processPackets(ConcurrentAddedStreams& addedStreams) {    SharedNodePointer node = _packetQueue.node;    assert(_packetQueue.empty() || node);    _packetQueue.node.clear();     while (!_packetQueue.empty()) {        auto& packet = _packetQueue.front();         switch (packet->getType()) {            case PacketType::MicrophoneAudioNoEcho:            case PacketType::MicrophoneAudioWithEcho:            case PacketType::InjectAudio:            case PacketType::SilentAudioFrame: {                if (node->isUpstream()) {                    setupCodecForReplicatedAgent(packet);                }                 processStreamPacket(*packet, addedStreams);                 optionallyReplicatePacket(*packet, *node);                break;            }            case PacketType::AudioStreamStats: {                parseData(*packet);                break;            }            case PacketType::NegotiateAudioFormat:                negotiateAudioFormat(*packet, node);                break;            case PacketType::RequestsDomainListData:                parseRequestsDomainListData(*packet);                break;            case PacketType::PerAvatarGainSet:                parsePerAvatarGainSet(*packet, node);                break;            case PacketType::InjectorGainSet:                parseInjectorGainSet(*packet, node);                break;            case PacketType::NodeIgnoreRequest:                parseNodeIgnoreRequest(packet, node);                break;            case PacketType::RadiusIgnoreRequest:                parseRadiusIgnoreRequest(packet, node);                break;            case PacketType::AudioSoloRequest:                parseSoloRequest(packet, node);                break;            case PacketType::StopInjector:                parseStopInjectorPacket(packet);                break;            default:                Q_UNREACHABLE();        }         _packetQueue.pop();    }    assert(_packetQueue.empty());     // now that we have processed all packets for this frame    // we can prepare the sources from this client to be ready for mixing    return checkBuffersBeforeFrameSend();} bool isReplicatedPacket(PacketType packetType) {    return packetType == PacketType::ReplicatedMicrophoneAudioNoEcho        || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho        || packetType == PacketType::ReplicatedInjectAudio        || packetType == PacketType::ReplicatedSilentAudioFrame;} void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) {     // first, make sure that this is a packet from a node we are supposed to replicate    if (node.isReplicated()) {         // now make sure it's a packet type that we want to replicate         // first check if it is an original type that we should replicate        PacketType mirroredType = PacketTypeEnum::getReplicatedPacketMapping().value(message.getType());         if (mirroredType == PacketType::Unknown) {            // if it wasn't check if it is a replicated type that we should re-replicate            if (PacketTypeEnum::getReplicatedPacketMapping().key(message.getType()) != PacketType::Unknown) {                mirroredType = message.getType();            } else {                qCDebug(audio) << "Packet passed to optionallyReplicatePacket was not a replicatable type - returning";                return;            }        }         std::unique_ptr<NLPacket> packet;        auto nodeList = DependencyManager::get<NodeList>();         // enumerate the downstream audio mixers and send them the replicated version of this packet        nodeList->unsafeEachNode([&](const SharedNodePointer& downstreamNode) {            if (AudioMixer::shouldReplicateTo(node, *downstreamNode)) {                // construct the packet only once, if we have any downstream audio mixers to send to                if (!packet) {                    // construct an NLPacket to send to the replicant that has the contents of the received packet                    packet = NLPacket::create(mirroredType);                     if (!isReplicatedPacket(message.getType())) {                        // since this packet will be non-sourced, we add the replicated node's ID here                        packet->write(node.getUUID().toRfc4122());                    }                     packet->write(message.getMessage());                }                                nodeList->sendUnreliablePacket(*packet, *downstreamNode);            }        });    } } void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) {    quint8 numberOfCodecs;    message.readPrimitive(&numberOfCodecs);    std::vector<QString> codecs;    for (auto i = 0; i < numberOfCodecs; i++) {        codecs.push_back(message.readString());    }    const std::pair<QString, CodecPluginPointer> codec = AudioMixer::negotiateCodec(codecs);     setupCodec(codec.second, codec.first);    sendSelectAudioFormat(node, codec.first);} void AudioMixerClientData::parseRequestsDomainListData(ReceivedMessage& message) {    bool isRequesting;    message.readPrimitive(&isRequesting);    setRequestsDomainListData(isRequesting);} void AudioMixerClientData::parsePerAvatarGainSet(ReceivedMessage& message, const SharedNodePointer& node) {    QUuid uuid = node->getUUID();    // parse the UUID from the packet    QUuid avatarUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));    uint8_t packedGain;    message.readPrimitive(&packedGain);    float gain = unpackFloatGainFromByte(packedGain);     if (avatarUUID.isNull()) {        // set the PRIMARY avatar gain        setPrimaryAvatarGain(gain);        qCDebug(audio) << "Setting PRIMARY avatar gain for" << uuid << "to" << gain;    } else {        // set the per-source avatar gain        setGainForAvatar(avatarUUID, gain);        qCDebug(audio) << "Setting avatar gain adjustment for hrtf[" << uuid << "][" << avatarUUID << "] to" << gain;    }} void AudioMixerClientData::parseInjectorGainSet(ReceivedMessage& message, const SharedNodePointer& node) {    QUuid uuid = node->getUUID();     uint8_t packedGain;    message.readPrimitive(&packedGain);    float gain = unpackFloatGainFromByte(packedGain);     setPrimaryInjectorGain(gain);    qCDebug(audio) << "Setting PRIMARY injector gain for" << uuid << "to" << gain;} bool setGainInStreams(const QUuid &nodeID, float gain, std::vector<AudioMixerClientData::MixableStream> &streamVector) {    auto itActive = std::find_if(streamVector.cbegin(), streamVector.cend(),                                 [nodeID](const AudioMixerClientData::MixableStream& mixableStream){        return mixableStream.nodeStreamID.nodeID == nodeID && mixableStream.nodeStreamID.streamID.isNull();    });     if (itActive != streamVector.cend()) {        itActive->hrtf->setGainAdjustment(gain);        return true;    } else {        return false;    }} void AudioMixerClientData::setGainForAvatar(QUuid nodeID, float gain) {    if (!setGainInStreams(nodeID, gain, _streams.active)) {        setGainInStreams(nodeID, gain, _streams.inactive);    }} void AudioMixerClientData::parseNodeIgnoreRequest(QSharedPointer<ReceivedMessage> message, const SharedNodePointer& node) {    auto ignoredNodesPair = node->parseIgnoreRequestMessage(message);     // we have a vector of ignored or unignored node UUIDs - update our internal data structures so that    // streams can be included or excluded next time a mix is being created    if (ignoredNodesPair.second) {        // we have newly ignored nodes, add them to our vector        _newIgnoredNodeIDs.insert(std::end(_newIgnoredNodeIDs),                                  std::begin(ignoredNodesPair.first), std::end(ignoredNodesPair.first));    } else {        // we have newly unignored nodes, add them to our vector        _newUnignoredNodeIDs.insert(std::end(_newUnignoredNodeIDs),                                    std::begin(ignoredNodesPair.first), std::end(ignoredNodesPair.first));    }     auto nodeList = DependencyManager::get<NodeList>();    for (auto& nodeID : ignoredNodesPair.first) {        auto otherNode = nodeList->nodeWithUUID(nodeID);        if (otherNode) {            auto otherNodeMixerClientData = static_cast<AudioMixerClientData*>(otherNode->getLinkedData());            if (otherNodeMixerClientData) {                if (ignoredNodesPair.second) {                    otherNodeMixerClientData->ignoredByNode(getNodeID());                } else {                    otherNodeMixerClientData->unignoredByNode(getNodeID());                }            }        }    }} void AudioMixerClientData::ignoredByNode(QUuid nodeID) {    // first add this ID to the concurrent vector for newly ignoring nodes    _newIgnoringNodeIDs.push_back(nodeID);     // now take a lock and on the consistent vector of ignoring nodes and make sure this node is in it    std::lock_guard<std::mutex> lock(_ignoringNodeIDsMutex);    if (std::find(_ignoringNodeIDs.begin(), _ignoringNodeIDs.end(), nodeID) == _ignoringNodeIDs.end()) {        _ignoringNodeIDs.push_back(nodeID);    }} void AudioMixerClientData::unignoredByNode(QUuid nodeID) {    // first add this ID to the concurrent vector for newly unignoring nodes    _newUnignoringNodeIDs.push_back(nodeID);     // now take a lock on the consistent vector of ignoring nodes and make sure this node isn't in it    std::lock_guard<std::mutex> lock(_ignoringNodeIDsMutex);    auto it = _ignoringNodeIDs.begin();    while (it != _ignoringNodeIDs.end()) {        if (*it == nodeID) {            it = _ignoringNodeIDs.erase(it);        } else {            ++it;        }    }} void AudioMixerClientData::clearStagedIgnoreChanges() {    _newIgnoredNodeIDs.clear();    _newUnignoredNodeIDs.clear();    _newIgnoringNodeIDs.clear();    _newUnignoringNodeIDs.clear();} void AudioMixerClientData::parseRadiusIgnoreRequest(QSharedPointer<ReceivedMessage> message, const SharedNodePointer& node) {    bool enabled;    message->readPrimitive(&enabled);     _isIgnoreRadiusEnabled = enabled;     auto avatarAudioStream = getAvatarAudioStream();     // if we have an avatar audio stream, tell it wether its ignore box should be enabled or disabled    if (avatarAudioStream) {        if (_isIgnoreRadiusEnabled) {            avatarAudioStream->enableIgnoreBox();        } else {            avatarAudioStream->disableIgnoreBox();        }    }}  void AudioMixerClientData::parseSoloRequest(QSharedPointer<ReceivedMessage> message, const SharedNodePointer& node) {     uint8_t addToSolo;    message->readPrimitive(&addToSolo);     while (message->getBytesLeftToRead()) {        // parse out the UUID being soloed from the packet        QUuid soloedUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID));         if (addToSolo) {            _soloedNodes.push_back(soloedUUID);        } else {            auto it = std::remove(std::begin(_soloedNodes), std::end(_soloedNodes), soloedUUID);            _soloedNodes.erase(it, std::end(_soloedNodes));        }    }} AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {    auto it = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){        return stream->getStreamIdentifier().isNull();    });     if (it != _audioStreams.end()) {        return dynamic_cast<AvatarAudioStream*>(it->get());    }     // no mic stream found - return NULL    return NULL;} void AudioMixerClientData::removeAgentAvatarAudioStream() {    auto it = std::remove_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){        return stream->getStreamIdentifier().isNull();    });     if (it != _audioStreams.end()) {        _audioStreams.erase(it);         // Clear mixing structures so that they get recreated with up to date        // data if the stream comes back        setHasReceivedFirstMix(false);        _streams.skipped.clear();        _streams.inactive.clear();        _streams.active.clear();    }} int AudioMixerClientData::parseData(ReceivedMessage& message) {    PacketType packetType = message.getType();     if (packetType == PacketType::AudioStreamStats) {        // skip over header, appendFlag, and num stats packed        message.seek(sizeof(quint8) + sizeof(quint16));         if (message.getBytesLeftToRead() != sizeof(AudioStreamStats)) {            qWarning() << "Received AudioStreamStats of wrong size" << message.getBytesLeftToRead()                << "instead of" << sizeof(AudioStreamStats) << "from"                << message.getSourceID() << "at" << message.getSenderSockAddr();                        return message.getPosition();        }         // read the downstream audio stream stats        message.readPrimitive(&_downstreamAudioStreamStats);         return message.getPosition();    }     return 0;} bool AudioMixerClientData::containsValidPosition(ReceivedMessage& message) const {    static const int SEQUENCE_NUMBER_BYTES = sizeof(quint16);     auto posBefore = message.getPosition();     message.seek(SEQUENCE_NUMBER_BYTES);     // skip over the codec string    message.readString();     switch (message.getType()) {        case PacketType::MicrophoneAudioNoEcho:        case PacketType::MicrophoneAudioWithEcho: {            // skip over the stereo flag            message.seek(message.getPosition() + sizeof(ChannelFlag));            break;        }        case PacketType::SilentAudioFrame: {            // skip the number of silent samples            message.seek(message.getPosition() + sizeof(SilentSamplesBytes));            break;        }        case PacketType::InjectAudio: {            // skip the stream ID, stereo flag, and loopback flag            message.seek(message.getPosition() + NUM_STREAM_ID_BYTES + sizeof(ChannelFlag) + sizeof(LoopbackFlag));            break;        }        default:            Q_UNREACHABLE();            break;    }     glm::vec3 peekPosition;    message.readPrimitive(&peekPosition);     // reset the position the message was at before we were called    message.seek(posBefore);     if (glm::any(glm::isnan(peekPosition))) {        return false;    }     return true;} void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) {     if (!containsValidPosition(message)) {        qDebug() << "Refusing to process audio stream from" << message.getSourceID() << "with invalid position";        return;    }     SharedStreamPointer matchingStream;     auto packetType = message.getType();    bool newStream = false;     if (packetType == PacketType::MicrophoneAudioWithEcho        || packetType == PacketType::MicrophoneAudioNoEcho        || packetType == PacketType::SilentAudioFrame) {         auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){            return stream->getStreamIdentifier().isNull();        });         if (micStreamIt == _audioStreams.end()) {            // we don't have a mic stream yet, so add it             // hop past the sequence number that leads the packet            message.seek(sizeof(StreamSequenceNumber));             // pull the codec string from the packet            auto codecString = message.readString();             // determine if the stream is stereo or not            bool isStereo;            if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) {                SilentSamplesBytes numSilentSamples;                message.readPrimitive(&numSilentSamples);                isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;            } else {                ChannelFlag channelFlag;                message.readPrimitive(&channelFlag);                isStereo = channelFlag == 1;            }             auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());            avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);             if (_isIgnoreRadiusEnabled) {                avatarAudioStream->enableIgnoreBox();            } else {                avatarAudioStream->disableIgnoreBox();            }             qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;             connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,                    this, &AudioMixerClientData::handleMismatchAudioFormat); 

Lines 1–500 of 848. Load more