//// Agent.cpp// assignment-client/src//// Created by Stephen Birarda on 7/1/13.// Copyright 2013 High Fidelity, Inc.// Copyright 2023 Overte e.V.//// Distributed under the Apache License, Version 2.0.// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// SPDX-License-Identifier: Apache-2.0// #include "Agent.h" #include <QtCore/QCoreApplication>#include <QtCore/QEventLoop>#include <QtCore/QStandardPaths>#include <QtNetwork/QNetworkDiskCache>#include <QtNetwork/QNetworkRequest>#include <QtNetwork/QNetworkReply>#include <QThread> #include <AnimationCacheScriptingInterface.h>#include <AssetClient.h>#include <AvatarHashMap.h>#include <AudioInjectorManager.h>#include <AssetClient.h>#include <DebugDraw.h>#include <EntityScriptingInterface.h>#include <LocationScriptingInterface.h>#include <AudioScriptingInterface.h>#include <MessagesClient.h>#include <NetworkAccessManager.h>#include <NodeList.h>#include <udt/PacketHeaders.h>#include <ResourceCache.h>#include <ResourceScriptingInterface.h>#include <ScriptCache.h>#include <ScriptEngine.h>#include <ScriptEngines.h>#include <ScriptManager.h>#include <SoundCacheScriptingInterface.h>#include <SoundCache.h>#include <UserActivityLoggerScriptingInterface.h>#include <UsersScriptingInterface.h>#include <UUID.h> #include <recording/ClipCache.h>#include <recording/Deck.h>#include <recording/Recorder.h>#include <recording/Frame.h> #include <plugins/CodecPlugin.h>#include <plugins/PluginManager.h> #include <WebSocketServerClass.h>#include <EntityScriptingInterface.h> // TODO: consider moving to scriptengine.h #include <hfm/ModelFormatRegistry.h> #include "entities/AssignmentParentFinder.h"#include "AssignmentDynamicFactory.h"#include <recording/RecordingScriptingInterface.h>#include "AbstractAudioInterface.h"#include "AgentScriptingInterface.h" static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 10; Agent::Agent(ReceivedMessage& message) : ThreadedAssignment(message), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), _audioGate(AudioConstants::SAMPLE_RATE, AudioConstants::MONO), _avatarAudioTimer(this){ DependencyManager::set<ScriptableAvatar>(); DependencyManager::registerInheritance<EntityDynamicFactoryInterface, AssignmentDynamicFactory>(); DependencyManager::set<AssignmentDynamicFactory>(); DependencyManager::set<AnimationCache>(); DependencyManager::set<AnimationCacheScriptingInterface>(); DependencyManager::set<EntityScriptingInterface>(false); DependencyManager::set<ResourceScriptingInterface>(); DependencyManager::set<UserActivityLoggerScriptingInterface>(); _entityEditSender.setPacketsPerSecond(DEFAULT_ENTITY_PPS_PER_SCRIPT); DependencyManager::get<EntityScriptingInterface>()->setPacketSender(&_entityEditSender); DependencyManager::set<ResourceManager>(); DependencyManager::set<PluginManager>()->instantiate(); DependencyManager::registerInheritance<SpatialParentFinder, AssignmentParentFinder>(); DependencyManager::set<ResourceCacheSharedItems>(); DependencyManager::set<SoundCache>(); DependencyManager::set<SoundCacheScriptingInterface>(); DependencyManager::set<AudioScriptingInterface>(); DependencyManager::set<AudioInjectorManager>(); DependencyManager::set<recording::Deck>(); DependencyManager::set<recording::Recorder>(); DependencyManager::set<recording::ClipCache>(); DependencyManager::set<RecordingScriptingInterface>(); DependencyManager::set<UsersScriptingInterface>(); DependencyManager::set<ModelFormatRegistry>(); DependencyManager::set<ModelCache>(); // Needed to ensure the creation of the DebugDraw instance on the main thread DebugDraw::getInstance(); auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver(); packetReceiver.registerListenerForTypes( { PacketType::MixedAudio, PacketType::SilentAudioFrame }, PacketReceiver::makeUnsourcedListenerReference<Agent>(this, &Agent::handleAudioPacket)); packetReceiver.registerListenerForTypes( { PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase }, PacketReceiver::makeSourcedListenerReference<Agent>(this, &Agent::handleOctreePacket)); packetReceiver.registerListener(PacketType::SelectedAudioFormat, PacketReceiver::makeUnsourcedListenerReference<Agent>(this, &Agent::handleSelectedAudioFormat)); // 100Hz timer for audio const int TARGET_INTERVAL_MSEC = 10; // 10ms connect(&_avatarAudioTimer, &QTimer::timeout, this, &Agent::processAgentAvatarAudio); _avatarAudioTimer.setSingleShot(false); _avatarAudioTimer.setInterval(TARGET_INTERVAL_MSEC); _avatarAudioTimer.setTimerType(Qt::PreciseTimer);} void Agent::playAvatarSound(SharedSoundPointer sound) { // this must happen on Agent's main thread if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "playAvatarSound", Q_ARG(SharedSoundPointer, sound)); return; } else { // TODO: seems to add occasional artifact in tests. I believe it is // correct to do this, but need to figure out for sure, so commenting this // out until I verify. // _numAvatarSoundSentBytes = 0; setAvatarSound(sound); }} void Agent::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) { auto packetType = message->getType(); if (packetType == PacketType::OctreeStats) { int statsMessageLength = OctreeHeadlessViewer::parseOctreeStats(message, senderNode); if (message->getSize() > statsMessageLength) { // pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it int piggyBackedSizeWithHeader = message->getSize() - statsMessageLength; auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]); memcpy(buffer.get(), message->getRawMessage() + statsMessageLength, piggyBackedSizeWithHeader); auto newPacket = NLPacket::fromReceivedPacket(std::move(buffer), piggyBackedSizeWithHeader, message->getSenderSockAddr()); message = QSharedPointer<ReceivedMessage>::create(*newPacket); } else { return; // bail since no piggyback data } packetType = message->getType(); } // fall through to piggyback message if (packetType == PacketType::EntityData) { _entityViewer.processDatagram(*message, senderNode); } else if (packetType == PacketType::EntityErase) { _entityViewer.processEraseMessage(*message, senderNode); }} void Agent::handleAudioPacket(QSharedPointer<ReceivedMessage> message) { _receivedAudioStream.parseData(*message); _lastReceivedAudioLoudness = _receivedAudioStream.getNextOutputFrameLoudness(); _receivedAudioStream.clearBuffer();} static const QString AGENT_LOGGING_NAME = "agent"; void Agent::run() { // Create ScriptEngines on threaded-assignment thread then move to main thread. DependencyManager::set<ScriptEngines>(ScriptManager::AGENT_SCRIPT)->moveToThread(qApp->thread()); DependencyManager::set<ScriptCache>(); // make sure we request our script once the agent connects to the domain auto nodeList = DependencyManager::get<NodeList>(); connect(&nodeList->getDomainHandler(), &DomainHandler::connectedToDomain, this, &Agent::requestScript); ThreadedAssignment::commonInit(AGENT_LOGGING_NAME, NodeType::Agent); // Setup MessagesClient auto messagesClient = DependencyManager::set<MessagesClient>(); messagesClient->startThread(); // make sure we hear about connected nodes so we can grab an ATP script if a request is pending connect(nodeList.data(), &LimitedNodeList::nodeActivated, this, &Agent::nodeActivated); // make sure we hear about dissappearing nodes so we can clear the entity tree if an entity server goes away connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &Agent::nodeKilled); nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::AudioMixer, NodeType::AvatarMixer, NodeType::EntityServer, NodeType::MessagesMixer, NodeType::AssetServer, NodeType::EntityScriptServer });} void Agent::requestScript() { auto nodeList = DependencyManager::get<NodeList>(); disconnect(&nodeList->getDomainHandler(), &DomainHandler::connectedToDomain, this, &Agent::requestScript); // figure out the URL for the script for this agent assignment QUrl scriptURL; if (_payload.isEmpty()) { scriptURL = QUrl(QString("http://%1:%2/assignment/%3/") .arg(nodeList->getDomainHandler().getIP().toString()) .arg(DOMAIN_SERVER_HTTP_PORT) .arg(uuidStringWithoutCurlyBraces(nodeList->getSessionUUID()))); } else { scriptURL = QUrl(_payload); } // make sure this is not a script request for the file scheme if (scriptURL.scheme() == HIFI_URL_SCHEME_FILE) { qWarning() << "Cannot load script for Agent from local filesystem."; scriptRequestFinished(); return; } auto request = DependencyManager::get<ResourceManager>()->createResourceRequest( this, scriptURL, true, -1, "Agent::requestScript"); if (!request) { qWarning() << "Could not create ResourceRequest for Agent script at" << scriptURL.toString(); scriptRequestFinished(); return; } // setup a timeout for script request static const int SCRIPT_TIMEOUT_MS = 10000; _scriptRequestTimeout = new QTimer; connect(_scriptRequestTimeout, &QTimer::timeout, this, &Agent::scriptRequestFinished); _scriptRequestTimeout->start(SCRIPT_TIMEOUT_MS); connect(request, &ResourceRequest::finished, this, &Agent::scriptRequestFinished); if (scriptURL.scheme() == URL_SCHEME_ATP) { // we have an ATP URL for the script - if we're not currently connected to the AssetServer // then wait for the nodeConnected signal to fire off the request auto assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (!assetServer || !assetServer->getActiveSocket()) { qDebug() << "Waiting to connect to Asset Server for ATP script download."; _pendingScriptRequest = request; return; } } qInfo() << "Requesting script at URL" << qPrintable(request->getUrl().toString()); request->send();} void Agent::nodeActivated(SharedNodePointer activatedNode) { if (_pendingScriptRequest && activatedNode->getType() == NodeType::AssetServer) { qInfo() << "Requesting script at URL" << qPrintable(_pendingScriptRequest->getUrl().toString()); _pendingScriptRequest->send(); _pendingScriptRequest = nullptr; } if (activatedNode->getType() == NodeType::AudioMixer) { negotiateAudioFormat(); }} void Agent::nodeKilled(SharedNodePointer killedNode) { if (killedNode->getType() == NodeType::EntityServer) { // an entity server has gone away, ask the headless viewer to clear its tree _entityViewer.clear(); }} void Agent::negotiateAudioFormat() { auto nodeList = DependencyManager::get<NodeList>(); auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); quint8 numberOfCodecs = (quint8)codecPlugins.size(); negotiateFormatPacket->writePrimitive(numberOfCodecs); for (const auto& plugin : codecPlugins) { auto codecName = plugin->getName(); negotiateFormatPacket->writeString(codecName); } // grab our audio mixer from the NodeList, if it exists SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer) { // send off this mute packet nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); }} void Agent::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) { QString selectedCodecName = message->readString(); selectAudioFormat(selectedCodecName);} void Agent::selectAudioFormat(const QString& selectedCodecName) { if (_selectedCodecName == selectedCodecName) { return; } _selectedCodecName = selectedCodecName; qDebug() << "Selected Codec:" << _selectedCodecName; // release any old codec encoder/decoder first... if (_codec && _encoder) { _codec->releaseEncoder(_encoder); _encoder = nullptr; _codec = nullptr; } _receivedAudioStream.cleanupCodec(); const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); for (const auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; } }} void Agent::scriptRequestFinished() { auto request = qobject_cast<ResourceRequest*>(sender()); // stop the script request timeout, if it's running if (_scriptRequestTimeout) { QMetaObject::invokeMethod(_scriptRequestTimeout, "stop"); _scriptRequestTimeout->deleteLater(); } if (request && request->getResult() == ResourceRequest::Success) { _scriptContents = request->getData(); qInfo() << "Downloaded script:" << _scriptContents; // we could just call executeScript directly - we use a QueuedConnection to allow scriptRequestFinished // to return before calling executeScript QMetaObject::invokeMethod(this, "executeScript", Qt::QueuedConnection); } else { if (request) { qWarning() << "Failed to download script at" << request->getUrl().toString() << " - bailing on assignment."; qWarning() << "ResourceRequest error was" << request->getResult(); } else { qWarning() << "Failed to download script - request timed out. Bailing on assignment."; } setFinished(true); } request->deleteLater();} void Agent::executeScript() { // the following block is scoped so that any shared pointers we take here // are cleared before we call setFinished at the end of the function { _scriptManager = scriptManagerFactory(ScriptManager::AGENT_SCRIPT, _scriptContents, _payload); // setup an Avatar for the script to use auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>(); scriptedAvatar->setID(getSessionUUID()); // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar->setSkeletonModelURL(QUrl()); // force lazy initialization of the head data for the scripted avatar // since it is referenced below by computeLoudness and getAudioLoudness scriptedAvatar->getHeadOrientation(); auto scriptEngine = _scriptManager->engine(); auto scopeGuard = scriptEngine->getScopeGuard(); auto* sgp = scopeGuard.get(); // give this AvatarData object to the script engine scriptEngine->registerGlobalObject(sgp, "Avatar", scriptedAvatar.data()); // give scripts access to the Users object scriptEngine->registerGlobalObject(sgp, "Users", DependencyManager::get<UsersScriptingInterface>().data()); auto player = DependencyManager::get<recording::Deck>(); connect(player.data(), &recording::Deck::playbackStateChanged, [&player, &scriptedAvatar] { if (player->isPlaying()) { auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>(); if (recordingInterface->getPlayFromCurrentLocation()) { scriptedAvatar->setRecordingBasis(); } // these procedural movements are included in the recordings scriptedAvatar->setHasScriptedBlendshapes(true); scriptedAvatar->setHasProceduralEyeFaceMovement(false); scriptedAvatar->setHasProceduralBlinkFaceMovement(false); scriptedAvatar->setHasAudioEnabledFaceMovement(false); } else { scriptedAvatar->clearRecordingBasis(); // restore procedural blendshape movement scriptedAvatar->setHasScriptedBlendshapes(false); scriptedAvatar->setHasProceduralEyeFaceMovement(true); scriptedAvatar->setHasProceduralBlinkFaceMovement(true); scriptedAvatar->setHasAudioEnabledFaceMovement(true); } }); using namespace recording; static const FrameType AVATAR_FRAME_TYPE = Frame::registerFrameType(AvatarData::FRAME_NAME); Frame::registerFrameHandler(AVATAR_FRAME_TYPE, [scriptedAvatar](Frame::ConstPointer frame) { auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>(); bool useFrameSkeleton = recordingInterface->getPlayerUseSkeletonModel(); // FIXME - the ability to switch the avatar URL is not actually supported when playing back from a recording if (!useFrameSkeleton) { static std::once_flag warning; std::call_once(warning, [] { qWarning() << "Recording.setPlayerUseSkeletonModel(false) is not currently supported."; }); } AvatarData::fromFrame(frame->data, *scriptedAvatar); }); using namespace recording; static const FrameType AUDIO_FRAME_TYPE = Frame::registerFrameType(AudioConstants::getAudioFrameName()); Frame::registerFrameHandler(AUDIO_FRAME_TYPE, [this, &player, &scriptedAvatar](Frame::ConstPointer frame) { if (_shouldMuteRecordingAudio) { return; } static quint16 audioSequenceNumber{ 0 }; QByteArray audio(frame->data); int16_t* samples = reinterpret_cast<int16_t*>(audio.data()); int numSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; auto volume = player->getVolume(); if (volume >= 0.0f && volume < 1.0f) { int32_t fract = (int32_t)(volume * (float)(1 << 16)); // Q16 for (int i = 0; i < numSamples; i++) { samples[i] = (fract * (int32_t)samples[i]) >> 16; } } if (_isNoiseGateEnabled) { _audioGate.render(samples, samples, numSamples); } computeLoudness(&audio, scriptedAvatar); // state machine to detect gate opening and closing bool audioGateOpen = (scriptedAvatar->getAudioLoudness() != 0.0f); bool openedInLastBlock = !_audioGateOpen && audioGateOpen; // the gate just opened bool closedInLastBlock = _audioGateOpen && !audioGateOpen; // the gate just closed _audioGateOpen = audioGateOpen; Q_UNUSED(openedInLastBlock); // the codec must be flushed to silence before sending silent packets, // so delay the transition to silent packets by one packet after becoming silent. auto packetType = PacketType::MicrophoneAudioNoEcho; if (!audioGateOpen && !closedInLastBlock) { packetType = PacketType::SilentAudioFrame; } Transform audioTransform; auto headOrientation = scriptedAvatar->getHeadOrientation(); audioTransform.setTranslation(scriptedAvatar->getWorldPosition()); audioTransform.setRotation(headOrientation); QByteArray encodedBuffer; if (_encoder) { _encoder->encode(audio, encodedBuffer); } else { encodedBuffer = audio; } AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, false, audioTransform, scriptedAvatar->getWorldPosition(), glm::vec3(0),Lines 1–500 of 914. Load more
// // Agent.cpp // assignment-client/src // // Created by Stephen Birarda on 7/1/13. // Copyright 2013 High Fidelity, Inc. // Copyright 2023 Overte e.V. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // SPDX-License-Identifier: Apache-2.0 // #include "Agent.h" #include <QtCore/QCoreApplication> #include <QtCore/QEventLoop> #include <QtCore/QStandardPaths> #include <QtNetwork/QNetworkDiskCache> #include <QtNetwork/QNetworkRequest> #include <QtNetwork/QNetworkReply> #include <QThread> #include <AnimationCacheScriptingInterface.h> #include <AssetClient.h> #include <AvatarHashMap.h> #include <AudioInjectorManager.h> #include <AssetClient.h> #include <DebugDraw.h> #include <EntityScriptingInterface.h> #include <LocationScriptingInterface.h> #include <AudioScriptingInterface.h> #include <MessagesClient.h> #include <NetworkAccessManager.h> #include <NodeList.h> #include <udt/PacketHeaders.h> #include <ResourceCache.h> #include <ResourceScriptingInterface.h> #include <ScriptCache.h> #include <ScriptEngine.h> #include <ScriptEngines.h> #include <ScriptManager.h> #include <SoundCacheScriptingInterface.h> #include <SoundCache.h> #include <UserActivityLoggerScriptingInterface.h> #include <UsersScriptingInterface.h> #include <UUID.h> #include <recording/ClipCache.h> #include <recording/Deck.h> #include <recording/Recorder.h> #include <recording/Frame.h> #include <plugins/CodecPlugin.h> #include <plugins/PluginManager.h> #include <WebSocketServerClass.h> #include <EntityScriptingInterface.h> // TODO: consider moving to scriptengine.h #include <hfm/ModelFormatRegistry.h> #include "entities/AssignmentParentFinder.h" #include "AssignmentDynamicFactory.h" #include <recording/RecordingScriptingInterface.h> #include "AbstractAudioInterface.h" #include "AgentScriptingInterface.h" static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 10; Agent::Agent(ReceivedMessage& message) : ThreadedAssignment(message), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), _audioGate(AudioConstants::SAMPLE_RATE, AudioConstants::MONO), _avatarAudioTimer(this) { DependencyManager::set<ScriptableAvatar>(); DependencyManager::registerInheritance<EntityDynamicFactoryInterface, AssignmentDynamicFactory>(); DependencyManager::set<AssignmentDynamicFactory>(); DependencyManager::set<AnimationCache>(); DependencyManager::set<AnimationCacheScriptingInterface>(); DependencyManager::set<EntityScriptingInterface>(false); DependencyManager::set<ResourceScriptingInterface>(); DependencyManager::set<UserActivityLoggerScriptingInterface>(); _entityEditSender.setPacketsPerSecond(DEFAULT_ENTITY_PPS_PER_SCRIPT); DependencyManager::get<EntityScriptingInterface>()->setPacketSender(&_entityEditSender); DependencyManager::set<ResourceManager>(); DependencyManager::set<PluginManager>()->instantiate(); DependencyManager::registerInheritance<SpatialParentFinder, AssignmentParentFinder>(); DependencyManager::set<ResourceCacheSharedItems>(); DependencyManager::set<SoundCache>(); DependencyManager::set<SoundCacheScriptingInterface>(); DependencyManager::set<AudioScriptingInterface>(); DependencyManager::set<AudioInjectorManager>(); DependencyManager::set<recording::Deck>(); DependencyManager::set<recording::Recorder>(); DependencyManager::set<recording::ClipCache>(); DependencyManager::set<RecordingScriptingInterface>(); DependencyManager::set<UsersScriptingInterface>(); DependencyManager::set<ModelFormatRegistry>(); DependencyManager::set<ModelCache>(); // Needed to ensure the creation of the DebugDraw instance on the main thread DebugDraw::getInstance(); auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver(); packetReceiver.registerListenerForTypes( { PacketType::MixedAudio, PacketType::SilentAudioFrame }, PacketReceiver::makeUnsourcedListenerReference<Agent>(this, &Agent::handleAudioPacket)); packetReceiver.registerListenerForTypes( { PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase }, PacketReceiver::makeSourcedListenerReference<Agent>(this, &Agent::handleOctreePacket)); packetReceiver.registerListener(PacketType::SelectedAudioFormat, PacketReceiver::makeUnsourcedListenerReference<Agent>(this, &Agent::handleSelectedAudioFormat)); // 100Hz timer for audio const int TARGET_INTERVAL_MSEC = 10; // 10ms connect(&_avatarAudioTimer, &QTimer::timeout, this, &Agent::processAgentAvatarAudio); _avatarAudioTimer.setSingleShot(false); _avatarAudioTimer.setInterval(TARGET_INTERVAL_MSEC); _avatarAudioTimer.setTimerType(Qt::PreciseTimer); } void Agent::playAvatarSound(SharedSoundPointer sound) { // this must happen on Agent's main thread if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "playAvatarSound", Q_ARG(SharedSoundPointer, sound)); return; } else { // TODO: seems to add occasional artifact in tests. I believe it is // correct to do this, but need to figure out for sure, so commenting this // out until I verify. // _numAvatarSoundSentBytes = 0; setAvatarSound(sound); } } void Agent::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) { auto packetType = message->getType(); if (packetType == PacketType::OctreeStats) { int statsMessageLength = OctreeHeadlessViewer::parseOctreeStats(message, senderNode); if (message->getSize() > statsMessageLength) { // pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it int piggyBackedSizeWithHeader = message->getSize() - statsMessageLength; auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]); memcpy(buffer.get(), message->getRawMessage() + statsMessageLength, piggyBackedSizeWithHeader); auto newPacket = NLPacket::fromReceivedPacket(std::move(buffer), piggyBackedSizeWithHeader, message->getSenderSockAddr()); message = QSharedPointer<ReceivedMessage>::create(*newPacket); } else { return; // bail since no piggyback data } packetType = message->getType(); } // fall through to piggyback message if (packetType == PacketType::EntityData) { _entityViewer.processDatagram(*message, senderNode); } else if (packetType == PacketType::EntityErase) { _entityViewer.processEraseMessage(*message, senderNode); } } void Agent::handleAudioPacket(QSharedPointer<ReceivedMessage> message) { _receivedAudioStream.parseData(*message); _lastReceivedAudioLoudness = _receivedAudioStream.getNextOutputFrameLoudness(); _receivedAudioStream.clearBuffer(); } static const QString AGENT_LOGGING_NAME = "agent"; void Agent::run() { // Create ScriptEngines on threaded-assignment thread then move to main thread. DependencyManager::set<ScriptEngines>(ScriptManager::AGENT_SCRIPT)->moveToThread(qApp->thread()); DependencyManager::set<ScriptCache>(); // make sure we request our script once the agent connects to the domain auto nodeList = DependencyManager::get<NodeList>(); connect(&nodeList->getDomainHandler(), &DomainHandler::connectedToDomain, this, &Agent::requestScript); ThreadedAssignment::commonInit(AGENT_LOGGING_NAME, NodeType::Agent); // Setup MessagesClient auto messagesClient = DependencyManager::set<MessagesClient>(); messagesClient->startThread(); // make sure we hear about connected nodes so we can grab an ATP script if a request is pending connect(nodeList.data(), &LimitedNodeList::nodeActivated, this, &Agent::nodeActivated); // make sure we hear about dissappearing nodes so we can clear the entity tree if an entity server goes away connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &Agent::nodeKilled); nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::AudioMixer, NodeType::AvatarMixer, NodeType::EntityServer, NodeType::MessagesMixer, NodeType::AssetServer, NodeType::EntityScriptServer }); } void Agent::requestScript() { auto nodeList = DependencyManager::get<NodeList>(); disconnect(&nodeList->getDomainHandler(), &DomainHandler::connectedToDomain, this, &Agent::requestScript); // figure out the URL for the script for this agent assignment QUrl scriptURL; if (_payload.isEmpty()) { scriptURL = QUrl(QString("http://%1:%2/assignment/%3/") .arg(nodeList->getDomainHandler().getIP().toString()) .arg(DOMAIN_SERVER_HTTP_PORT) .arg(uuidStringWithoutCurlyBraces(nodeList->getSessionUUID()))); } else { scriptURL = QUrl(_payload); } // make sure this is not a script request for the file scheme if (scriptURL.scheme() == HIFI_URL_SCHEME_FILE) { qWarning() << "Cannot load script for Agent from local filesystem."; scriptRequestFinished(); return; } auto request = DependencyManager::get<ResourceManager>()->createResourceRequest( this, scriptURL, true, -1, "Agent::requestScript"); if (!request) { qWarning() << "Could not create ResourceRequest for Agent script at" << scriptURL.toString(); scriptRequestFinished(); return; } // setup a timeout for script request static const int SCRIPT_TIMEOUT_MS = 10000; _scriptRequestTimeout = new QTimer; connect(_scriptRequestTimeout, &QTimer::timeout, this, &Agent::scriptRequestFinished); _scriptRequestTimeout->start(SCRIPT_TIMEOUT_MS); connect(request, &ResourceRequest::finished, this, &Agent::scriptRequestFinished); if (scriptURL.scheme() == URL_SCHEME_ATP) { // we have an ATP URL for the script - if we're not currently connected to the AssetServer // then wait for the nodeConnected signal to fire off the request auto assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (!assetServer || !assetServer->getActiveSocket()) { qDebug() << "Waiting to connect to Asset Server for ATP script download."; _pendingScriptRequest = request; return; } } qInfo() << "Requesting script at URL" << qPrintable(request->getUrl().toString()); request->send(); } void Agent::nodeActivated(SharedNodePointer activatedNode) { if (_pendingScriptRequest && activatedNode->getType() == NodeType::AssetServer) { qInfo() << "Requesting script at URL" << qPrintable(_pendingScriptRequest->getUrl().toString()); _pendingScriptRequest->send(); _pendingScriptRequest = nullptr; } if (activatedNode->getType() == NodeType::AudioMixer) { negotiateAudioFormat(); } } void Agent::nodeKilled(SharedNodePointer killedNode) { if (killedNode->getType() == NodeType::EntityServer) { // an entity server has gone away, ask the headless viewer to clear its tree _entityViewer.clear(); } } void Agent::negotiateAudioFormat() { auto nodeList = DependencyManager::get<NodeList>(); auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); quint8 numberOfCodecs = (quint8)codecPlugins.size(); negotiateFormatPacket->writePrimitive(numberOfCodecs); for (const auto& plugin : codecPlugins) { auto codecName = plugin->getName(); negotiateFormatPacket->writeString(codecName); } // grab our audio mixer from the NodeList, if it exists SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer) { // send off this mute packet nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); } } void Agent::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) { QString selectedCodecName = message->readString(); selectAudioFormat(selectedCodecName); } void Agent::selectAudioFormat(const QString& selectedCodecName) { if (_selectedCodecName == selectedCodecName) { return; } _selectedCodecName = selectedCodecName; qDebug() << "Selected Codec:" << _selectedCodecName; // release any old codec encoder/decoder first... if (_codec && _encoder) { _codec->releaseEncoder(_encoder); _encoder = nullptr; _codec = nullptr; } _receivedAudioStream.cleanupCodec(); const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); for (const auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; } } } void Agent::scriptRequestFinished() { auto request = qobject_cast<ResourceRequest*>(sender()); // stop the script request timeout, if it's running if (_scriptRequestTimeout) { QMetaObject::invokeMethod(_scriptRequestTimeout, "stop"); _scriptRequestTimeout->deleteLater(); } if (request && request->getResult() == ResourceRequest::Success) { _scriptContents = request->getData(); qInfo() << "Downloaded script:" << _scriptContents; // we could just call executeScript directly - we use a QueuedConnection to allow scriptRequestFinished // to return before calling executeScript QMetaObject::invokeMethod(this, "executeScript", Qt::QueuedConnection); } else { if (request) { qWarning() << "Failed to download script at" << request->getUrl().toString() << " - bailing on assignment."; qWarning() << "ResourceRequest error was" << request->getResult(); } else { qWarning() << "Failed to download script - request timed out. Bailing on assignment."; } setFinished(true); } request->deleteLater(); } void Agent::executeScript() { // the following block is scoped so that any shared pointers we take here // are cleared before we call setFinished at the end of the function { _scriptManager = scriptManagerFactory(ScriptManager::AGENT_SCRIPT, _scriptContents, _payload); // setup an Avatar for the script to use auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>(); scriptedAvatar->setID(getSessionUUID()); // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar->setSkeletonModelURL(QUrl()); // force lazy initialization of the head data for the scripted avatar // since it is referenced below by computeLoudness and getAudioLoudness scriptedAvatar->getHeadOrientation(); auto scriptEngine = _scriptManager->engine(); auto scopeGuard = scriptEngine->getScopeGuard(); auto* sgp = scopeGuard.get(); // give this AvatarData object to the script engine scriptEngine->registerGlobalObject(sgp, "Avatar", scriptedAvatar.data()); // give scripts access to the Users object scriptEngine->registerGlobalObject(sgp, "Users", DependencyManager::get<UsersScriptingInterface>().data()); auto player = DependencyManager::get<recording::Deck>(); connect(player.data(), &recording::Deck::playbackStateChanged, [&player, &scriptedAvatar] { if (player->isPlaying()) { auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>(); if (recordingInterface->getPlayFromCurrentLocation()) { scriptedAvatar->setRecordingBasis(); } // these procedural movements are included in the recordings scriptedAvatar->setHasScriptedBlendshapes(true); scriptedAvatar->setHasProceduralEyeFaceMovement(false); scriptedAvatar->setHasProceduralBlinkFaceMovement(false); scriptedAvatar->setHasAudioEnabledFaceMovement(false); } else { scriptedAvatar->clearRecordingBasis(); // restore procedural blendshape movement scriptedAvatar->setHasScriptedBlendshapes(false); scriptedAvatar->setHasProceduralEyeFaceMovement(true); scriptedAvatar->setHasProceduralBlinkFaceMovement(true); scriptedAvatar->setHasAudioEnabledFaceMovement(true); } }); using namespace recording; static const FrameType AVATAR_FRAME_TYPE = Frame::registerFrameType(AvatarData::FRAME_NAME); Frame::registerFrameHandler(AVATAR_FRAME_TYPE, [scriptedAvatar](Frame::ConstPointer frame) { auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>(); bool useFrameSkeleton = recordingInterface->getPlayerUseSkeletonModel(); // FIXME - the ability to switch the avatar URL is not actually supported when playing back from a recording if (!useFrameSkeleton) { static std::once_flag warning; std::call_once(warning, [] { qWarning() << "Recording.setPlayerUseSkeletonModel(false) is not currently supported."; }); } AvatarData::fromFrame(frame->data, *scriptedAvatar); }); using namespace recording; static const FrameType AUDIO_FRAME_TYPE = Frame::registerFrameType(AudioConstants::getAudioFrameName()); Frame::registerFrameHandler(AUDIO_FRAME_TYPE, [this, &player, &scriptedAvatar](Frame::ConstPointer frame) { if (_shouldMuteRecordingAudio) { return; } static quint16 audioSequenceNumber{ 0 }; QByteArray audio(frame->data); int16_t* samples = reinterpret_cast<int16_t*>(audio.data()); int numSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; auto volume = player->getVolume(); if (volume >= 0.0f && volume < 1.0f) { int32_t fract = (int32_t)(volume * (float)(1 << 16)); // Q16 for (int i = 0; i < numSamples; i++) { samples[i] = (fract * (int32_t)samples[i]) >> 16; } } if (_isNoiseGateEnabled) { _audioGate.render(samples, samples, numSamples); } computeLoudness(&audio, scriptedAvatar); // state machine to detect gate opening and closing bool audioGateOpen = (scriptedAvatar->getAudioLoudness() != 0.0f); bool openedInLastBlock = !_audioGateOpen && audioGateOpen; // the gate just opened bool closedInLastBlock = _audioGateOpen && !audioGateOpen; // the gate just closed _audioGateOpen = audioGateOpen; Q_UNUSED(openedInLastBlock); // the codec must be flushed to silence before sending silent packets, // so delay the transition to silent packets by one packet after becoming silent. auto packetType = PacketType::MicrophoneAudioNoEcho; if (!audioGateOpen && !closedInLastBlock) { packetType = PacketType::SilentAudioFrame; } Transform audioTransform; auto headOrientation = scriptedAvatar->getHeadOrientation(); audioTransform.setTranslation(scriptedAvatar->getWorldPosition()); audioTransform.setRotation(headOrientation); QByteArray encodedBuffer; if (_encoder) { _encoder->encode(audio, encodedBuffer); } else { encodedBuffer = audio; } AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, false, audioTransform, scriptedAvatar->getWorldPosition(), glm::vec3(0),