//// AvatarData.cpp// libraries/avatars/src//// Created by Stephen Birarda on 4/9/13.// Copyright 2013 High Fidelity, Inc.// Copyright 2022-2023 Overte e.V.//// Distributed under the Apache License, Version 2.0.// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// SPDX-License-Identifier: Apache-2.0// #include "AvatarData.h" #include <cstdio>#include <cstring>#include <stdint.h> #include <QtCore/QDataStream>#include <QtCore/QThread>#include <QtCore/QUuid>#include <QtCore/QJsonDocument>#include <QtCore/QJsonArray>#include <QtCore/QJsonObject>#include <QtNetwork/QNetworkReply>#include <QtNetwork/QNetworkRequest> #include <shared/QtHelpers.h>#include <QVariantGLM.h>#include <Transform.h>#include <NetworkAccessManager.h>#include <NodeList.h>#include <udt/PacketHeaders.h>#include <GLMHelpers.h>#include <StreamUtils.h>#include <UUID.h>#include <shared/JSONHelpers.h>#include <ScriptEngine.h>#include <ScriptEngineCast.h>#include <ScriptManager.h>#include <ScriptValueIterator.h>#include <ScriptValueUtils.h>#include <ShapeInfo.h>#include <AudioHelpers.h>#include <Profile.h>#include <VariantMapToScriptValue.h>#include <BitVectorHelpers.h> #include "AvatarLogging.h"#include "AvatarTraits.h"#include "ClientTraitsHandler.h"#include "ResourceRequestObserver.h"#include "WarningsSuppression.h" //#define WANT_DEBUG quint64 DEFAULT_FILTERED_LOG_EXPIRY = 2 * USECS_PER_SECOND; using namespace std; const QString AvatarData::FRAME_NAME = "com.highfidelity.recording.AvatarData"; static const int TRANSLATION_COMPRESSION_RADIX = 14;static const int HAND_CONTROLLER_COMPRESSION_RADIX = 12;static const int SENSOR_TO_WORLD_SCALE_RADIX = 10;static const float AUDIO_LOUDNESS_SCALE = 1024.0f;static const float DEFAULT_AVATAR_DENSITY = 1000.0f; // density of water #define ASSERT(COND) do { if (!(COND)) { abort(); } } while(0) STATIC_SCRIPT_TYPES_INITIALIZER((+[](ScriptManager* manager) { auto scriptEngine = manager->engine().get(); scriptRegisterMetaType<RayToAvatarIntersectionResult, RayToAvatarIntersectionResultToScriptValue, RayToAvatarIntersectionResultFromScriptValue>(scriptEngine); scriptRegisterMetaType<AvatarEntityMap, AvatarEntityMapToScriptValue, AvatarEntityMapFromScriptValue>(scriptEngine);})); size_t AvatarDataPacket::maxFaceTrackerInfoSize(size_t numBlendshapeCoefficients) { return FACE_TRACKER_INFO_SIZE + numBlendshapeCoefficients * sizeof(float);} size_t AvatarDataPacket::maxJointDataSize(size_t numJoints) { const size_t validityBitsSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints totalSize += validityBitsSize; // Orientations mask totalSize += numJoints * sizeof(SixByteQuat); // Orientations totalSize += validityBitsSize; // Translations mask totalSize += sizeof(float); // maxTranslationDimension totalSize += numJoints * sizeof(SixByteTrans); // Translations return totalSize;} size_t AvatarDataPacket::minJointDataSize(size_t numJoints) { const size_t validityBitsSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints totalSize += validityBitsSize; // Orientations mask // assume no valid rotations totalSize += validityBitsSize; // Translations mask totalSize += sizeof(float); // maxTranslationDimension // assume no valid translations return totalSize;} size_t AvatarDataPacket::maxJointDefaultPoseFlagsSize(size_t numJoints) { const size_t bitVectorSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints // one set of bits for rotation and one for translation const size_t NUM_BIT_VECTORS_IN_DEFAULT_POSE_FLAGS_SECTION = 2; totalSize += NUM_BIT_VECTORS_IN_DEFAULT_POSE_FLAGS_SECTION * bitVectorSize; return totalSize;} AvatarData::AvatarData() : SpatiallyNestable(NestableType::Avatar, QUuid()), _handPosition(0.0f), _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), _targetVelocity(0.0f), _density(DEFAULT_AVATAR_DENSITY){ connect(this, &AvatarData::lookAtSnappingChanged, this, &AvatarData::markIdentityDataChanged);} AvatarData::~AvatarData() { delete _headData;} // We cannot have a file-level variable (const or otherwise) in the AvatarInfo if it uses PathUtils, because that references Application, which will not yet initialized.// Thus we have a static class getter, referencing a static class var.QUrl AvatarData::_defaultFullAvatarModelUrl = {}; // In C++, if this initialization were in the AvatarInfo, every file would have it's own copy, even for class vars.const QUrl& AvatarData::defaultFullAvatarModelUrl() { if (_defaultFullAvatarModelUrl.isEmpty()) { _defaultFullAvatarModelUrl = PathUtils::resourcesUrl("/meshes/defaultAvatar_full.fst"); } return _defaultFullAvatarModelUrl;} void AvatarData::setTargetScale(float targetScale) { auto newValue = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); if (_targetScale != newValue) { _targetScale = newValue; _scaleChanged = usecTimestampNow(); _avatarScaleChanged = _scaleChanged; }} float AvatarData::getDomainLimitedScale() const { if (canMeasureEyeHeight()) { const float minScale = getDomainMinScale(); const float maxScale = getDomainMaxScale(); return glm::clamp(_targetScale, minScale, maxScale); } else { // We can't make a good estimate. return _targetScale; }} void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { if (hasScriptedBlendshapes == _headData->getHasScriptedBlendshapes()) { return; } if (!hasScriptedBlendshapes) { // send a forced avatarData update to make sure the script can send neutal blendshapes on unload // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true // before sending the update, or else it won't send the neutal blendshapes to the receiving clients sendAvatarDataPacket(true); } _headData->setHasScriptedBlendshapes(hasScriptedBlendshapes);} bool AvatarData::getHasScriptedBlendshapes() const { return _headData->getHasScriptedBlendshapes();} void AvatarData::setHasProceduralBlinkFaceMovement(bool value) { _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, value);} bool AvatarData::getHasProceduralBlinkFaceMovement() const { return _headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation);} void AvatarData::setHasProceduralEyeFaceMovement(bool value) { _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, value);} bool AvatarData::getHasProceduralEyeFaceMovement() const { return _headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation);} void AvatarData::setHasAudioEnabledFaceMovement(bool value) { _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, value);} bool AvatarData::getHasAudioEnabledFaceMovement() const { return _headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation);} void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { _domainMinimumHeight = glm::clamp(domainMinimumHeight, MIN_AVATAR_HEIGHT, MAX_AVATAR_HEIGHT);} void AvatarData::setDomainMaximumHeight(float domainMaximumHeight) { _domainMaximumHeight = glm::clamp(domainMaximumHeight, MIN_AVATAR_HEIGHT, MAX_AVATAR_HEIGHT);} float AvatarData::getDomainMinScale() const { float unscaledHeight = getUnscaledHeight(); const float EPSILON = 1.0e-4f; if (unscaledHeight <= EPSILON) { unscaledHeight = DEFAULT_AVATAR_HEIGHT; } return _domainMinimumHeight / unscaledHeight;} float AvatarData::getDomainMaxScale() const { float unscaledHeight = getUnscaledHeight(); const float EPSILON = 1.0e-4f; if (unscaledHeight <= EPSILON) { unscaledHeight = DEFAULT_AVATAR_HEIGHT; } return _domainMaximumHeight / unscaledHeight;} float AvatarData::getUnscaledHeight() const { const float eyeHeight = getUnscaledEyeHeight(); const float ratio = eyeHeight / DEFAULT_AVATAR_HEIGHT; return eyeHeight + ratio * DEFAULT_AVATAR_EYE_TO_TOP_OF_HEAD;} float AvatarData::getHeight() const { const float eyeHeight = getEyeHeight(); const float ratio = eyeHeight / DEFAULT_AVATAR_HEIGHT; return eyeHeight + ratio * DEFAULT_AVATAR_EYE_TO_TOP_OF_HEAD;} glm::vec3 AvatarData::getHandPosition() const { return getWorldOrientation() * _handPosition + getWorldPosition();} void AvatarData::setHandPosition(const glm::vec3& handPosition) { // store relative to position/orientation _handPosition = glm::inverse(getWorldOrientation()) * (handPosition - getWorldPosition());} void AvatarData::lazyInitHeadData() const { // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(const_cast<AvatarData*>(this)); }} float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const { auto distance = glm::distance(_globalPosition, viewerPosition); float result = ROTATION_CHANGE_179D; // assume worst if (distance < AVATAR_DISTANCE_LEVEL_1) { result = AVATAR_MIN_ROTATION_DOT; } else if (distance < AVATAR_DISTANCE_LEVEL_2) { result = ROTATION_CHANGE_2D; } else if (distance < AVATAR_DISTANCE_LEVEL_3) { result = ROTATION_CHANGE_4D; } else if (distance < AVATAR_DISTANCE_LEVEL_4) { result = ROTATION_CHANGE_6D; } else if (distance < AVATAR_DISTANCE_LEVEL_5) { result = ROTATION_CHANGE_15D; } return result;} float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const { return AVATAR_MIN_TRANSLATION; // Eventually make this distance sensitive as well} // we want to track outbound data in this case...QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) { auto lastSentTime = _lastToByteArray; _lastToByteArray = usecTimestampNow(); AvatarDataPacket::SendStatus sendStatus; auto avatarByteArray = AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), sendStatus, dropFaceTracking, false, glm::vec3(0), nullptr, 0, &_outboundDataRate); return avatarByteArray;} QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData, AvatarDataPacket::SendStatus& sendStatus, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut, int maxDataSize, AvatarDataRate* outboundDataRateOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); bool sendAll = (dataDetail == SendAllData); bool sendMinimum = (dataDetail == MinimumData); bool sendPALMinimum = (dataDetail == PALMinimum); lazyInitHeadData(); ASSERT(maxDataSize == 0 || (size_t)maxDataSize >= AvatarDataPacket::MIN_BULK_PACKET_SIZE); // Leading flags, to indicate how much data is actually included in the packet... AvatarDataPacket::HasFlags wantedFlags = 0; AvatarDataPacket::HasFlags includedFlags = 0; AvatarDataPacket::HasFlags extraReturnedFlags = 0; // For partial joint data. // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { sendStatus.itemFlags = wantedFlags; QByteArray avatarDataByteArray; if (sendStatus.sendUUID) { avatarDataByteArray.append(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID); } avatarDataByteArray.append((char*) &wantedFlags, sizeof wantedFlags); return avatarDataByteArray; } // FIXME - // // BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... // this is an iFrame issue... what to do about that? // // BUG -- Resizing avatar seems to "take too long"... the avatar doesn't redraw at smaller size right away // // TODO consider these additional optimizations in the future // 1) SensorToWorld - should we only send this for avatars with attachments?? - 20 bytes - 7.20 kbps // 2) GUIID for the session change to 2byte index (savings) - 14 bytes - 5.04 kbps // 3) Improve Joints -- currently we use rotational tolerances, but if we had skeleton/bone length data // we could do a better job of determining if the change in joints actually translates to visible // changes at distance. // // Potential savings: // 63 rotations * 6 bytes = 136kbps // 3 translations * 6 bytes = 6.48kbps // QUuid parentID; glm::mat4 leftFarGrabMatrix; glm::mat4 rightFarGrabMatrix; glm::mat4 mouseFarGrabMatrix; if (sendStatus.itemFlags == 0) { // New avatar ... bool hasAvatarGlobalPosition = true; // always include global position bool hasAvatarOrientation = false; bool hasAvatarBoundingBox = false; bool hasAvatarScale = false; bool hasLookAtPosition = false; bool hasAudioLoudness = false; bool hasSensorToWorldMatrix = false; bool hasJointData = false; bool hasJointDefaultPoseFlags = false; bool hasAdditionalFlags = false; // local position, and parent info only apply to avatars that are parented. The local position // and the parent info can change independently though, so we track their "changed since" // separately bool hasParentInfo = false; bool hasAvatarLocalPosition = false; bool hasHandControllers = false; bool hasFaceTrackerInfo = false; if (sendPALMinimum) { hasAudioLoudness = true; } else { hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); hasAvatarLocalPosition = hasParent() && (sendAll || tranlationChangedSince(lastSentTime) || parentInfoChangedSince(lastSentTime)); hasHandControllers = _controllerLeftHandMatrixCache.isValid() || _controllerRightHandMatrixCache.isValid(); hasFaceTrackerInfo = !dropFaceTracking && (getHasScriptedBlendshapes() || _headData->_hasInputDrivenBlendshapes) && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); hasJointData = !sendMinimum; hasJointDefaultPoseFlags = hasJointData; } wantedFlags = (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) | (hasHandControllers ? AvatarDataPacket::PACKET_HAS_HAND_CONTROLLERS : 0) | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0) | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); sendStatus.itemFlags = wantedFlags; sendStatus.rotationsSent = 0; sendStatus.translationsSent = 0; } else { // Continuing avatar ... wantedFlags = sendStatus.itemFlags; if (wantedFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { // Must send joints for grab joints - wantedFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; } } if (wantedFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { bool leftValid; leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); if (!leftValid) { leftFarGrabMatrix = glm::mat4(); } bool rightValid; rightFarGrabMatrix = _farGrabRightMatrixCache.get(rightValid); if (!rightValid) { rightFarGrabMatrix = glm::mat4(); } bool mouseValid; mouseFarGrabMatrix = _farGrabMouseMatrixCache.get(mouseValid); if (!mouseValid) { mouseFarGrabMatrix = glm::mat4(); } if (!(leftValid || rightValid || mouseValid)) { wantedFlags &= ~AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; } } if (wantedFlags & (AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS | AvatarDataPacket::PACKET_HAS_PARENT_INFO)) { parentID = getParentID(); } const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + NUM_BYTES_RFC4122_UUID + AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) + AvatarDataPacket::maxJointDataSize(_jointData.size()) + AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) + AvatarDataPacket::FAR_GRAB_JOINTS_SIZE; if (maxDataSize == 0) { maxDataSize = (int)byteArraySize; } QByteArray avatarDataByteArray((int)byteArraySize, 0); unsigned char* destinationBuffer = reinterpret_cast<unsigned char*>(avatarDataByteArray.data()); const unsigned char* const startPosition = destinationBuffer; const unsigned char* const packetEnd = destinationBuffer + maxDataSize; #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); // If we want an item and there's sufficient space:#define IF_AVATAR_SPACE(flag, space) \ if ((wantedFlags & AvatarDataPacket::flag) \ && (packetEnd - destinationBuffer) >= (ptrdiff_t)(space) \ && (includedFlags |= AvatarDataPacket::flag)) if (sendStatus.sendUUID) { memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); destinationBuffer += NUM_BYTES_RFC4122_UUID; } unsigned char * packetFlagsLocation = destinationBuffer; destinationBuffer += sizeof(wantedFlags); IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_globalPosition); int numBytes = destinationBuffer - startSection; if (outboundDataRateOut) { outboundDataRateOut->globalPositionRate.increment(numBytes); } } IF_AVATAR_SPACE(PACKET_HAS_AVATAR_BOUNDING_BOX, sizeof _globalBoundingBoxDimensions + sizeof _globalBoundingBoxOffset) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_globalBoundingBoxDimensions); AVATAR_MEMCPY(_globalBoundingBoxOffset); Lines 1–500 of 3037. Load more
// // AvatarData.cpp // libraries/avatars/src // // Created by Stephen Birarda on 4/9/13. // Copyright 2013 High Fidelity, Inc. // Copyright 2022-2023 Overte e.V. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // SPDX-License-Identifier: Apache-2.0 // #include "AvatarData.h" #include <cstdio> #include <cstring> #include <stdint.h> #include <QtCore/QDataStream> #include <QtCore/QThread> #include <QtCore/QUuid> #include <QtCore/QJsonDocument> #include <QtCore/QJsonArray> #include <QtCore/QJsonObject> #include <QtNetwork/QNetworkReply> #include <QtNetwork/QNetworkRequest> #include <shared/QtHelpers.h> #include <QVariantGLM.h> #include <Transform.h> #include <NetworkAccessManager.h> #include <NodeList.h> #include <udt/PacketHeaders.h> #include <GLMHelpers.h> #include <StreamUtils.h> #include <UUID.h> #include <shared/JSONHelpers.h> #include <ScriptEngine.h> #include <ScriptEngineCast.h> #include <ScriptManager.h> #include <ScriptValueIterator.h> #include <ScriptValueUtils.h> #include <ShapeInfo.h> #include <AudioHelpers.h> #include <Profile.h> #include <VariantMapToScriptValue.h> #include <BitVectorHelpers.h> #include "AvatarLogging.h" #include "AvatarTraits.h" #include "ClientTraitsHandler.h" #include "ResourceRequestObserver.h" #include "WarningsSuppression.h" //#define WANT_DEBUG quint64 DEFAULT_FILTERED_LOG_EXPIRY = 2 * USECS_PER_SECOND; using namespace std; const QString AvatarData::FRAME_NAME = "com.highfidelity.recording.AvatarData"; static const int TRANSLATION_COMPRESSION_RADIX = 14; static const int HAND_CONTROLLER_COMPRESSION_RADIX = 12; static const int SENSOR_TO_WORLD_SCALE_RADIX = 10; static const float AUDIO_LOUDNESS_SCALE = 1024.0f; static const float DEFAULT_AVATAR_DENSITY = 1000.0f; // density of water #define ASSERT(COND) do { if (!(COND)) { abort(); } } while(0) STATIC_SCRIPT_TYPES_INITIALIZER((+[](ScriptManager* manager) { auto scriptEngine = manager->engine().get(); scriptRegisterMetaType<RayToAvatarIntersectionResult, RayToAvatarIntersectionResultToScriptValue, RayToAvatarIntersectionResultFromScriptValue>(scriptEngine); scriptRegisterMetaType<AvatarEntityMap, AvatarEntityMapToScriptValue, AvatarEntityMapFromScriptValue>(scriptEngine); })); size_t AvatarDataPacket::maxFaceTrackerInfoSize(size_t numBlendshapeCoefficients) { return FACE_TRACKER_INFO_SIZE + numBlendshapeCoefficients * sizeof(float); } size_t AvatarDataPacket::maxJointDataSize(size_t numJoints) { const size_t validityBitsSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints totalSize += validityBitsSize; // Orientations mask totalSize += numJoints * sizeof(SixByteQuat); // Orientations totalSize += validityBitsSize; // Translations mask totalSize += sizeof(float); // maxTranslationDimension totalSize += numJoints * sizeof(SixByteTrans); // Translations return totalSize; } size_t AvatarDataPacket::minJointDataSize(size_t numJoints) { const size_t validityBitsSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints totalSize += validityBitsSize; // Orientations mask // assume no valid rotations totalSize += validityBitsSize; // Translations mask totalSize += sizeof(float); // maxTranslationDimension // assume no valid translations return totalSize; } size_t AvatarDataPacket::maxJointDefaultPoseFlagsSize(size_t numJoints) { const size_t bitVectorSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints // one set of bits for rotation and one for translation const size_t NUM_BIT_VECTORS_IN_DEFAULT_POSE_FLAGS_SECTION = 2; totalSize += NUM_BIT_VECTORS_IN_DEFAULT_POSE_FLAGS_SECTION * bitVectorSize; return totalSize; } AvatarData::AvatarData() : SpatiallyNestable(NestableType::Avatar, QUuid()), _handPosition(0.0f), _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), _targetVelocity(0.0f), _density(DEFAULT_AVATAR_DENSITY) { connect(this, &AvatarData::lookAtSnappingChanged, this, &AvatarData::markIdentityDataChanged); } AvatarData::~AvatarData() { delete _headData; } // We cannot have a file-level variable (const or otherwise) in the AvatarInfo if it uses PathUtils, because that references Application, which will not yet initialized. // Thus we have a static class getter, referencing a static class var. QUrl AvatarData::_defaultFullAvatarModelUrl = {}; // In C++, if this initialization were in the AvatarInfo, every file would have it's own copy, even for class vars. const QUrl& AvatarData::defaultFullAvatarModelUrl() { if (_defaultFullAvatarModelUrl.isEmpty()) { _defaultFullAvatarModelUrl = PathUtils::resourcesUrl("/meshes/defaultAvatar_full.fst"); } return _defaultFullAvatarModelUrl; } void AvatarData::setTargetScale(float targetScale) { auto newValue = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); if (_targetScale != newValue) { _targetScale = newValue; _scaleChanged = usecTimestampNow(); _avatarScaleChanged = _scaleChanged; } } float AvatarData::getDomainLimitedScale() const { if (canMeasureEyeHeight()) { const float minScale = getDomainMinScale(); const float maxScale = getDomainMaxScale(); return glm::clamp(_targetScale, minScale, maxScale); } else { // We can't make a good estimate. return _targetScale; } } void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { if (hasScriptedBlendshapes == _headData->getHasScriptedBlendshapes()) { return; } if (!hasScriptedBlendshapes) { // send a forced avatarData update to make sure the script can send neutal blendshapes on unload // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true // before sending the update, or else it won't send the neutal blendshapes to the receiving clients sendAvatarDataPacket(true); } _headData->setHasScriptedBlendshapes(hasScriptedBlendshapes); } bool AvatarData::getHasScriptedBlendshapes() const { return _headData->getHasScriptedBlendshapes(); } void AvatarData::setHasProceduralBlinkFaceMovement(bool value) { _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, value); } bool AvatarData::getHasProceduralBlinkFaceMovement() const { return _headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation); } void AvatarData::setHasProceduralEyeFaceMovement(bool value) { _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, value); } bool AvatarData::getHasProceduralEyeFaceMovement() const { return _headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation); } void AvatarData::setHasAudioEnabledFaceMovement(bool value) { _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, value); } bool AvatarData::getHasAudioEnabledFaceMovement() const { return _headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation); } void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { _domainMinimumHeight = glm::clamp(domainMinimumHeight, MIN_AVATAR_HEIGHT, MAX_AVATAR_HEIGHT); } void AvatarData::setDomainMaximumHeight(float domainMaximumHeight) { _domainMaximumHeight = glm::clamp(domainMaximumHeight, MIN_AVATAR_HEIGHT, MAX_AVATAR_HEIGHT); } float AvatarData::getDomainMinScale() const { float unscaledHeight = getUnscaledHeight(); const float EPSILON = 1.0e-4f; if (unscaledHeight <= EPSILON) { unscaledHeight = DEFAULT_AVATAR_HEIGHT; } return _domainMinimumHeight / unscaledHeight; } float AvatarData::getDomainMaxScale() const { float unscaledHeight = getUnscaledHeight(); const float EPSILON = 1.0e-4f; if (unscaledHeight <= EPSILON) { unscaledHeight = DEFAULT_AVATAR_HEIGHT; } return _domainMaximumHeight / unscaledHeight; } float AvatarData::getUnscaledHeight() const { const float eyeHeight = getUnscaledEyeHeight(); const float ratio = eyeHeight / DEFAULT_AVATAR_HEIGHT; return eyeHeight + ratio * DEFAULT_AVATAR_EYE_TO_TOP_OF_HEAD; } float AvatarData::getHeight() const { const float eyeHeight = getEyeHeight(); const float ratio = eyeHeight / DEFAULT_AVATAR_HEIGHT; return eyeHeight + ratio * DEFAULT_AVATAR_EYE_TO_TOP_OF_HEAD; } glm::vec3 AvatarData::getHandPosition() const { return getWorldOrientation() * _handPosition + getWorldPosition(); } void AvatarData::setHandPosition(const glm::vec3& handPosition) { // store relative to position/orientation _handPosition = glm::inverse(getWorldOrientation()) * (handPosition - getWorldPosition()); } void AvatarData::lazyInitHeadData() const { // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(const_cast<AvatarData*>(this)); } } float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const { auto distance = glm::distance(_globalPosition, viewerPosition); float result = ROTATION_CHANGE_179D; // assume worst if (distance < AVATAR_DISTANCE_LEVEL_1) { result = AVATAR_MIN_ROTATION_DOT; } else if (distance < AVATAR_DISTANCE_LEVEL_2) { result = ROTATION_CHANGE_2D; } else if (distance < AVATAR_DISTANCE_LEVEL_3) { result = ROTATION_CHANGE_4D; } else if (distance < AVATAR_DISTANCE_LEVEL_4) { result = ROTATION_CHANGE_6D; } else if (distance < AVATAR_DISTANCE_LEVEL_5) { result = ROTATION_CHANGE_15D; } return result; } float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const { return AVATAR_MIN_TRANSLATION; // Eventually make this distance sensitive as well } // we want to track outbound data in this case... QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) { auto lastSentTime = _lastToByteArray; _lastToByteArray = usecTimestampNow(); AvatarDataPacket::SendStatus sendStatus; auto avatarByteArray = AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), sendStatus, dropFaceTracking, false, glm::vec3(0), nullptr, 0, &_outboundDataRate); return avatarByteArray; } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData, AvatarDataPacket::SendStatus& sendStatus, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut, int maxDataSize, AvatarDataRate* outboundDataRateOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); bool sendAll = (dataDetail == SendAllData); bool sendMinimum = (dataDetail == MinimumData); bool sendPALMinimum = (dataDetail == PALMinimum); lazyInitHeadData(); ASSERT(maxDataSize == 0 || (size_t)maxDataSize >= AvatarDataPacket::MIN_BULK_PACKET_SIZE); // Leading flags, to indicate how much data is actually included in the packet... AvatarDataPacket::HasFlags wantedFlags = 0; AvatarDataPacket::HasFlags includedFlags = 0; AvatarDataPacket::HasFlags extraReturnedFlags = 0; // For partial joint data. // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { sendStatus.itemFlags = wantedFlags; QByteArray avatarDataByteArray; if (sendStatus.sendUUID) { avatarDataByteArray.append(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID); } avatarDataByteArray.append((char*) &wantedFlags, sizeof wantedFlags); return avatarDataByteArray; } // FIXME - // // BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... // this is an iFrame issue... what to do about that? // // BUG -- Resizing avatar seems to "take too long"... the avatar doesn't redraw at smaller size right away // // TODO consider these additional optimizations in the future // 1) SensorToWorld - should we only send this for avatars with attachments?? - 20 bytes - 7.20 kbps // 2) GUIID for the session change to 2byte index (savings) - 14 bytes - 5.04 kbps // 3) Improve Joints -- currently we use rotational tolerances, but if we had skeleton/bone length data // we could do a better job of determining if the change in joints actually translates to visible // changes at distance. // // Potential savings: // 63 rotations * 6 bytes = 136kbps // 3 translations * 6 bytes = 6.48kbps // QUuid parentID; glm::mat4 leftFarGrabMatrix; glm::mat4 rightFarGrabMatrix; glm::mat4 mouseFarGrabMatrix; if (sendStatus.itemFlags == 0) { // New avatar ... bool hasAvatarGlobalPosition = true; // always include global position bool hasAvatarOrientation = false; bool hasAvatarBoundingBox = false; bool hasAvatarScale = false; bool hasLookAtPosition = false; bool hasAudioLoudness = false; bool hasSensorToWorldMatrix = false; bool hasJointData = false; bool hasJointDefaultPoseFlags = false; bool hasAdditionalFlags = false; // local position, and parent info only apply to avatars that are parented. The local position // and the parent info can change independently though, so we track their "changed since" // separately bool hasParentInfo = false; bool hasAvatarLocalPosition = false; bool hasHandControllers = false; bool hasFaceTrackerInfo = false; if (sendPALMinimum) { hasAudioLoudness = true; } else { hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); hasAvatarLocalPosition = hasParent() && (sendAll || tranlationChangedSince(lastSentTime) || parentInfoChangedSince(lastSentTime)); hasHandControllers = _controllerLeftHandMatrixCache.isValid() || _controllerRightHandMatrixCache.isValid(); hasFaceTrackerInfo = !dropFaceTracking && (getHasScriptedBlendshapes() || _headData->_hasInputDrivenBlendshapes) && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); hasJointData = !sendMinimum; hasJointDefaultPoseFlags = hasJointData; } wantedFlags = (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) | (hasHandControllers ? AvatarDataPacket::PACKET_HAS_HAND_CONTROLLERS : 0) | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0) | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); sendStatus.itemFlags = wantedFlags; sendStatus.rotationsSent = 0; sendStatus.translationsSent = 0; } else { // Continuing avatar ... wantedFlags = sendStatus.itemFlags; if (wantedFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { // Must send joints for grab joints - wantedFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; } } if (wantedFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { bool leftValid; leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); if (!leftValid) { leftFarGrabMatrix = glm::mat4(); } bool rightValid; rightFarGrabMatrix = _farGrabRightMatrixCache.get(rightValid); if (!rightValid) { rightFarGrabMatrix = glm::mat4(); } bool mouseValid; mouseFarGrabMatrix = _farGrabMouseMatrixCache.get(mouseValid); if (!mouseValid) { mouseFarGrabMatrix = glm::mat4(); } if (!(leftValid || rightValid || mouseValid)) { wantedFlags &= ~AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; } } if (wantedFlags & (AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS | AvatarDataPacket::PACKET_HAS_PARENT_INFO)) { parentID = getParentID(); } const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + NUM_BYTES_RFC4122_UUID + AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) + AvatarDataPacket::maxJointDataSize(_jointData.size()) + AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) + AvatarDataPacket::FAR_GRAB_JOINTS_SIZE; if (maxDataSize == 0) { maxDataSize = (int)byteArraySize; } QByteArray avatarDataByteArray((int)byteArraySize, 0); unsigned char* destinationBuffer = reinterpret_cast<unsigned char*>(avatarDataByteArray.data()); const unsigned char* const startPosition = destinationBuffer; const unsigned char* const packetEnd = destinationBuffer + maxDataSize; #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); // If we want an item and there's sufficient space: #define IF_AVATAR_SPACE(flag, space) \ if ((wantedFlags & AvatarDataPacket::flag) \ && (packetEnd - destinationBuffer) >= (ptrdiff_t)(space) \ && (includedFlags |= AvatarDataPacket::flag)) if (sendStatus.sendUUID) { memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); destinationBuffer += NUM_BYTES_RFC4122_UUID; } unsigned char * packetFlagsLocation = destinationBuffer; destinationBuffer += sizeof(wantedFlags); IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_globalPosition); int numBytes = destinationBuffer - startSection; if (outboundDataRateOut) { outboundDataRateOut->globalPositionRate.increment(numBytes); } } IF_AVATAR_SPACE(PACKET_HAS_AVATAR_BOUNDING_BOX, sizeof _globalBoundingBoxDimensions + sizeof _globalBoundingBoxOffset) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_globalBoundingBoxDimensions); AVATAR_MEMCPY(_globalBoundingBoxOffset);