////  DdeFaceTracker.cpp//////  Created by Clement on 8/2/14.//  Copyright 2014 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include <SharedUtil.h> #include <QCoreApplication>#include <QJsonDocument>#include <QJsonArray>#include <QJsonObject>#include <QTimer> #include <GLMHelpers.h>#include <NumericalConstants.h> #include "Application.h"#include "DdeFaceTracker.h"#include "FaceshiftConstants.h"#include "InterfaceLogging.h"#include "Menu.h"  static const QHostAddress DDE_SERVER_ADDR("127.0.0.1");static const quint16 DDE_SERVER_PORT = 64204;static const quint16 DDE_CONTROL_PORT = 64205;#if defined(Q_OS_WIN)static const QString DDE_PROGRAM_PATH = "/dde/dde.exe";#elif defined(Q_OS_MAC)static const QString DDE_PROGRAM_PATH = "/dde.app/Contents/MacOS/dde";#endifstatic const QStringList DDE_ARGUMENTS = QStringList()     << "--udp=" + DDE_SERVER_ADDR.toString() + ":" + QString::number(DDE_SERVER_PORT)     << "--receiver=" + QString::number(DDE_CONTROL_PORT)    << "--facedet_interval=500"  // ms    << "--headless"; static const int NUM_EXPRESSIONS = 46;static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSIONS) * sizeof(float) + sizeof(int);static const int MAX_NAME_SIZE = 31; // There's almost but not quite a 1-1 correspondence between DDE's 46 and Faceshift 1.3's 48 packets.// The best guess at mapping is to:// - Swap L and R values// - Skip two Faceshift values: JawChew (22) and LipsLowerDown (37)static const int DDE_TO_FACESHIFT_MAPPING[] = {    1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14,    16,    18, 17,    19,    23,    21,    // Skip JawChew    20,    25, 24, 27, 26, 29, 28, 31, 30, 33, 32,    34, 35, 36,    // Skip LipsLowerDown    38, 39, 40, 41, 42, 43, 44, 45,    47, 46}; // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much // less than this.static const float DDE_COEFFICIENT_SCALES[] = {    1.0f, // EyeBlink_L    1.0f, // EyeBlink_R    1.0f, // EyeSquint_L    1.0f, // EyeSquint_R    1.0f, // EyeDown_L    1.0f, // EyeDown_R    1.0f, // EyeIn_L    1.0f, // EyeIn_R    1.0f, // EyeOpen_L    1.0f, // EyeOpen_R    1.0f, // EyeOut_L    1.0f, // EyeOut_R    1.0f, // EyeUp_L    1.0f, // EyeUp_R    3.0f, // BrowsD_L    3.0f, // BrowsD_R    3.0f, // BrowsU_C    3.0f, // BrowsU_L    3.0f, // BrowsU_R    1.0f, // JawFwd    2.0f, // JawLeft    1.8f, // JawOpen    1.0f, // JawChew    2.0f, // JawRight    1.5f, // MouthLeft    1.5f, // MouthRight    1.5f, // MouthFrown_L    1.5f, // MouthFrown_R    2.5f, // MouthSmile_L    2.5f, // MouthSmile_R    1.0f, // MouthDimple_L    1.0f, // MouthDimple_R    1.0f, // LipsStretch_L    1.0f, // LipsStretch_R    1.0f, // LipsUpperClose    1.0f, // LipsLowerClose    1.0f, // LipsUpperUp    1.0f, // LipsLowerDown    1.0f, // LipsUpperOpen    1.0f, // LipsLowerOpen    1.5f, // LipsFunnel    2.5f, // LipsPucker    1.5f, // ChinLowerRaise    1.5f, // ChinUpperRaise    1.0f, // Sneer    3.0f, // Puff    1.0f, // CheekSquint_L    1.0f  // CheekSquint_R}; struct DDEPacket {    //roughly in mm    float focal_length[1];    float translation[3];        //quaternion    float rotation[4];        // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much     // less than this.    float expressions[NUM_EXPRESSIONS];        //avatar id selected on the UI    int avatar_id;        //client name, arbitrary length    char name[MAX_NAME_SIZE + 1];}; static const float STARTING_DDE_MESSAGE_TIME = 0.033f;static const float DEFAULT_DDE_EYE_CLOSING_THRESHOLD = 0.8f;static const int CALIBRATION_SAMPLES = 150; DdeFaceTracker::DdeFaceTracker() :    DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT){ } DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort) :    _ddeProcess(NULL),    _ddeStopping(false),    _host(host),    _serverPort(serverPort),    _controlPort(controlPort),    _lastReceiveTimestamp(0),    _reset(false),    _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes    _rightBlinkIndex(1),    _leftEyeDownIndex(4),    _rightEyeDownIndex(5),    _leftEyeInIndex(6),    _rightEyeInIndex(7),    _leftEyeOpenIndex(8),    _rightEyeOpenIndex(9),    _browDownLeftIndex(14),    _browDownRightIndex(15),    _browUpCenterIndex(16),    _browUpLeftIndex(17),    _browUpRightIndex(18),    _mouthSmileLeftIndex(28),    _mouthSmileRightIndex(29),    _jawOpenIndex(21),    _lastMessageReceived(0),    _averageMessageTime(STARTING_DDE_MESSAGE_TIME),    _lastHeadTranslation(glm::vec3(0.0f)),    _filteredHeadTranslation(glm::vec3(0.0f)),    _lastBrowUp(0.0f),    _filteredBrowUp(0.0f),    _eyePitch(0.0f),    _eyeYaw(0.0f),    _lastEyePitch(0.0f),    _lastEyeYaw(0.0f),    _filteredEyePitch(0.0f),    _filteredEyeYaw(0.0f),    _longTermAverageEyePitch(0.0f),    _longTermAverageEyeYaw(0.0f),    _lastEyeBlinks(),    _filteredEyeBlinks(),    _lastEyeCoefficients(),    _eyeClosingThreshold("ddeEyeClosingThreshold", DEFAULT_DDE_EYE_CLOSING_THRESHOLD),    _isCalibrating(false),    _calibrationCount(0),    _calibrationValues(),    _calibrationBillboard(NULL),    _calibrationBillboardID(0),    _calibrationMessage(QString()),    _isCalibrated(false){    _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES);    _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES);    _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES);    _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES);     _eyeStates[0] = EYE_UNCONTROLLED;    _eyeStates[1] = EYE_UNCONTROLLED;     connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));    connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));    connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)),         SLOT(socketStateChanged(QAbstractSocket::SocketState)));} DdeFaceTracker::~DdeFaceTracker() {    setEnabled(false);     if (_isCalibrating) {        cancelCalibration();    }} void DdeFaceTracker::init() {    FaceTracker::init();    setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) && !_isMuted);    Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!_isMuted);} void DdeFaceTracker::setEnabled(bool enabled) {    if (!_isInitialized) {        // Don't enable until have explicitly initialized        return;    }#ifdef HAVE_DDE     if (_isCalibrating) {        cancelCalibration();    }     // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket.    _udpSocket.close();     // Terminate any existing DDE process, perhaps left running after an Interface crash.    // Do this even if !enabled in case user reset their settings after crash.    const char* DDE_EXIT_COMMAND = "exit";    _udpSocket.bind(_host, _serverPort);    _udpSocket.writeDatagram(DDE_EXIT_COMMAND, DDE_SERVER_ADDR, _controlPort);     if (enabled && !_ddeProcess) {        _ddeStopping = false;        qCDebug(interfaceapp) << "DDE Face Tracker: Starting";        _ddeProcess = new QProcess(qApp);        connect(_ddeProcess, SIGNAL(finished(int, QProcess::ExitStatus)), SLOT(processFinished(int, QProcess::ExitStatus)));        _ddeProcess->start(QCoreApplication::applicationDirPath() + DDE_PROGRAM_PATH, DDE_ARGUMENTS);    }        if (!enabled && _ddeProcess) {        _ddeStopping = true;        qCDebug(interfaceapp) << "DDE Face Tracker: Stopping";    }#endif} void DdeFaceTracker::processFinished(int exitCode, QProcess::ExitStatus exitStatus) {    if (_ddeProcess) {        if (_ddeStopping) {            qCDebug(interfaceapp) << "DDE Face Tracker: Stopped";         } else {            qCWarning(interfaceapp) << "DDE Face Tracker: Stopped unexpectedly";            Menu::getInstance()->setIsOptionChecked(MenuOption::NoFaceTracking, true);        }        _udpSocket.close();        delete _ddeProcess;        _ddeProcess = NULL;    }} void DdeFaceTracker::reset() {    if (_udpSocket.state() == QAbstractSocket::BoundState) {        _reset = true;         qCDebug(interfaceapp) << "DDE Face Tracker: Reset";         const char* DDE_RESET_COMMAND = "reset";        _udpSocket.writeDatagram(DDE_RESET_COMMAND, DDE_SERVER_ADDR, _controlPort);         FaceTracker::reset();         _reset = true;    }} void DdeFaceTracker::update(float deltaTime) {    if (!isActive()) {        return;    }    FaceTracker::update(deltaTime);     glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation));    _estimatedEyePitch = _eyePitch - headEulers.x;    _estimatedEyeYaw = _eyeYaw - headEulers.y;} bool DdeFaceTracker::isActive() const {    return (_ddeProcess != NULL);} bool DdeFaceTracker::isTracking() const {    static const quint64 ACTIVE_TIMEOUT_USECS = 3000000; //3 secs    return (usecTimestampNow() - _lastReceiveTimestamp < ACTIVE_TIMEOUT_USECS);} //private slots and methodsvoid DdeFaceTracker::socketErrorOccurred(QAbstractSocket::SocketError socketError) {    qCWarning(interfaceapp) << "DDE Face Tracker: Socket error: " << _udpSocket.errorString();} void DdeFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState) {    QString state;    switch(socketState) {        case QAbstractSocket::BoundState:            state = "Bound";            break;        case QAbstractSocket::ClosingState:            state = "Closing";            break;        case QAbstractSocket::ConnectedState:            state = "Connected";            break;        case QAbstractSocket::ConnectingState:            state = "Connecting";            break;        case QAbstractSocket::HostLookupState:            state = "Host Lookup";            break;        case QAbstractSocket::ListeningState:            state = "Listening";            break;        case QAbstractSocket::UnconnectedState:            state = "Unconnected";            break;    }    qCDebug(interfaceapp) << "DDE Face Tracker: Socket: " << state;} void DdeFaceTracker::readPendingDatagrams() {    QByteArray buffer;    while (_udpSocket.hasPendingDatagrams()) {        buffer.resize(_udpSocket.pendingDatagramSize());        _udpSocket.readDatagram(buffer.data(), buffer.size());    }    decodePacket(buffer);} float DdeFaceTracker::getBlendshapeCoefficient(int index) const {    return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f;} void DdeFaceTracker::decodePacket(const QByteArray& buffer) {    _lastReceiveTimestamp = usecTimestampNow();     if (buffer.size() > MIN_PACKET_SIZE) {        if (!_isCalibrated) {            calibrate();        }         bool isFiltering = Menu::getInstance()->isOptionChecked(MenuOption::VelocityFilter);         DDEPacket packet;        int bytesToCopy = glm::min((int)sizeof(packet), buffer.size());        memset(&packet.name, '\n', MAX_NAME_SIZE + 1);        memcpy(&packet, buffer.data(), bytesToCopy);                glm::vec3 translation;        memcpy(&translation, packet.translation, sizeof(packet.translation));        glm::quat rotation;        memcpy(&rotation, &packet.rotation, sizeof(packet.rotation));        if (_reset || (_lastMessageReceived == 0)) {            memcpy(&_referenceTranslation, &translation, sizeof(glm::vec3));            memcpy(&_referenceRotation, &rotation, sizeof(glm::quat));            _reset = false;        }         // Compute relative translation        float LEAN_DAMPING_FACTOR = 75.0f;        translation -= _referenceTranslation;        translation /= LEAN_DAMPING_FACTOR;        translation.x *= -1;        if (isFiltering) {            glm::vec3 linearVelocity = (translation - _lastHeadTranslation) / _averageMessageTime;            const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f;            float velocityFilter = glm::clamp(1.0f - glm::length(linearVelocity) *                LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);            _filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * translation;            _lastHeadTranslation = translation;            _headTranslation = _filteredHeadTranslation;        } else {            _headTranslation = translation;        }         // Compute relative rotation        rotation = glm::inverse(_referenceRotation) * rotation;        if (isFiltering) {            glm::quat r = glm::normalize(rotation * glm::inverse(_headRotation));            float theta = 2 * acos(r.w);            glm::vec3 angularVelocity;            if (theta > EPSILON) {                float rMag = glm::length(glm::vec3(r.x, r.y, r.z));                angularVelocity = theta / _averageMessageTime * glm::vec3(r.x, r.y, r.z) / rMag;            } else {                angularVelocity = glm::vec3(0, 0, 0);            }            const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f;            _headRotation = safeMix(_headRotation, rotation, glm::clamp(glm::length(angularVelocity) *                ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f));        } else {            _headRotation = rotation;        }         // Translate DDE coefficients to Faceshift compatible coefficients        for (int i = 0; i < NUM_EXPRESSIONS; i++) {            _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i];        }         // Calibration        if (_isCalibrating) {            addCalibrationDatum();        }        for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {            _coefficients[i] -= _coefficientAverages[i];        }         // Use BrowsU_C to control both brows' up and down        float browUp = _coefficients[_browUpCenterIndex];        if (isFiltering) {            const float BROW_VELOCITY_FILTER_STRENGTH = 0.5f;            float velocity = fabsf(browUp - _lastBrowUp) / _averageMessageTime;            float velocityFilter = glm::clamp(velocity * BROW_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);            _filteredBrowUp = velocityFilter * browUp + (1.0f - velocityFilter) * _filteredBrowUp;            _lastBrowUp = browUp;            browUp = _filteredBrowUp;            _coefficients[_browUpCenterIndex] = browUp;        }        _coefficients[_browUpLeftIndex] = browUp;        _coefficients[_browUpRightIndex] = browUp;        _coefficients[_browDownLeftIndex] = -browUp;        _coefficients[_browDownRightIndex] = -browUp;         // Offset jaw open coefficient        static const float JAW_OPEN_THRESHOLD = 0.1f;        _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD;         // Offset smile coefficients        static const float SMILE_THRESHOLD = 0.5f;        _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD;        _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD;         // Eye pitch and yaw        // EyeDown coefficients work better over both +ve and -ve values than EyeUp values.        // EyeIn coefficients work better over both +ve and -ve values than EyeOut values.        // Pitch and yaw values are relative to the screen.        const float EYE_PITCH_SCALE = -1500.0f;  // Sign, scale, and average to be similar to Faceshift values.        _eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]);        const float EYE_YAW_SCALE = 2000.0f;  // Scale and average to be similar to Faceshift values.        _eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]);        if (isFiltering) {            const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f;            float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime;            float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);            _filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch;            _lastEyePitch = _eyePitch;            _eyePitch = _filteredEyePitch;            float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime;            float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);            _filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw;            _lastEyeYaw = _eyeYaw;            _eyeYaw = _filteredEyeYaw;        }         // Velocity filter EyeBlink values        const float DDE_EYEBLINK_SCALE = 3.0f;        float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex],                              DDE_EYEBLINK_SCALE * _coefficients[_rightBlinkIndex] };        if (isFiltering) {            const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f;            for (int i = 0; i < 2; i++) {                float velocity = fabsf(eyeBlinks[i] - _lastEyeBlinks[i]) / _averageMessageTime;                float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);                _filteredEyeBlinks[i] = velocityFilter * eyeBlinks[i] + (1.0f - velocityFilter) * _filteredEyeBlinks[i];                _lastEyeBlinks[i] = eyeBlinks[i];            }        }         // Finesse EyeBlink values        float eyeCoefficients[2];        if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) {            if (_eyeStates[0] == EYE_UNCONTROLLED) {                _eyeStates[0] = EYE_OPEN;                _eyeStates[1] = EYE_OPEN;            }

Lines 1–500 of 694. Load more