// do better detection in the offscreen UI of what has focus offscreenUi->setNavigationFocused(false); auto engine = rootContext->engine(); connect(engine, &QQmlEngine::quit, [] { qApp->quit(); }); setupPreferences(); // For some reason there is already an "Application" object in the QML context, // though I can't find it. Hence, "ApplicationInterface" rootContext->setContextProperty("SnapshotUploader", new SnapshotUploader()); rootContext->setContextProperty("ApplicationInterface", this); rootContext->setContextProperty("Audio", &AudioScriptingInterface::getInstance()); rootContext->setContextProperty("Controller", DependencyManager::get<controller::ScriptingInterface>().data()); rootContext->setContextProperty("Entities", DependencyManager::get<EntityScriptingInterface>().data()); rootContext->setContextProperty("MyAvatar", getMyAvatar()); rootContext->setContextProperty("Messages", DependencyManager::get<MessagesClient>().data()); rootContext->setContextProperty("Recording", DependencyManager::get<RecordingScriptingInterface>().data()); rootContext->setContextProperty("Preferences", DependencyManager::get<Preferences>().data()); rootContext->setContextProperty("AddressManager", DependencyManager::get<AddressManager>().data()); rootContext->setContextProperty("FrameTimings", &_frameTimingsScriptingInterface); rootContext->setContextProperty("Rates", new RatesScriptingInterface(this)); rootContext->setContextProperty("TREE_SCALE", TREE_SCALE); rootContext->setContextProperty("Quat", new Quat()); rootContext->setContextProperty("Vec3", new Vec3()); rootContext->setContextProperty("Uuid", new ScriptUUID()); rootContext->setContextProperty("Assets", new AssetMappingsScriptingInterface()); rootContext->setContextProperty("AvatarList", DependencyManager::get<AvatarManager>().data()); rootContext->setContextProperty("Camera", &_myCamera); #if defined(Q_OS_MAC) || defined(Q_OS_WIN) rootContext->setContextProperty("SpeechRecognizer", DependencyManager::get<SpeechRecognizer>().data());#endif rootContext->setContextProperty("Overlays", &_overlays); rootContext->setContextProperty("Window", DependencyManager::get<WindowScriptingInterface>().data()); rootContext->setContextProperty("MenuInterface", MenuScriptingInterface::getInstance()); rootContext->setContextProperty("Stats", Stats::getInstance()); rootContext->setContextProperty("Settings", SettingsScriptingInterface::getInstance()); rootContext->setContextProperty("ScriptDiscoveryService", DependencyManager::get<ScriptEngines>().data()); rootContext->setContextProperty("AudioDevice", AudioDeviceScriptingInterface::getInstance()); // Caches rootContext->setContextProperty("AnimationCache", DependencyManager::get<AnimationCache>().data()); rootContext->setContextProperty("TextureCache", DependencyManager::get<TextureCache>().data()); rootContext->setContextProperty("ModelCache", DependencyManager::get<ModelCache>().data()); rootContext->setContextProperty("SoundCache", DependencyManager::get<SoundCache>().data()); rootContext->setContextProperty("Account", AccountScriptingInterface::getInstance()); rootContext->setContextProperty("DialogsManager", _dialogsManagerScriptingInterface); rootContext->setContextProperty("GlobalServices", GlobalServicesScriptingInterface::getInstance()); rootContext->setContextProperty("FaceTracker", DependencyManager::get<DdeFaceTracker>().data()); rootContext->setContextProperty("AvatarManager", DependencyManager::get<AvatarManager>().data()); rootContext->setContextProperty("UndoStack", &_undoStackScriptingInterface); rootContext->setContextProperty("LODManager", DependencyManager::get<LODManager>().data()); rootContext->setContextProperty("Paths", DependencyManager::get<PathUtils>().data()); rootContext->setContextProperty("HMD", DependencyManager::get<HMDScriptingInterface>().data()); rootContext->setContextProperty("Scene", DependencyManager::get<SceneScriptingInterface>().data()); rootContext->setContextProperty("Render", _renderEngine->getConfiguration().get()); rootContext->setContextProperty("Reticle", getApplicationCompositor().getReticleInterface()); rootContext->setContextProperty("ApplicationCompositor", &getApplicationCompositor()); _glWidget->installEventFilter(offscreenUi.data()); offscreenUi->setMouseTranslator([=](const QPointF& pt) { QPointF result = pt; auto displayPlugin = getActiveDisplayPlugin(); if (displayPlugin->isHmd()) { getApplicationCompositor().handleRealMouseMoveEvent(false); auto resultVec = getApplicationCompositor().getReticlePosition(); result = QPointF(resultVec.x, resultVec.y); } return result.toPoint(); }); offscreenUi->resume(); connect(_window, &MainWindow::windowGeometryChanged, [this](const QRect& r){ resizeGL(); }); // This will set up the input plugins UI _activeInputPlugins.clear(); foreach(auto inputPlugin, PluginManager::getInstance()->getInputPlugins()) { if (KeyboardMouseDevice::NAME == inputPlugin->getName()) { _keyboardMouseDevice = std::dynamic_pointer_cast<KeyboardMouseDevice>(inputPlugin); } } _window->setMenuBar(new Menu()); auto compositorHelper = DependencyManager::get<CompositorHelper>(); connect(compositorHelper.data(), &CompositorHelper::allowMouseCaptureChanged, [=] { if (isHMDMode()) { showCursor(compositorHelper->getAllowMouseCapture() ? Qt::BlankCursor : Qt::ArrowCursor); } });} void Application::paintGL() { // Some plugins process message events, allowing paintGL to be called reentrantly. if (_inPaint || _aboutToQuit) { return; } _inPaint = true; Finally clearFlag([this] { _inPaint = false; }); _frameCount++; _frameCounter.increment(); auto lastPaintBegin = usecTimestampNow(); PROFILE_RANGE_EX(__FUNCTION__, 0xff0000ff, (uint64_t)_frameCount); PerformanceTimer perfTimer("paintGL"); if (nullptr == _displayPlugin) { return; } auto displayPlugin = getActiveDisplayPlugin(); // FIXME not needed anymore? _offscreenContext->makeCurrent(); // If a display plugin loses it's underlying support, it // needs to be able to signal us to not use it if (!displayPlugin->beginFrameRender(_frameCount)) { _inPaint = false; updateDisplayMode(); return; } // update the avatar with a fresh HMD pose getMyAvatar()->updateFromHMDSensorMatrix(getHMDSensorPose()); auto lodManager = DependencyManager::get<LODManager>(); { QMutexLocker viewLocker(&_viewMutex); _viewFrustum.calculate(); } RenderArgs renderArgs(_gpuContext, getEntities(), lodManager->getOctreeSizeScale(), lodManager->getBoundaryLevelAdjust(), RenderArgs::DEFAULT_RENDER_MODE, RenderArgs::MONO, RenderArgs::RENDER_DEBUG_NONE); { QMutexLocker viewLocker(&_viewMutex); renderArgs.setViewFrustum(_viewFrustum); } PerformanceWarning::setSuppressShortTimings(Menu::getInstance()->isOptionChecked(MenuOption::SuppressShortTimings)); bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); PerformanceWarning warn(showWarnings, "Application::paintGL()"); resizeGL(); // Before anything else, let's sync up the gpuContext with the true glcontext used in case anything happened { PerformanceTimer perfTimer("syncCache"); renderArgs._context->syncCache(); } auto inputs = AvatarInputs::getInstance(); if (inputs->mirrorVisible()) { PerformanceTimer perfTimer("Mirror"); auto primaryFbo = DependencyManager::get<FramebufferCache>()->getPrimaryFramebuffer(); renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; renderArgs._blitFramebuffer = DependencyManager::get<FramebufferCache>()->getSelfieFramebuffer(); _mirrorViewRect.moveTo(inputs->x(), inputs->y()); renderRearViewMirror(&renderArgs, _mirrorViewRect, inputs->mirrorZoomed()); renderArgs._blitFramebuffer.reset(); renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; } { PerformanceTimer perfTimer("renderOverlay"); // NOTE: There is no batch associated with this renderArgs // the ApplicationOverlay class assumes it's viewport is setup to be the device size QSize size = getDeviceSize(); renderArgs._viewport = glm::ivec4(0, 0, size.width(), size.height()); _applicationOverlay.renderOverlay(&renderArgs); auto overlayTexture = _applicationOverlay.acquireOverlay(); if (overlayTexture) { displayPlugin->submitOverlayTexture(overlayTexture); } } glm::vec3 boomOffset; { PerformanceTimer perfTimer("CameraUpdates"); auto myAvatar = getMyAvatar(); boomOffset = myAvatar->getScale() * myAvatar->getBoomLength() * -IDENTITY_FRONT; if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON || _myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { Menu::getInstance()->setIsOptionChecked(MenuOption::FirstPerson, myAvatar->getBoomLength() <= MyAvatar::ZOOM_MIN); Menu::getInstance()->setIsOptionChecked(MenuOption::ThirdPerson, !(myAvatar->getBoomLength() <= MyAvatar::ZOOM_MIN)); cameraMenuChanged(); } // The render mode is default or mirror if the camera is in mirror mode, assigned further below renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; // Always use the default eye position, not the actual head eye position. // Using the latter will cause the camera to wobble with idle animations, // or with changes from the face tracker if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { if (isHMDMode()) { mat4 camMat = myAvatar->getSensorToWorldMatrix() * myAvatar->getHMDSensorMatrix(); _myCamera.setPosition(extractTranslation(camMat)); _myCamera.setOrientation(glm::quat_cast(camMat)); } else { _myCamera.setPosition(myAvatar->getDefaultEyePosition()); _myCamera.setOrientation(myAvatar->getHead()->getCameraOrientation()); } } else if (_myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { if (isHMDMode()) { auto hmdWorldMat = myAvatar->getSensorToWorldMatrix() * myAvatar->getHMDSensorMatrix(); _myCamera.setOrientation(glm::normalize(glm::quat_cast(hmdWorldMat))); _myCamera.setPosition(extractTranslation(hmdWorldMat) + myAvatar->getOrientation() * boomOffset); } else { _myCamera.setOrientation(myAvatar->getHead()->getOrientation()); if (Menu::getInstance()->isOptionChecked(MenuOption::CenterPlayerInView)) { _myCamera.setPosition(myAvatar->getDefaultEyePosition() + _myCamera.getOrientation() * boomOffset); } else { _myCamera.setPosition(myAvatar->getDefaultEyePosition() + myAvatar->getOrientation() * boomOffset); } } } else if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { if (isHMDMode()) { auto mirrorBodyOrientation = myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f)); glm::quat hmdRotation = extractRotation(myAvatar->getHMDSensorMatrix()); // Mirror HMD yaw and roll glm::vec3 mirrorHmdEulers = glm::eulerAngles(hmdRotation); mirrorHmdEulers.y = -mirrorHmdEulers.y; mirrorHmdEulers.z = -mirrorHmdEulers.z; glm::quat mirrorHmdRotation = glm::quat(mirrorHmdEulers); glm::quat worldMirrorRotation = mirrorBodyOrientation * mirrorHmdRotation; _myCamera.setOrientation(worldMirrorRotation); glm::vec3 hmdOffset = extractTranslation(myAvatar->getHMDSensorMatrix()); // Mirror HMD lateral offsets hmdOffset.x = -hmdOffset.x; _myCamera.setPosition(myAvatar->getDefaultEyePosition() + glm::vec3(0, _raiseMirror * myAvatar->getUniformScale(), 0) + mirrorBodyOrientation * glm::vec3(0.0f, 0.0f, 1.0f) * MIRROR_FULLSCREEN_DISTANCE * _scaleMirror + mirrorBodyOrientation * hmdOffset); } else { _myCamera.setOrientation(myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f))); _myCamera.setPosition(myAvatar->getDefaultEyePosition() + glm::vec3(0, _raiseMirror * myAvatar->getUniformScale(), 0) + (myAvatar->getOrientation() * glm::quat(glm::vec3(0.0f, _rotateMirror, 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f) * MIRROR_FULLSCREEN_DISTANCE * _scaleMirror); } renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; } else if (_myCamera.getMode() == CAMERA_MODE_ENTITY) { EntityItemPointer cameraEntity = _myCamera.getCameraEntityPointer(); if (cameraEntity != nullptr) { if (isHMDMode()) { glm::quat hmdRotation = extractRotation(myAvatar->getHMDSensorMatrix()); _myCamera.setOrientation(cameraEntity->getRotation() * hmdRotation); glm::vec3 hmdOffset = extractTranslation(myAvatar->getHMDSensorMatrix()); _myCamera.setPosition(cameraEntity->getPosition() + (hmdRotation * hmdOffset)); } else { _myCamera.setOrientation(cameraEntity->getRotation()); _myCamera.setPosition(cameraEntity->getPosition()); } } } // Update camera position if (!isHMDMode()) { _myCamera.update(1.0f / _frameCounter.rate()); } } getApplicationCompositor().setFrameInfo(_frameCount, _myCamera.getTransform()); // Primary rendering pass auto framebufferCache = DependencyManager::get<FramebufferCache>(); const QSize size = framebufferCache->getFrameBufferSize(); // Final framebuffer that will be handled to the display-plugin auto finalFramebuffer = framebufferCache->getFramebuffer(); { PROFILE_RANGE(__FUNCTION__ "/mainRender"); PerformanceTimer perfTimer("mainRender"); renderArgs._boomOffset = boomOffset; // Viewport is assigned to the size of the framebuffer renderArgs._viewport = ivec4(0, 0, size.width(), size.height()); if (displayPlugin->isStereo()) { // Stereo modes will typically have a larger projection matrix overall, // so we ask for the 'mono' projection matrix, which for stereo and HMD // plugins will imply the combined projection for both eyes. // // This is properly implemented for the Oculus plugins, but for OpenVR // and Stereo displays I'm not sure how to get / calculate it, so we're // just relying on the left FOV in each case and hoping that the // overall culling margin of error doesn't cause popping in the // right eye. There are FIXMEs in the relevant plugins _myCamera.setProjection(displayPlugin->getCullingProjection(_myCamera.getProjection())); renderArgs._context->enableStereo(true); mat4 eyeOffsets[2]; mat4 eyeProjections[2]; auto baseProjection = renderArgs.getViewFrustum().getProjection(); auto hmdInterface = DependencyManager::get<HMDScriptingInterface>(); float IPDScale = hmdInterface->getIPDScale(); mat4 headPose = displayPlugin->getHeadPose(); // FIXME we probably don't need to set the projection matrix every frame, // only when the display plugin changes (or in non-HMD modes when the user // changes the FOV manually, which right now I don't think they can. for_each_eye([&](Eye eye) { // For providing the stereo eye views, the HMD head pose has already been // applied to the avatar, so we need to get the difference between the head // pose applied to the avatar and the per eye pose, and use THAT as // the per-eye stereo matrix adjustment. mat4 eyeToHead = displayPlugin->getEyeToHeadTransform(eye); // Grab the translation vec3 eyeOffset = glm::vec3(eyeToHead[3]); // Apply IPD scaling mat4 eyeOffsetTransform = glm::translate(mat4(), eyeOffset * -1.0f * IPDScale); eyeOffsets[eye] = eyeOffsetTransform; // Tell the plugin what pose we're using to render. In this case we're just using the // unmodified head pose because the only plugin that cares (the Oculus plugin) uses it // for rotational timewarp. If we move to support positonal timewarp, we need to // ensure this contains the full pose composed with the eye offsets. displayPlugin->setEyeRenderPose(_frameCount, eye, headPose * glm::inverse(eyeOffsetTransform)); eyeProjections[eye] = displayPlugin->getEyeProjection(eye, baseProjection); }); renderArgs._context->setStereoProjections(eyeProjections); renderArgs._context->setStereoViews(eyeOffsets); } renderArgs._blitFramebuffer = finalFramebuffer; displaySide(&renderArgs, _myCamera); renderArgs._blitFramebuffer.reset(); renderArgs._context->enableStereo(false); } // deliver final composited scene to the display plugin { PROFILE_RANGE(__FUNCTION__ "/pluginOutput"); PerformanceTimer perfTimer("pluginOutput"); auto finalTexture = finalFramebuffer->getRenderBuffer(0); Q_ASSERT(!_lockedFramebufferMap.contains(finalTexture)); _lockedFramebufferMap[finalTexture] = finalFramebuffer; Q_ASSERT(isCurrentContext(_offscreenContext->getContext())); { PROFILE_RANGE(__FUNCTION__ "/pluginSubmitScene"); PerformanceTimer perfTimer("pluginSubmitScene"); displayPlugin->submitSceneTexture(_frameCount, finalTexture); } Q_ASSERT(isCurrentContext(_offscreenContext->getContext())); } { Stats::getInstance()->setRenderDetails(renderArgs._details); // Reset the gpu::Context Stages // Back to the default framebuffer; gpu::doInBatch(renderArgs._context, [&](gpu::Batch& batch) { batch.resetStages(); }); } uint64_t lastPaintDuration = usecTimestampNow() - lastPaintBegin; _frameTimingsScriptingInterface.addValue(lastPaintDuration);} void Application::runTests() { runTimingTests(); runUnitTests();} void Application::audioMuteToggled() const { QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteAudio); Q_CHECK_PTR(muteAction); muteAction->setChecked(DependencyManager::get<AudioClient>()->isMuted());} void Application::faceTrackerMuteToggled() { QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteFaceTracking); Q_CHECK_PTR(muteAction); bool isMuted = getSelectedFaceTracker()->isMuted(); muteAction->setChecked(isMuted); getSelectedFaceTracker()->setEnabled(!isMuted); Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!isMuted);} void Application::setFieldOfView(float fov) { if (fov != _fieldOfView.get()) { _fieldOfView.set(fov); resizeGL(); }} void Application::aboutApp() { InfoView::show(INFO_HELP_PATH);} void Application::showHelp() { InfoView::show(INFO_EDIT_ENTITIES_PATH);} void Application::resizeEvent(QResizeEvent* event) { resizeGL();} void Application::resizeGL() { PROFILE_RANGE(__FUNCTION__); if (nullptr == _displayPlugin) { return; } auto displayPlugin = getActiveDisplayPlugin(); // Set the desired FBO texture size. If it hasn't changed, this does nothing. // Otherwise, it must rebuild the FBOs uvec2 framebufferSize = displayPlugin->getRecommendedRenderSize(); uvec2 renderSize = uvec2(vec2(framebufferSize) * getRenderResolutionScale()); if (_renderResolution != renderSize) { _renderResolution = renderSize; DependencyManager::get<FramebufferCache>()->setFrameBufferSize(fromGlm(renderSize)); } // FIXME the aspect ratio for stereo displays is incorrect based on this. float aspectRatio = displayPlugin->getRecommendedAspectRatio(); _myCamera.setProjection(glm::perspective(glm::radians(_fieldOfView.get()), aspectRatio, DEFAULT_NEAR_CLIP, DEFAULT_FAR_CLIP)); // Possible change in aspect ratio { QMutexLocker viewLocker(&_viewMutex); loadViewFrustum(_myCamera, _viewFrustum); } auto offscreenUi = DependencyManager::get<OffscreenUi>(); auto uiSize = displayPlugin->getRecommendedUiSize(); // Bit of a hack since there's no device pixel ratio change event I can find. static qreal lastDevicePixelRatio = 0; qreal devicePixelRatio = _window->devicePixelRatio(); if (offscreenUi->size() != fromGlm(uiSize) || devicePixelRatio != lastDevicePixelRatio) { qDebug() << "Device pixel ratio changed, triggering resize"; offscreenUi->resize(fromGlm(uiSize), true); _offscreenContext->makeCurrent(); lastDevicePixelRatio = devicePixelRatio; }} bool Application::importSVOFromURL(const QString& urlString) { emit svoImportRequested(urlString); return true;} bool Application::event(QEvent* event) { if (!Menu::getInstance()) { return false; } // Presentation/painting logic // TODO: Decouple presentation and painting loops static bool isPaintingThrottled = false; if ((int)event->type() == (int)Present) { if (isPaintingThrottled) { // If painting (triggered by presentation) is hogging the main thread, // repost as low priority to avoid hanging the GUI. // This has the effect of allowing presentation to exceed the paint budget by X times and // only dropping every (1/X) frames, instead of every ceil(X) frames // (e.g. at a 60FPS target, painting for 17us would fall to 58.82FPS instead of 30FPS). removePostedEvents(this, Present); postEvent(this, new QEvent(static_cast<QEvent::Type>(Present)), Qt::LowEventPriority); isPaintingThrottled = false; return true; } float nsecsElapsed = (float)_lastTimeUpdated.nsecsElapsed(); if (shouldPaint(nsecsElapsed)) { _lastTimeUpdated.start(); idle(nsecsElapsed); postEvent(this, new QEvent(static_cast<QEvent::Type>(Paint)), Qt::HighEventPriority); } isPaintingThrottled = true; return true;Lines 1501–2000 of 5476. Previous lines Load more
// do better detection in the offscreen UI of what has focus offscreenUi->setNavigationFocused(false); auto engine = rootContext->engine(); connect(engine, &QQmlEngine::quit, [] { qApp->quit(); }); setupPreferences(); // For some reason there is already an "Application" object in the QML context, // though I can't find it. Hence, "ApplicationInterface" rootContext->setContextProperty("SnapshotUploader", new SnapshotUploader()); rootContext->setContextProperty("ApplicationInterface", this); rootContext->setContextProperty("Audio", &AudioScriptingInterface::getInstance()); rootContext->setContextProperty("Controller", DependencyManager::get<controller::ScriptingInterface>().data()); rootContext->setContextProperty("Entities", DependencyManager::get<EntityScriptingInterface>().data()); rootContext->setContextProperty("MyAvatar", getMyAvatar()); rootContext->setContextProperty("Messages", DependencyManager::get<MessagesClient>().data()); rootContext->setContextProperty("Recording", DependencyManager::get<RecordingScriptingInterface>().data()); rootContext->setContextProperty("Preferences", DependencyManager::get<Preferences>().data()); rootContext->setContextProperty("AddressManager", DependencyManager::get<AddressManager>().data()); rootContext->setContextProperty("FrameTimings", &_frameTimingsScriptingInterface); rootContext->setContextProperty("Rates", new RatesScriptingInterface(this)); rootContext->setContextProperty("TREE_SCALE", TREE_SCALE); rootContext->setContextProperty("Quat", new Quat()); rootContext->setContextProperty("Vec3", new Vec3()); rootContext->setContextProperty("Uuid", new ScriptUUID()); rootContext->setContextProperty("Assets", new AssetMappingsScriptingInterface()); rootContext->setContextProperty("AvatarList", DependencyManager::get<AvatarManager>().data()); rootContext->setContextProperty("Camera", &_myCamera); #if defined(Q_OS_MAC) || defined(Q_OS_WIN) rootContext->setContextProperty("SpeechRecognizer", DependencyManager::get<SpeechRecognizer>().data()); #endif rootContext->setContextProperty("Overlays", &_overlays); rootContext->setContextProperty("Window", DependencyManager::get<WindowScriptingInterface>().data()); rootContext->setContextProperty("MenuInterface", MenuScriptingInterface::getInstance()); rootContext->setContextProperty("Stats", Stats::getInstance()); rootContext->setContextProperty("Settings", SettingsScriptingInterface::getInstance()); rootContext->setContextProperty("ScriptDiscoveryService", DependencyManager::get<ScriptEngines>().data()); rootContext->setContextProperty("AudioDevice", AudioDeviceScriptingInterface::getInstance()); // Caches rootContext->setContextProperty("AnimationCache", DependencyManager::get<AnimationCache>().data()); rootContext->setContextProperty("TextureCache", DependencyManager::get<TextureCache>().data()); rootContext->setContextProperty("ModelCache", DependencyManager::get<ModelCache>().data()); rootContext->setContextProperty("SoundCache", DependencyManager::get<SoundCache>().data()); rootContext->setContextProperty("Account", AccountScriptingInterface::getInstance()); rootContext->setContextProperty("DialogsManager", _dialogsManagerScriptingInterface); rootContext->setContextProperty("GlobalServices", GlobalServicesScriptingInterface::getInstance()); rootContext->setContextProperty("FaceTracker", DependencyManager::get<DdeFaceTracker>().data()); rootContext->setContextProperty("AvatarManager", DependencyManager::get<AvatarManager>().data()); rootContext->setContextProperty("UndoStack", &_undoStackScriptingInterface); rootContext->setContextProperty("LODManager", DependencyManager::get<LODManager>().data()); rootContext->setContextProperty("Paths", DependencyManager::get<PathUtils>().data()); rootContext->setContextProperty("HMD", DependencyManager::get<HMDScriptingInterface>().data()); rootContext->setContextProperty("Scene", DependencyManager::get<SceneScriptingInterface>().data()); rootContext->setContextProperty("Render", _renderEngine->getConfiguration().get()); rootContext->setContextProperty("Reticle", getApplicationCompositor().getReticleInterface()); rootContext->setContextProperty("ApplicationCompositor", &getApplicationCompositor()); _glWidget->installEventFilter(offscreenUi.data()); offscreenUi->setMouseTranslator([=](const QPointF& pt) { QPointF result = pt; auto displayPlugin = getActiveDisplayPlugin(); if (displayPlugin->isHmd()) { getApplicationCompositor().handleRealMouseMoveEvent(false); auto resultVec = getApplicationCompositor().getReticlePosition(); result = QPointF(resultVec.x, resultVec.y); } return result.toPoint(); }); offscreenUi->resume(); connect(_window, &MainWindow::windowGeometryChanged, [this](const QRect& r){ resizeGL(); }); // This will set up the input plugins UI _activeInputPlugins.clear(); foreach(auto inputPlugin, PluginManager::getInstance()->getInputPlugins()) { if (KeyboardMouseDevice::NAME == inputPlugin->getName()) { _keyboardMouseDevice = std::dynamic_pointer_cast<KeyboardMouseDevice>(inputPlugin); } } _window->setMenuBar(new Menu()); auto compositorHelper = DependencyManager::get<CompositorHelper>(); connect(compositorHelper.data(), &CompositorHelper::allowMouseCaptureChanged, [=] { if (isHMDMode()) { showCursor(compositorHelper->getAllowMouseCapture() ? Qt::BlankCursor : Qt::ArrowCursor); } }); } void Application::paintGL() { // Some plugins process message events, allowing paintGL to be called reentrantly. if (_inPaint || _aboutToQuit) { return; } _inPaint = true; Finally clearFlag([this] { _inPaint = false; }); _frameCount++; _frameCounter.increment(); auto lastPaintBegin = usecTimestampNow(); PROFILE_RANGE_EX(__FUNCTION__, 0xff0000ff, (uint64_t)_frameCount); PerformanceTimer perfTimer("paintGL"); if (nullptr == _displayPlugin) { return; } auto displayPlugin = getActiveDisplayPlugin(); // FIXME not needed anymore? _offscreenContext->makeCurrent(); // If a display plugin loses it's underlying support, it // needs to be able to signal us to not use it if (!displayPlugin->beginFrameRender(_frameCount)) { _inPaint = false; updateDisplayMode(); return; } // update the avatar with a fresh HMD pose getMyAvatar()->updateFromHMDSensorMatrix(getHMDSensorPose()); auto lodManager = DependencyManager::get<LODManager>(); { QMutexLocker viewLocker(&_viewMutex); _viewFrustum.calculate(); } RenderArgs renderArgs(_gpuContext, getEntities(), lodManager->getOctreeSizeScale(), lodManager->getBoundaryLevelAdjust(), RenderArgs::DEFAULT_RENDER_MODE, RenderArgs::MONO, RenderArgs::RENDER_DEBUG_NONE); { QMutexLocker viewLocker(&_viewMutex); renderArgs.setViewFrustum(_viewFrustum); } PerformanceWarning::setSuppressShortTimings(Menu::getInstance()->isOptionChecked(MenuOption::SuppressShortTimings)); bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); PerformanceWarning warn(showWarnings, "Application::paintGL()"); resizeGL(); // Before anything else, let's sync up the gpuContext with the true glcontext used in case anything happened { PerformanceTimer perfTimer("syncCache"); renderArgs._context->syncCache(); } auto inputs = AvatarInputs::getInstance(); if (inputs->mirrorVisible()) { PerformanceTimer perfTimer("Mirror"); auto primaryFbo = DependencyManager::get<FramebufferCache>()->getPrimaryFramebuffer(); renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; renderArgs._blitFramebuffer = DependencyManager::get<FramebufferCache>()->getSelfieFramebuffer(); _mirrorViewRect.moveTo(inputs->x(), inputs->y()); renderRearViewMirror(&renderArgs, _mirrorViewRect, inputs->mirrorZoomed()); renderArgs._blitFramebuffer.reset(); renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; } { PerformanceTimer perfTimer("renderOverlay"); // NOTE: There is no batch associated with this renderArgs // the ApplicationOverlay class assumes it's viewport is setup to be the device size QSize size = getDeviceSize(); renderArgs._viewport = glm::ivec4(0, 0, size.width(), size.height()); _applicationOverlay.renderOverlay(&renderArgs); auto overlayTexture = _applicationOverlay.acquireOverlay(); if (overlayTexture) { displayPlugin->submitOverlayTexture(overlayTexture); } } glm::vec3 boomOffset; { PerformanceTimer perfTimer("CameraUpdates"); auto myAvatar = getMyAvatar(); boomOffset = myAvatar->getScale() * myAvatar->getBoomLength() * -IDENTITY_FRONT; if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON || _myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { Menu::getInstance()->setIsOptionChecked(MenuOption::FirstPerson, myAvatar->getBoomLength() <= MyAvatar::ZOOM_MIN); Menu::getInstance()->setIsOptionChecked(MenuOption::ThirdPerson, !(myAvatar->getBoomLength() <= MyAvatar::ZOOM_MIN)); cameraMenuChanged(); } // The render mode is default or mirror if the camera is in mirror mode, assigned further below renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; // Always use the default eye position, not the actual head eye position. // Using the latter will cause the camera to wobble with idle animations, // or with changes from the face tracker if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { if (isHMDMode()) { mat4 camMat = myAvatar->getSensorToWorldMatrix() * myAvatar->getHMDSensorMatrix(); _myCamera.setPosition(extractTranslation(camMat)); _myCamera.setOrientation(glm::quat_cast(camMat)); } else { _myCamera.setPosition(myAvatar->getDefaultEyePosition()); _myCamera.setOrientation(myAvatar->getHead()->getCameraOrientation()); } } else if (_myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { if (isHMDMode()) { auto hmdWorldMat = myAvatar->getSensorToWorldMatrix() * myAvatar->getHMDSensorMatrix(); _myCamera.setOrientation(glm::normalize(glm::quat_cast(hmdWorldMat))); _myCamera.setPosition(extractTranslation(hmdWorldMat) + myAvatar->getOrientation() * boomOffset); } else { _myCamera.setOrientation(myAvatar->getHead()->getOrientation()); if (Menu::getInstance()->isOptionChecked(MenuOption::CenterPlayerInView)) { _myCamera.setPosition(myAvatar->getDefaultEyePosition() + _myCamera.getOrientation() * boomOffset); } else { _myCamera.setPosition(myAvatar->getDefaultEyePosition() + myAvatar->getOrientation() * boomOffset); } } } else if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { if (isHMDMode()) { auto mirrorBodyOrientation = myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f)); glm::quat hmdRotation = extractRotation(myAvatar->getHMDSensorMatrix()); // Mirror HMD yaw and roll glm::vec3 mirrorHmdEulers = glm::eulerAngles(hmdRotation); mirrorHmdEulers.y = -mirrorHmdEulers.y; mirrorHmdEulers.z = -mirrorHmdEulers.z; glm::quat mirrorHmdRotation = glm::quat(mirrorHmdEulers); glm::quat worldMirrorRotation = mirrorBodyOrientation * mirrorHmdRotation; _myCamera.setOrientation(worldMirrorRotation); glm::vec3 hmdOffset = extractTranslation(myAvatar->getHMDSensorMatrix()); // Mirror HMD lateral offsets hmdOffset.x = -hmdOffset.x; _myCamera.setPosition(myAvatar->getDefaultEyePosition() + glm::vec3(0, _raiseMirror * myAvatar->getUniformScale(), 0) + mirrorBodyOrientation * glm::vec3(0.0f, 0.0f, 1.0f) * MIRROR_FULLSCREEN_DISTANCE * _scaleMirror + mirrorBodyOrientation * hmdOffset); } else { _myCamera.setOrientation(myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f))); _myCamera.setPosition(myAvatar->getDefaultEyePosition() + glm::vec3(0, _raiseMirror * myAvatar->getUniformScale(), 0) + (myAvatar->getOrientation() * glm::quat(glm::vec3(0.0f, _rotateMirror, 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f) * MIRROR_FULLSCREEN_DISTANCE * _scaleMirror); } renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; } else if (_myCamera.getMode() == CAMERA_MODE_ENTITY) { EntityItemPointer cameraEntity = _myCamera.getCameraEntityPointer(); if (cameraEntity != nullptr) { if (isHMDMode()) { glm::quat hmdRotation = extractRotation(myAvatar->getHMDSensorMatrix()); _myCamera.setOrientation(cameraEntity->getRotation() * hmdRotation); glm::vec3 hmdOffset = extractTranslation(myAvatar->getHMDSensorMatrix()); _myCamera.setPosition(cameraEntity->getPosition() + (hmdRotation * hmdOffset)); } else { _myCamera.setOrientation(cameraEntity->getRotation()); _myCamera.setPosition(cameraEntity->getPosition()); } } } // Update camera position if (!isHMDMode()) { _myCamera.update(1.0f / _frameCounter.rate()); } } getApplicationCompositor().setFrameInfo(_frameCount, _myCamera.getTransform()); // Primary rendering pass auto framebufferCache = DependencyManager::get<FramebufferCache>(); const QSize size = framebufferCache->getFrameBufferSize(); // Final framebuffer that will be handled to the display-plugin auto finalFramebuffer = framebufferCache->getFramebuffer(); { PROFILE_RANGE(__FUNCTION__ "/mainRender"); PerformanceTimer perfTimer("mainRender"); renderArgs._boomOffset = boomOffset; // Viewport is assigned to the size of the framebuffer renderArgs._viewport = ivec4(0, 0, size.width(), size.height()); if (displayPlugin->isStereo()) { // Stereo modes will typically have a larger projection matrix overall, // so we ask for the 'mono' projection matrix, which for stereo and HMD // plugins will imply the combined projection for both eyes. // // This is properly implemented for the Oculus plugins, but for OpenVR // and Stereo displays I'm not sure how to get / calculate it, so we're // just relying on the left FOV in each case and hoping that the // overall culling margin of error doesn't cause popping in the // right eye. There are FIXMEs in the relevant plugins _myCamera.setProjection(displayPlugin->getCullingProjection(_myCamera.getProjection())); renderArgs._context->enableStereo(true); mat4 eyeOffsets[2]; mat4 eyeProjections[2]; auto baseProjection = renderArgs.getViewFrustum().getProjection(); auto hmdInterface = DependencyManager::get<HMDScriptingInterface>(); float IPDScale = hmdInterface->getIPDScale(); mat4 headPose = displayPlugin->getHeadPose(); // FIXME we probably don't need to set the projection matrix every frame, // only when the display plugin changes (or in non-HMD modes when the user // changes the FOV manually, which right now I don't think they can. for_each_eye([&](Eye eye) { // For providing the stereo eye views, the HMD head pose has already been // applied to the avatar, so we need to get the difference between the head // pose applied to the avatar and the per eye pose, and use THAT as // the per-eye stereo matrix adjustment. mat4 eyeToHead = displayPlugin->getEyeToHeadTransform(eye); // Grab the translation vec3 eyeOffset = glm::vec3(eyeToHead[3]); // Apply IPD scaling mat4 eyeOffsetTransform = glm::translate(mat4(), eyeOffset * -1.0f * IPDScale); eyeOffsets[eye] = eyeOffsetTransform; // Tell the plugin what pose we're using to render. In this case we're just using the // unmodified head pose because the only plugin that cares (the Oculus plugin) uses it // for rotational timewarp. If we move to support positonal timewarp, we need to // ensure this contains the full pose composed with the eye offsets. displayPlugin->setEyeRenderPose(_frameCount, eye, headPose * glm::inverse(eyeOffsetTransform)); eyeProjections[eye] = displayPlugin->getEyeProjection(eye, baseProjection); }); renderArgs._context->setStereoProjections(eyeProjections); renderArgs._context->setStereoViews(eyeOffsets); } renderArgs._blitFramebuffer = finalFramebuffer; displaySide(&renderArgs, _myCamera); renderArgs._blitFramebuffer.reset(); renderArgs._context->enableStereo(false); } // deliver final composited scene to the display plugin { PROFILE_RANGE(__FUNCTION__ "/pluginOutput"); PerformanceTimer perfTimer("pluginOutput"); auto finalTexture = finalFramebuffer->getRenderBuffer(0); Q_ASSERT(!_lockedFramebufferMap.contains(finalTexture)); _lockedFramebufferMap[finalTexture] = finalFramebuffer; Q_ASSERT(isCurrentContext(_offscreenContext->getContext())); { PROFILE_RANGE(__FUNCTION__ "/pluginSubmitScene"); PerformanceTimer perfTimer("pluginSubmitScene"); displayPlugin->submitSceneTexture(_frameCount, finalTexture); } Q_ASSERT(isCurrentContext(_offscreenContext->getContext())); } { Stats::getInstance()->setRenderDetails(renderArgs._details); // Reset the gpu::Context Stages // Back to the default framebuffer; gpu::doInBatch(renderArgs._context, [&](gpu::Batch& batch) { batch.resetStages(); }); } uint64_t lastPaintDuration = usecTimestampNow() - lastPaintBegin; _frameTimingsScriptingInterface.addValue(lastPaintDuration); } void Application::runTests() { runTimingTests(); runUnitTests(); } void Application::audioMuteToggled() const { QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteAudio); Q_CHECK_PTR(muteAction); muteAction->setChecked(DependencyManager::get<AudioClient>()->isMuted()); } void Application::faceTrackerMuteToggled() { QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteFaceTracking); Q_CHECK_PTR(muteAction); bool isMuted = getSelectedFaceTracker()->isMuted(); muteAction->setChecked(isMuted); getSelectedFaceTracker()->setEnabled(!isMuted); Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!isMuted); } void Application::setFieldOfView(float fov) { if (fov != _fieldOfView.get()) { _fieldOfView.set(fov); resizeGL(); } } void Application::aboutApp() { InfoView::show(INFO_HELP_PATH); } void Application::showHelp() { InfoView::show(INFO_EDIT_ENTITIES_PATH); } void Application::resizeEvent(QResizeEvent* event) { resizeGL(); } void Application::resizeGL() { PROFILE_RANGE(__FUNCTION__); if (nullptr == _displayPlugin) { return; } auto displayPlugin = getActiveDisplayPlugin(); // Set the desired FBO texture size. If it hasn't changed, this does nothing. // Otherwise, it must rebuild the FBOs uvec2 framebufferSize = displayPlugin->getRecommendedRenderSize(); uvec2 renderSize = uvec2(vec2(framebufferSize) * getRenderResolutionScale()); if (_renderResolution != renderSize) { _renderResolution = renderSize; DependencyManager::get<FramebufferCache>()->setFrameBufferSize(fromGlm(renderSize)); } // FIXME the aspect ratio for stereo displays is incorrect based on this. float aspectRatio = displayPlugin->getRecommendedAspectRatio(); _myCamera.setProjection(glm::perspective(glm::radians(_fieldOfView.get()), aspectRatio, DEFAULT_NEAR_CLIP, DEFAULT_FAR_CLIP)); // Possible change in aspect ratio { QMutexLocker viewLocker(&_viewMutex); loadViewFrustum(_myCamera, _viewFrustum); } auto offscreenUi = DependencyManager::get<OffscreenUi>(); auto uiSize = displayPlugin->getRecommendedUiSize(); // Bit of a hack since there's no device pixel ratio change event I can find. static qreal lastDevicePixelRatio = 0; qreal devicePixelRatio = _window->devicePixelRatio(); if (offscreenUi->size() != fromGlm(uiSize) || devicePixelRatio != lastDevicePixelRatio) { qDebug() << "Device pixel ratio changed, triggering resize"; offscreenUi->resize(fromGlm(uiSize), true); _offscreenContext->makeCurrent(); lastDevicePixelRatio = devicePixelRatio; } } bool Application::importSVOFromURL(const QString& urlString) { emit svoImportRequested(urlString); return true; } bool Application::event(QEvent* event) { if (!Menu::getInstance()) { return false; } // Presentation/painting logic // TODO: Decouple presentation and painting loops static bool isPaintingThrottled = false; if ((int)event->type() == (int)Present) { if (isPaintingThrottled) { // If painting (triggered by presentation) is hogging the main thread, // repost as low priority to avoid hanging the GUI. // This has the effect of allowing presentation to exceed the paint budget by X times and // only dropping every (1/X) frames, instead of every ceil(X) frames // (e.g. at a 60FPS target, painting for 17us would fall to 58.82FPS instead of 30FPS). removePostedEvents(this, Present); postEvent(this, new QEvent(static_cast<QEvent::Type>(Present)), Qt::LowEventPriority); isPaintingThrottled = false; return true; } float nsecsElapsed = (float)_lastTimeUpdated.nsecsElapsed(); if (shouldPaint(nsecsElapsed)) { _lastTimeUpdated.start(); idle(nsecsElapsed); postEvent(this, new QEvent(static_cast<QEvent::Type>(Paint)), Qt::HighEventPriority); } isPaintingThrottled = true; return true;