////  AssetServer.cpp//  assignment-client/src/assets////  Created by Ryan Huffman on 2015/07/21//  Copyright 2015 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html//  #include "AssetServer.h" #include <thread>#include <memory> #include <QtCore/QCoreApplication>#include <QtCore/QCryptographicHash>#include <QtCore/QDateTime>#include <QtCore/QDir>#include <QtCore/QDirIterator>#include <QtCore/QFile>#include <QtCore/QFileInfo>#include <QtCore/QJsonDocument>#include <QtCore/QSaveFile>#include <QtCore/QString>#include <QtGui/QImageReader>#include <QtCore/QVector>#include <QtCore/QUrlQuery> #include <ClientServerUtils.h>#include <NodeType.h>#include <SharedUtil.h>#include <PathUtils.h>#include <image/TextureProcessing.h> #include "AssetServerLogging.h"#include "BakeAssetTask.h"#include "SendAssetTask.h"#include "UploadAssetTask.h" static const uint8_t MIN_CORES_FOR_MULTICORE = 4;static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2;static const uint8_t CPU_AFFINITY_COUNT_LOW = 1;#ifdef Q_OS_WINstatic const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000;#endif static const QStringList BAKEABLE_MODEL_EXTENSIONS = { "fbx" };static QStringList BAKEABLE_TEXTURE_EXTENSIONS;static const QStringList BAKEABLE_SCRIPT_EXTENSIONS = { }; static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx";static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx";static const QString BAKED_SCRIPT_SIMPLE_NAME = "asset.js"; static const ModelBakeVersion CURRENT_MODEL_BAKE_VERSION = (ModelBakeVersion)((BakeVersion)ModelBakeVersion::COUNT - 1);static const TextureBakeVersion CURRENT_TEXTURE_BAKE_VERSION = (TextureBakeVersion)((BakeVersion)TextureBakeVersion::COUNT - 1);static const ScriptBakeVersion CURRENT_SCRIPT_BAKE_VERSION = (ScriptBakeVersion)((BakeVersion)ScriptBakeVersion::COUNT - 1); BakedAssetType assetTypeForExtension(const QString& extension) {    auto extensionLower = extension.toLower();    if (BAKEABLE_MODEL_EXTENSIONS.contains(extensionLower)) {        return BakedAssetType::Model;    } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extensionLower.toLocal8Bit())) {        return BakedAssetType::Texture;    } else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extensionLower)) {        return BakedAssetType::Script;    }    return BakedAssetType::Undefined;} BakedAssetType assetTypeForFilename(const QString& filename) {    auto dotIndex = filename.lastIndexOf(".");    if (dotIndex == -1) {        return BakedAssetType::Undefined;    }     auto extension = filename.mid(dotIndex + 1);    return assetTypeForExtension(extension);} QString bakedFilenameForAssetType(BakedAssetType type) {    switch (type) {        case BakedAssetType::Model:            return BAKED_MODEL_SIMPLE_NAME;        case BakedAssetType::Texture:            return BAKED_TEXTURE_SIMPLE_NAME;        case BakedAssetType::Script:            return BAKED_SCRIPT_SIMPLE_NAME;        default:            return "";    }} BakeVersion currentBakeVersionForAssetType(BakedAssetType type) {    switch (type) {        case BakedAssetType::Model:            return (BakeVersion)CURRENT_MODEL_BAKE_VERSION;        case BakedAssetType::Texture:            return (BakeVersion)CURRENT_TEXTURE_BAKE_VERSION;        case BakedAssetType::Script:            return (BakeVersion)CURRENT_SCRIPT_BAKE_VERSION;        default:            return 0;    }} QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativeFilePath) {    return AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath;} const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; void AssetServer::bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) {    qDebug() << "Starting bake for: " << assetPath << assetHash;    auto it = _pendingBakes.find(assetHash);    if (it == _pendingBakes.end()) {        auto task = std::make_shared<BakeAssetTask>(assetHash, assetPath, filePath);        task->setAutoDelete(false);        _pendingBakes[assetHash] = task;         connect(task.get(), &BakeAssetTask::bakeComplete, this, &AssetServer::handleCompletedBake);        connect(task.get(), &BakeAssetTask::bakeFailed, this, &AssetServer::handleFailedBake);        connect(task.get(), &BakeAssetTask::bakeAborted, this, &AssetServer::handleAbortedBake);         _bakingTaskPool.start(task.get());    } else {        qDebug() << "Already in queue";    }} QString AssetServer::getPathToAssetHash(const AssetUtils::AssetHash& assetHash) {    return _filesDirectory.absoluteFilePath(assetHash);} std::pair<AssetUtils::BakingStatus, QString> AssetServer::getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) {    auto it = _pendingBakes.find(hash);    if (it != _pendingBakes.end()) {        return { (*it)->isBaking() ? AssetUtils::Baking : AssetUtils::Pending, "" };    }     if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {        return { AssetUtils::Baked, "" };    }     BakedAssetType type = assetTypeForFilename(path);    if (type == BakedAssetType::Undefined) {        return { AssetUtils::Irrelevant, "" };    }     bool loaded;    AssetMeta meta;    std::tie(loaded, meta) = readMetaFile(hash);     // We create a meta file for Skyboxes at runtime when they get requested    // Otherwise, textures don't get baked by themselves.    if (type == BakedAssetType::Texture && !loaded) {        return { AssetUtils::Irrelevant, "" };    }     QString bakedFilename = bakedFilenameForAssetType(type);    auto bakedPath = getBakeMapping(hash, bakedFilename);    if (loaded && !meta.redirectTarget.isEmpty()) {        bakedPath = meta.redirectTarget;    }     auto jt = _fileMappings.find(bakedPath);    if (jt != _fileMappings.end()) {        if (jt->second == hash) {            return { AssetUtils::NotBaked, "" };        } else {            return { AssetUtils::Baked, "" };        }    } else if (loaded && meta.failedLastBake) {        return { AssetUtils::Error, meta.lastBakeErrors };    }     return { AssetUtils::Pending, "" };} void AssetServer::bakeAssets() {    auto it = _fileMappings.cbegin();    for (; it != _fileMappings.cend(); ++it) {        auto path = it->first;        auto hash = it->second;        maybeBake(path, hash);    }} void AssetServer::maybeBake(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) {    if (needsToBeBaked(path, hash)) {        qDebug() << "Queuing bake of: " << path;        bakeAsset(hash, path, getPathToAssetHash(hash));    }} void AssetServer::createEmptyMetaFile(const AssetUtils::AssetHash& hash) {    QString metaFilePath = "atp:/" + hash + "/meta.json";    QFile metaFile { metaFilePath };     if (!metaFile.exists()) {        qDebug() << "Creating metafile for " << hash;        if (metaFile.open(QFile::WriteOnly)) {            qDebug() << "Created metafile for " << hash;            metaFile.write("{}");        }    }} bool AssetServer::hasMetaFile(const AssetUtils::AssetHash& hash) {    QString metaFilePath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/meta.json";     return _fileMappings.find(metaFilePath) != _fileMappings.end();} bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& assetHash) {    if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {        return false;    }     BakedAssetType type = assetTypeForFilename(path);     if (type == BakedAssetType::Undefined) {        return false;    }     bool loaded;    AssetMeta meta;    std::tie(loaded, meta) = readMetaFile(assetHash);     QString bakedFilename = bakedFilenameForAssetType(type);    auto bakedPath = getBakeMapping(assetHash, bakedFilename);    if (loaded && !meta.redirectTarget.isEmpty()) {        bakedPath = meta.redirectTarget;    }     auto mappingIt = _fileMappings.find(bakedPath);    bool bakedMappingExists = mappingIt != _fileMappings.end();     // If the path is mapped to the original file's hash, baking has been disabled for this    // asset    if (bakedMappingExists && mappingIt->second == assetHash) {        return false;    }     // We create a meta file for Skyboxes at runtime when they get requested    // Otherwise, textures don't get baked by themselves.    if (type == BakedAssetType::Texture && !loaded) {        return false;    }     auto currentVersion = currentBakeVersionForAssetType(type);     if (loaded && (meta.failedLastBake && meta.bakeVersion >= currentVersion)) {        return false;    }     return !bakedMappingExists || (meta.bakeVersion < currentVersion);} bool interfaceRunning() {    bool result = false; #ifdef Q_OS_WIN    QSharedMemory sharedMemory { getInterfaceSharedMemoryName() };    result = sharedMemory.attach(QSharedMemory::ReadOnly);    if (result) {        sharedMemory.detach();    }#endif    return result;} void updateConsumedCores() {    static bool wasInterfaceRunning = false;    bool isInterfaceRunning = interfaceRunning();    // If state is unchanged, return early    if (isInterfaceRunning == wasInterfaceRunning) {        return;    }     wasInterfaceRunning = isInterfaceRunning;    auto coreCount = std::thread::hardware_concurrency();    if (isInterfaceRunning) {        coreCount = coreCount > MIN_CORES_FOR_MULTICORE ? CPU_AFFINITY_COUNT_HIGH : CPU_AFFINITY_COUNT_LOW;    }    qCDebug(asset_server) << "Setting max consumed cores to " << coreCount;    setMaxCores(coreCount);}  AssetServer::AssetServer(ReceivedMessage& message) :    ThreadedAssignment(message),    _transferTaskPool(this),    _bakingTaskPool(this),    _filesizeLimit(AssetUtils::MAX_UPLOAD_SIZE){    BAKEABLE_TEXTURE_EXTENSIONS = image::getSupportedFormats();    qDebug() << "Supported baking texture formats:" << BAKEABLE_MODEL_EXTENSIONS;     // Most of the work will be I/O bound, reading from disk and constructing packet objects,    // so the ideal is greater than the number of cores on the system.    static const int TASK_POOL_THREAD_COUNT = 50;    _transferTaskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT);    _bakingTaskPool.setMaxThreadCount(1);     // Queue all requests until the Asset Server is fully setup    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();    packetReceiver.registerListenerForTypes({ PacketType::AssetGet, PacketType::AssetGetInfo, PacketType::AssetUpload, PacketType::AssetMappingOperation },        PacketReceiver::makeSourcedListenerReference<AssetServer>(this, &AssetServer::queueRequests)); #ifdef Q_OS_WIN    updateConsumedCores();    QTimer* timer = new QTimer(this);    auto timerConnection = connect(timer, &QTimer::timeout, [] {        updateConsumedCores();    });    connect(qApp, &QCoreApplication::aboutToQuit, [this, timerConnection] {        disconnect(timerConnection);    });    timer->setInterval(INTERFACE_RUNNING_CHECK_FREQUENCY_MS);    timer->setTimerType(Qt::CoarseTimer);    timer->start();#endif} void AssetServer::aboutToFinish() {     // remove pending transfer tasks    _transferTaskPool.clear();     // abort each of our still running bake tasks, remove pending bakes that were never put on the thread pool    auto it = _pendingBakes.begin();    while (it != _pendingBakes.end()) {        auto pendingRunnable =  _bakingTaskPool.tryTake(it->get());         if (pendingRunnable) {            it = _pendingBakes.erase(it);        } else {            qDebug() << "Aborting bake for" << it.key();            it.value()->abort();            ++it;        }    }     // make sure all bakers are finished or aborted    while (_pendingBakes.size() > 0) {        QCoreApplication::processEvents();    }} void AssetServer::run() {     qCDebug(asset_server) << "Waiting for connection to domain to request settings from domain-server.";     // wait until we have the domain-server settings, otherwise we bail    DomainHandler& domainHandler = DependencyManager::get<NodeList>()->getDomainHandler();    connect(&domainHandler, &DomainHandler::settingsReceived, this, &AssetServer::completeSetup);    connect(&domainHandler, &DomainHandler::settingsReceiveFail, this, &AssetServer::domainSettingsRequestFailed);     ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer);} static const QString ASSET_FILES_SUBDIR = "files"; void AssetServer::completeSetup() {    auto nodeList = DependencyManager::get<NodeList>();     auto& domainHandler = nodeList->getDomainHandler();    const QJsonObject& settingsObject = domainHandler.getSettingsObject();     commonParseSettingsObject(settingsObject);     static const QString ASSET_SERVER_SETTINGS_KEY = "asset_server";     if (!settingsObject.contains(ASSET_SERVER_SETTINGS_KEY)) {        qCCritical(asset_server) << "Received settings from the domain-server with no asset-server section. Stopping assignment.";        setFinished(true);        return;    }     auto assetServerObject = settingsObject[ASSET_SERVER_SETTINGS_KEY].toObject();     static const QString MAX_BANDWIDTH_OPTION = "max_bandwidth";    auto maxBandwidthValue = assetServerObject[MAX_BANDWIDTH_OPTION];    auto maxBandwidthFloat = maxBandwidthValue.toDouble(-1);     const int BITS_PER_MEGABITS = 1000 * 1000;    if (maxBandwidthFloat > 0.0) {        int maxBandwidth = maxBandwidthFloat * BITS_PER_MEGABITS;        nodeList->setConnectionMaxBandwidth(maxBandwidth);        qCInfo(asset_server) << "Set maximum bandwith per connection to" << maxBandwidthFloat << "Mb/s."                    " (" << maxBandwidth << "bits/s)";    }     // get the path to the asset folder from the domain server settings    static const QString ASSETS_PATH_OPTION = "assets_path";    auto assetsJSONValue = assetServerObject[ASSETS_PATH_OPTION];     if (!assetsJSONValue.isString()) {        qCCritical(asset_server) << "Received an assets path from the domain-server that could not be parsed. Stopping assignment.";        setFinished(true);        return;    }     auto assetsPathString = assetsJSONValue.toString();    QDir assetsPath { assetsPathString };    QString absoluteFilePath = assetsPath.absolutePath();     if (assetsPath.isRelative()) {        // if the domain settings passed us a relative path, make an absolute path that is relative to the        // default data directory        absoluteFilePath = PathUtils::getAppDataFilePath("assets/" + assetsPathString);    }     _resourcesDirectory = QDir(absoluteFilePath);     qCDebug(asset_server) << "Creating resources directory";    _resourcesDirectory.mkpath(".");    _filesDirectory = _resourcesDirectory;     if (!_resourcesDirectory.mkpath(ASSET_FILES_SUBDIR) || !_filesDirectory.cd(ASSET_FILES_SUBDIR)) {        qCCritical(asset_server) << "Unable to create file directory for asset-server files. Stopping assignment.";        setFinished(true);        return;    }     // load whatever mappings we currently have from the local file    if (loadMappingsFromFile()) {        qCInfo(asset_server) << "Serving files from: " << _filesDirectory.path();         // Check the asset directory to output some information about what we have        auto files = _filesDirectory.entryList(QDir::Files);         QRegExp hashFileRegex { AssetUtils::ASSET_HASH_REGEX_STRING };        auto hashedFiles = files.filter(hashFileRegex);         qCInfo(asset_server) << "There are" << hashedFiles.size() << "asset files in the asset directory.";         if (_fileMappings.size() > 0) {            cleanupUnmappedFiles();            cleanupBakedFilesForDeletedAssets();        }         nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });         bakeAssets();    } else {        qCCritical(asset_server) << "Asset Server assignment will not continue because mapping file could not be loaded.";        setFinished(true);    }     // get file size limit for an asset    static const QString ASSETS_FILESIZE_LIMIT_OPTION = "assets_filesize_limit";    auto assetsFilesizeLimitJSONValue = assetServerObject[ASSETS_FILESIZE_LIMIT_OPTION];    auto assetsFilesizeLimit = (uint64_t)assetsFilesizeLimitJSONValue.toInt(AssetUtils::MAX_UPLOAD_SIZE);     if (assetsFilesizeLimit != 0 && assetsFilesizeLimit < AssetUtils::MAX_UPLOAD_SIZE) {        _filesizeLimit = assetsFilesizeLimit * BITS_PER_MEGABITS;    }     PathUtils::removeTemporaryApplicationDirs();    PathUtils::removeTemporaryApplicationDirs("Oven");     qCDebug(asset_server) << "Overriding temporary queuing packet handler.";    // We're fully setup, override the request queueing handler and replay all requests    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();    packetReceiver.registerListener(PacketType::AssetGet,        PacketReceiver::makeSourcedListenerReference<AssetServer>(this, &AssetServer::handleAssetGet));    packetReceiver.registerListener(PacketType::AssetGetInfo,        PacketReceiver::makeSourcedListenerReference<AssetServer>(this, &AssetServer::handleAssetGetInfo));    packetReceiver.registerListener(PacketType::AssetUpload,        PacketReceiver::makeSourcedListenerReference<AssetServer>(this, &AssetServer::handleAssetUpload));    packetReceiver.registerListener(PacketType::AssetMappingOperation,        PacketReceiver::makeSourcedListenerReference<AssetServer>(this, &AssetServer::handleAssetMappingOperation));     replayRequests();} void AssetServer::queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode) {    qCDebug(asset_server) << "Queuing requests until fully setup";     QMutexLocker lock { &_queuedRequestsMutex };    _queuedRequests.push_back({ packet, senderNode });     // If we've stopped queueing but the callback was already in flight,    // then replay it immediately.    if (!_isQueueingRequests) {        lock.unlock();        replayRequests();    }} void AssetServer::replayRequests() {    RequestQueue queue;    {        QMutexLocker lock { &_queuedRequestsMutex };        std::swap(queue, _queuedRequests);

Lines 1–500 of 1603. Load more