////  ATPAssetMigrator.cpp//  interface/src/assets////  Created by Stephen Birarda on 2015-10-12.//  Copyright 2015 High Fidelity, Inc.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #include "ATPAssetMigrator.h" #include <QtCore/QDebug>#include <QtCore/QJsonDocument>#include <QtCore/QJsonObject>#include <QtCore/QLoggingCategory>#include <QtCore/QTemporaryFile>#include <QtWidgets/QFileDialog>#include <QtWidgets/QMessageBox> #include <Gzip.h> #include <AssetClient.h>#include <AssetUpload.h>#include <ResourceManager.h>#include <MappingRequest.h> #include "OffscreenUi.h" Q_DECLARE_LOGGING_CATEGORY(asset_migrator);Q_LOGGING_CATEGORY(asset_migrator, "hf.asset_migrator"); ATPAssetMigrator& ATPAssetMigrator::getInstance() {    static ATPAssetMigrator instance;    return instance;} static const QString ENTITIES_OBJECT_KEY = "Entities";static const QString MODEL_URL_KEY = "modelURL";static const QString COMPOUND_SHAPE_URL_KEY = "compoundShapeURL";static const QString MESSAGE_BOX_TITLE = "ATP Asset Migration"; void ATPAssetMigrator::loadEntityServerFile() {    ModalDialogListener* dlg = OffscreenUi::getOpenFileNameAsync(_dialogParent, tr("Select an entity-server content file to migrate"),                                      QString(), tr("Entity-Server Content (*.gz)"));     connect(dlg, &ModalDialogListener::response, this, [=, this] (QVariant response) {        const QString& filename = response.toString();        disconnect(dlg, &ModalDialogListener::response, this, nullptr);        if (!filename.isEmpty()) {            qCDebug(asset_migrator) << "Selected filename for ATP asset migration: " << filename;             auto migrateResources = [=, this](QUrl migrationURL, QJsonValueRef jsonValue, bool isModelURL) {                auto request =                        DependencyManager::get<ResourceManager>()->createResourceRequest(                            this, migrationURL, true, -1, "ATPAssetMigrator::loadEntityServerFile");                 if (request) {                    qCDebug(asset_migrator) << "Requesting" << migrationURL << "for ATP asset migration";                     // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL                    // to an ATP one once ready                    _pendingReplacements.insert(migrationURL, { jsonValue, (isModelURL ? 0 : 1)});                     connect(request, &ResourceRequest::finished, this, [=, this]() {                        if (request->getResult() == ResourceRequest::Success) {                            migrateResource(request);                        } else {                            ++_errorCount;                            _pendingReplacements.remove(migrationURL);                            qWarning() << "Could not retrieve asset";                             checkIfFinished();                        }                        request->deleteLater();                    });                     request->send();                } else {                    ++_errorCount;                    qWarning() << "Could not create request for asset";                }             };            static const QString MIGRATION_CONFIRMATION_TEXT {                "The ATP Asset Migration process will scan the selected entity-server file,\nupload discovered resources to the"\                " current asset-server\nand then save a new entity-server file with the ATP URLs.\n\nAre you ready to"\                " continue?\n\nMake sure you are connected to the right domain."            };            ModalDialogListener* migrationConfirmDialog = OffscreenUi::asyncQuestion(_dialogParent, MESSAGE_BOX_TITLE, MIGRATION_CONFIRMATION_TEXT,                                       QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);            QObject::connect(migrationConfirmDialog, &ModalDialogListener::response, this, [=, this] (QVariant answer) {                QObject::disconnect(migrationConfirmDialog, &ModalDialogListener::response, this, nullptr);                 if (QMessageBox::Yes == static_cast<QMessageBox::StandardButton>(answer.toInt())) {                    // try to open the file at the given filename                    QFile modelsFile { filename };                     if (modelsFile.open(QIODevice::ReadOnly)) {                        QByteArray compressedJsonData = modelsFile.readAll();                        QByteArray jsonData;                         if (!gunzip(compressedJsonData, jsonData)) {                            OffscreenUi::asyncWarning(_dialogParent, "Error", "The file at" + filename + "was not in gzip format.");                        }                         QJsonDocument modelsJSON = QJsonDocument::fromJson(jsonData);                        _entitiesArray = modelsJSON.object()["Entities"].toArray();                         for (auto jsonValue : _entitiesArray) {                            QJsonObject entityObject = jsonValue.toObject();                            QString modelURLString = entityObject.value(MODEL_URL_KEY).toString();                            QString compoundURLString = entityObject.value(COMPOUND_SHAPE_URL_KEY).toString();                             for (int i = 0; i < 2; ++i) {                                bool isModelURL = (i == 0);                                quint8 replacementType = i;                                auto migrationURLString = (isModelURL) ? modelURLString : compoundURLString;                                 if (!migrationURLString.isEmpty()) {                                    QUrl migrationURL = QUrl(migrationURLString);                                     if (!_ignoredUrls.contains(migrationURL)                                            && (migrationURL.scheme() == HIFI_URL_SCHEME_HTTP || migrationURL.scheme() == HIFI_URL_SCHEME_HTTPS                                                || migrationURL.scheme() == HIFI_URL_SCHEME_FILE || migrationURL.scheme() == HIFI_URL_SCHEME_FTP)) {                                         if (_pendingReplacements.contains(migrationURL)) {                                            // we already have a request out for this asset, just store the QJsonValueRef                                            // so we can do the hash replacement when the request comes back                                            _pendingReplacements.insert(migrationURL, { jsonValue, replacementType });                                        } else if (_uploadedAssets.contains(migrationURL)) {                                            // we already have a hash for this asset                                            // so just do the replacement immediately                                            if (isModelURL) {                                                entityObject[MODEL_URL_KEY] = _uploadedAssets.value(migrationURL).toString();                                            } else {                                                entityObject[COMPOUND_SHAPE_URL_KEY] = _uploadedAssets.value(migrationURL).toString();                                            }                                             jsonValue = entityObject;                                        } else {                                             static bool hasAskedForCompleteMigration { false };                                            static bool wantsCompleteMigration { false };                                             if (!hasAskedForCompleteMigration) {                                                // this is the first resource migration - ask the user if they just want to migrate everything                                                static const QString COMPLETE_MIGRATION_TEXT { "Do you want to migrate all assets found in this entity-server file?\n"\                                                                                               "Select \"Yes\" to upload all discovered assets to the current asset-server immediately.\n"\                                                                                               "Select \"No\" to be prompted for each discovered asset."                                                                                             };                                                ModalDialogListener* migrationConfirmDialog1 = OffscreenUi::asyncQuestion(_dialogParent, MESSAGE_BOX_TITLE,                                                                           "Would you like to migrate the following resource?\n" + migrationURL.toString(),                                                                           QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);                                                 QObject::connect(migrationConfirmDialog1, &ModalDialogListener::response, this, [=, this] (QVariant answer) {                                                    QObject::disconnect(migrationConfirmDialog1, &ModalDialogListener::response, this, nullptr);                                                    if (static_cast<QMessageBox::StandardButton>(answer.toInt()) ==                                                            QMessageBox::Yes) {                                                        wantsCompleteMigration = true;                                                        migrateResources(migrationURL, jsonValue, isModelURL);                                                    } else {                                                        ModalDialogListener* migrationConfirmDialog2 = OffscreenUi::asyncQuestion(_dialogParent, MESSAGE_BOX_TITLE, COMPLETE_MIGRATION_TEXT,                                                                                   QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);                                                         QObject::connect(migrationConfirmDialog2, &ModalDialogListener::response, this, [=, this] (QVariant answer) {                                                            QObject::disconnect(migrationConfirmDialog2, &ModalDialogListener::response, this, nullptr);                                                            if (static_cast<QMessageBox::StandardButton>(answer.toInt()) ==                                                                    QMessageBox::Yes) {                                                                migrateResources(migrationURL, jsonValue, isModelURL);                                                            } else {                                                                _ignoredUrls.insert(migrationURL);                                                            }                                                        });                                                    }                                                });                                                hasAskedForCompleteMigration = true;                                            }                                            if (wantsCompleteMigration) {                                                migrateResources(migrationURL, jsonValue, isModelURL);                                            }                                        }                                    }                                }                            }                        }                         _doneReading = true;                         checkIfFinished();                     } else {                        OffscreenUi::asyncWarning(_dialogParent, "Error",                                                  "There was a problem loading that entity-server file for ATP asset migration. Please try again");                    }                }            });        }    });} void ATPAssetMigrator::migrateResource(ResourceRequest* request) {    // use an asset client to upload the asset    auto assetClient = DependencyManager::get<AssetClient>();     auto upload = assetClient->createUpload(request->getData());     // add this URL to our hash of AssetUpload to original URL    _originalURLs.insert(upload, request->getUrl());     qCDebug(asset_migrator) << "Starting upload of asset from" << request->getUrl();     // connect to the finished signal so we know when the AssetUpload is done    QObject::connect(upload, &AssetUpload::finished, this, &ATPAssetMigrator::assetUploadFinished);     // start the upload now    upload->start();} void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& hash) {    // remove this migrationURL from the key for the AssetUpload pointer    auto migrationURL = _originalURLs.take(upload);     if (upload->getError() == AssetUpload::NoError) {         // use the path of the migrationURL to add a mapping in the Asset Server        auto assetClient = DependencyManager::get<AssetClient>();        auto setMappingRequest = assetClient->createSetMappingRequest(migrationURL.path(), hash);         connect(setMappingRequest, &SetMappingRequest::finished, this, &ATPAssetMigrator::setMappingFinished);         // add this migrationURL with the key for the SetMappingRequest pointer        _originalURLs[setMappingRequest] = migrationURL;         setMappingRequest->start();    } else {        // this is a fail for this modelURL, remove it from pending replacements        _pendingReplacements.remove(migrationURL);         ++_errorCount;        qWarning() << "Failed to upload" << "- error was" << upload->getErrorString();    }     checkIfFinished();     upload->deleteLater();} void ATPAssetMigrator::setMappingFinished(SetMappingRequest* request) {    // take the migrationURL for this SetMappingRequest    auto migrationURL = _originalURLs.take(request);     if (request->getError() == MappingRequest::NoError) {         // successfully uploaded asset - make any required replacements found in the pending replacements        auto values = _pendingReplacements.values(migrationURL);         QString atpURL = QString("atp:%1").arg(request->getPath());         for (auto value : values) {            // replace the modelURL in this QJsonValueRef with the hash            QJsonObject valueObject = value.first.toObject();             if (value.second == 0) {                valueObject[MODEL_URL_KEY] = atpURL;            } else {                valueObject[COMPOUND_SHAPE_URL_KEY] = atpURL;            }             value.first = valueObject;        }         // add this URL to our list of uploaded assets        _uploadedAssets.insert(migrationURL, atpURL);         // pull the replaced urls from _pendingReplacements        _pendingReplacements.remove(migrationURL);    } else {        // this is a fail for this migrationURL, remove it from pending replacements        _pendingReplacements.remove(migrationURL);         ++_errorCount;        qWarning() << "Error setting mapping for" << "- error was " << request->getErrorString();    }     checkIfFinished();     request->deleteLater();} void ATPAssetMigrator::checkIfFinished() {    // are we out of pending replacements? if so it is time to save the entity-server file    if (_doneReading && _pendingReplacements.empty()) {        saveEntityServerFile();    }} bool ATPAssetMigrator::wantsToMigrateResource(const QUrl& url) {    static bool hasAskedForCompleteMigration { false };    static bool wantsCompleteMigration { false };     if (!hasAskedForCompleteMigration) {        // this is the first resource migration - ask the user if they just want to migrate everything        static const QString COMPLETE_MIGRATION_TEXT { "Do you want to migrate all assets found in this entity-server file?\n"\            "Select \"Yes\" to upload all discovered assets to the current asset-server immediately.\n"\            "Select \"No\" to be prompted for each discovered asset."        };         auto button = OffscreenUi::question(_dialogParent, MESSAGE_BOX_TITLE, COMPLETE_MIGRATION_TEXT,                                            QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);         if (button == QMessageBox::Yes) {            wantsCompleteMigration = true;        }         hasAskedForCompleteMigration = true;    }     if (wantsCompleteMigration) {        return true;    } else {        // present a dialog asking the user if they want to migrate this specific resource        auto button = OffscreenUi::question(_dialogParent, MESSAGE_BOX_TITLE,                                            "Would you like to migrate the following resource?\n" + url.toString(),                                            QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);        return button == QMessageBox::Yes;    }} void ATPAssetMigrator::saveEntityServerFile() {    // show a dialog to ask the user where they want to save the file    ModalDialogListener* dlg = OffscreenUi::getSaveFileNameAsync(_dialogParent, "Save Migrated Entities File");    connect(dlg, &ModalDialogListener::response, this, [=, this] (QVariant response) {        const QString& saveName = response.toString();        QFile saveFile { saveName };         if (saveFile.open(QIODevice::WriteOnly)) {            QJsonObject rootObject;            rootObject[ENTITIES_OBJECT_KEY] = _entitiesArray;             QJsonDocument newDocument { rootObject };            QByteArray jsonDataForFile;             if (gzip(newDocument.toJson(), jsonDataForFile, -1)) {                 saveFile.write(jsonDataForFile);                saveFile.close();                 QString infoMessage = QString("Your new entities file has been saved at\n%1.").arg(saveName);                 if (_errorCount > 0) {                    infoMessage += QString("\nThere were %1 models that could not be migrated.\n").arg(_errorCount);                    infoMessage += "Check the warnings in your log for details.\n";                    infoMessage += "You can re-attempt migration on those models\nby restarting this process with the newly saved file.";                }                 OffscreenUi::asyncInformation(_dialogParent, "Success", infoMessage);            } else {                OffscreenUi::asyncWarning(_dialogParent, "Error", "Could not gzip JSON data for new entities file.");            }         } else {            OffscreenUi::asyncWarning(_dialogParent, "Error",                                 QString("Could not open file at %1 to write new entities file to.").arg(saveName));        }        // reset after the attempted save, success or fail        reset();    });} void ATPAssetMigrator::reset() {    _entitiesArray = QJsonArray();    _doneReading = false;    _pendingReplacements.clear();    _uploadedAssets.clear();    _originalURLs.clear();    _ignoredUrls.clear();    _errorCount = 0;}