History

expanded Assets scripting interface

humbletim committed 8 years ago

Commit 8c7a8f01 parent 15fcf6612 files · +1090 −47Browse files

Showing the changes to libraries/networking/src/BaseAssetScriptingInterface.cpp (1 of 12 files). Show all

Unified · Split

libraries/networking/src/BaseAssetScriptingInterface.cpp+399−0
@@ -0,0 +1,399 @@
1+//
2+// BaseAssetScriptingInterface.cpp
3+// libraries/networking/src
4+//
5+// Copyright 2017 High Fidelity, Inc.
6+//
7+// Distributed under the Apache License, Version 2.0.
8+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
9+
10+#include "BaseAssetScriptingInterface.h"
11+
12+#include <QJsonDocument>
13+#include <QJsonArray>
14+#include <QMimeDatabase>
15+#include <QThread>
16+
17+#include "AssetRequest.h"
18+#include "AssetUpload.h"
19+#include "AssetUtils.h"
20+#include "MappingRequest.h"
21+#include "NetworkLogging.h"
22+
23+#include <RegisteredMetaTypes.h>
24+
25+#include <shared/QtHelpers.h>
26+#include "Gzip.h"
27+
28+using Promise = MiniPromise::Promise;
29+
30+QSharedPointer<AssetClient> BaseAssetScriptingInterface::assetClient() {
31+ return DependencyManager::get<AssetClient>();
32+}
33+
34+BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent), _cache(this) {
35+}
36+
37+
38+bool BaseAssetScriptingInterface::initializeCache() {
39+ qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- current values" << _cache.cacheDirectory() << _cache.cacheSize() << _cache.maximumCacheSize();
40+
41+ // NOTE: *instances* of QNetworkDiskCache are not thread-safe -- however, different threads can effectively
42+ // use the same underlying cache if configured with identical settings. Once AssetClient's disk cache settings
43+ // become available we configure our instance to match.
44+ auto assets = assetClient();
45+ if (!assets) {
46+ return false; // not yet possible to initialize the cache
47+ }
48+ if (_cache.cacheDirectory().size()) {
49+ return true; // cache is ready
50+ }
51+
52+ // attempt to initialize the cache
53+ qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- invoking AssetClient::init" << assetClient().data();
54+ QMetaObject::invokeMethod(assetClient().data(), "init");
55+
56+ qDebug() << "BaseAssetScriptingInterface::getCacheStatus querying cache status" << QThread::currentThread();
57+ Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus");
58+ deferred->then([&](QVariantMap result) {
59+ qDebug() << "//queryCacheStatus" << QThread::currentThread();
60+ auto cacheDirectory = result.value("cacheDirectory").toString();
61+ auto cacheSize = result.value("cacheSize").toLongLong();
62+ auto maximumCacheSize = result.value("maximumCacheSize").toLongLong();
63+ qDebug() << "///queryCacheStatus" << cacheDirectory << cacheSize << maximumCacheSize;
64+ _cache.setCacheDirectory(cacheDirectory);
65+ _cache.setMaximumCacheSize(maximumCacheSize);
66+ });
67+ deferred->fail([&](QString error) {
68+ qDebug() << "//queryCacheStatus ERROR" << QThread::currentThread() << error;
69+ });
70+ assets->cacheInfoRequest(deferred);
71+ return false; // cache is not ready yet
72+}
73+
74+QVariantMap BaseAssetScriptingInterface::getCacheStatus() {
75+ //assetClient()->cacheInfoRequest(this, "onCacheInfoResponse");
76+ return {
77+ { "cacheDirectory", _cache.cacheDirectory() },
78+ { "cacheSize", _cache.cacheSize() },
79+ { "maximumCacheSize", _cache.maximumCacheSize() },
80+ };
81+}
82+
83+QVariantMap BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) {
84+ QNetworkCacheMetaData metaData = _cache.metaData(url);
85+ QVariantMap attributes, rawHeaders;
86+
87+ QHashIterator<QNetworkRequest::Attribute, QVariant> i(metaData.attributes());
88+ while (i.hasNext()) {
89+ i.next();
90+ attributes[QString::number(i.key())] = i.value();
91+ }
92+ for (const auto& i : metaData.rawHeaders()) {
93+ rawHeaders[i.first] = i.second;
94+ }
95+ return {
96+ { "isValid", metaData.isValid() },
97+ { "url", metaData.url() },
98+ { "expirationDate", metaData.expirationDate() },
99+ { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() },
100+ { "saveToDisk", metaData.saveToDisk() },
101+ { "attributes", attributes },
102+ { "rawHeaders", rawHeaders },
103+ };
104+}
105+
106+QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) {
107+ qDebug() << "loadFromCache" << url;
108+ QVariantMap result = {
109+ { "metadata", queryCacheMeta(url) },
110+ { "data", QByteArray() },
111+ };
112+ // caller is responsible for the deletion of the ioDevice, hence the unique_ptr
113+ if (auto ioDevice = std::unique_ptr<QIODevice>(_cache.data(url))) {
114+ QByteArray data = ioDevice->readAll();
115+ qCDebug(asset_client) << url.toDisplayString() << "loaded from disk cache (" << data.size() << " bytes)";
116+ result["data"] = data;
117+ }
118+ return result;
119+}
120+
121+namespace {
122+ // parse RFC 1123 HTTP date format
123+ QDateTime parseHttpDate(const QString& dateString) {
124+ QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss");
125+ dt.setTimeSpec(Qt::UTC);
126+ return dt;
127+ }
128+}
129+
130+bool BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) {
131+ QDateTime lastModified = headers.contains("last-modified") ?
132+ parseHttpDate(headers["last-modified"].toString()) :
133+ QDateTime::currentDateTimeUtc();
134+ QDateTime expirationDate = headers.contains("expires") ?
135+ parseHttpDate(headers["expires"].toString()) :
136+ QDateTime(); // never expires
137+ QNetworkCacheMetaData metaData;
138+ metaData.setUrl(url);
139+ metaData.setSaveToDisk(true);
140+ metaData.setLastModified(lastModified);
141+ metaData.setExpirationDate(expirationDate);
142+ if (auto ioDevice = _cache.prepare(metaData)) {
143+ ioDevice->write(data);
144+ _cache.insert(ioDevice);
145+ qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)";
146+ return true;
147+ }
148+ qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache.";
149+ return false;
150+}
151+
152+Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) {
153+ auto hash = AssetUtils::extractAssetHash(asset);
154+ auto url = AssetUtils::getATPUrl(hash).toString();
155+
156+ QVariantMap metaData = {
157+ { "_asset", asset },
158+ { "_type", "download" },
159+ { "hash", hash },
160+ { "url", url },
161+ { "responseType", responseType },
162+ };
163+
164+ Promise fetched = makePromise("loadAsset::fetched"),
165+ loaded = makePromise("loadAsset::loaded");
166+
167+ downloadBytes(hash)
168+ ->mixin(metaData)
169+ ->ready([=](QString error, QVariantMap result) {
170+ Q_ASSERT(thread() == QThread::currentThread());
171+ fetched->mixin(result);
172+ if (decompress) {
173+ qDebug() << "loadAsset::decompressBytes...";
174+ decompressBytes(result.value("data").toByteArray())
175+ ->mixin(result)
176+ ->ready([=](QString error, QVariantMap result) {
177+ fetched->handle(error, result);
178+ });
179+ } else {
180+ fetched->handle(error, result);
181+ }
182+ });
183+
184+ fetched->ready([=](QString error, QVariantMap result) {
185+ qDebug() << "loadAsset::fetched" << error;
186+ if (responseType == "arraybuffer") {
187+ loaded->resolve(NoError, result);
188+ } else {
189+ convertBytes(result.value("data").toByteArray(), responseType)
190+ ->mixin(result)
191+ ->ready([=](QString error, QVariantMap result) {
192+ loaded->resolve(NoError, result);
193+ });
194+ }
195+ });
196+
197+ return loaded;
198+}
199+
200+Promise BaseAssetScriptingInterface::convertBytes(const QByteArray& dataByteArray, const QString& responseType) {
201+ QVariantMap result;
202+ Promise conversion = makePromise(__FUNCTION__);
203+ if (dataByteArray.size() == 0) {
204+ result["response"] = QString();
205+ } else if (responseType == "text") {
206+ result["response"] = QString::fromUtf8(dataByteArray);
207+ } else if (responseType == "json") {
208+ QJsonParseError status;
209+ auto parsed = QJsonDocument::fromJson(dataByteArray, &status);
210+ if (status.error == QJsonParseError::NoError) {
211+ result["response"] = parsed.isArray() ?
212+ QVariant(parsed.array().toVariantList()) :
213+ QVariant(parsed.object().toVariantMap());
214+ } else {
215+ QVariantMap errorResult = {
216+ { "error", status.error },
217+ { "offset", status.offset },
218+ };
219+ return conversion->reject("JSON Parse Error: " + status.errorString(), errorResult);
220+ }
221+ } else if (responseType == "arraybuffer") {
222+ result["response"] = dataByteArray;
223+ }
224+ return conversion->resolve(NoError, result);
225+}
226+
227+Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteArray) {
228+ QByteArray inflated;
229+ auto start = usecTimestampNow();
230+ if (gunzip(dataByteArray, inflated)) {
231+ auto end = usecTimestampNow();
232+ return makePromise(__FUNCTION__)->resolve(NoError, {
233+ { "_compressedByteLength", dataByteArray.size() },
234+ { "_compressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
235+ { "_compressMS", (double)(end - start) / 1000.0 },
236+ { "decompressed", true },
237+ { "byteLength", inflated.size() },
238+ { "contentType", QMimeDatabase().mimeTypeForData(inflated).name() },
239+ { "data", inflated },
240+ });
241+ } else {
242+ return makePromise(__FUNCTION__)->reject("gunzip error", {});
243+ }
244+}
245+
246+Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArray, int level) {
247+ QByteArray deflated;
248+ auto start = usecTimestampNow();
249+ if (gzip(dataByteArray, deflated, level)) {
250+ auto end = usecTimestampNow();
251+ return makePromise(__FUNCTION__)->resolve(NoError, {
252+ { "_uncompressedByteLength", dataByteArray.size() },
253+ { "_uncompressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
254+ { "_compressMS", (double)(end - start) / 1000.0 },
255+ { "compressed", true },
256+ { "byteLength", deflated.size() },
257+ { "contentType", QMimeDatabase().mimeTypeForData(deflated).name() },
258+ { "data", deflated },
259+ });
260+ } else {
261+ return makePromise(__FUNCTION__)->reject("gzip error", {});
262+ }
263+}
264+
265+Promise BaseAssetScriptingInterface::downloadBytes(QString hash) {
266+ auto assetClient = DependencyManager::get<AssetClient>();
267+ QPointer<AssetRequest> assetRequest = assetClient->createRequest(hash);
268+ Promise deferred = makePromise(__FUNCTION__);
269+
270+ QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) {
271+ qDebug() << "...BaseAssetScriptingInterface::downloadBytes" << request->getErrorString();
272+ // note: we are now on the "Resource Manager" thread
273+ Q_ASSERT(QThread::currentThread() == request->thread());
274+ Q_ASSERT(request->getState() == AssetRequest::Finished);
275+ QString error;
276+ QVariantMap result;
277+ if (request->getError() == AssetRequest::Error::NoError) {
278+ QByteArray data = request->getData();
279+ result = {
280+ { "url", request->getUrl() },
281+ { "hash", request->getHash() },
282+ { "cached", request->loadedFromCache() },
283+ { "content-type", QMimeDatabase().mimeTypeForData(data).name() },
284+ { "data", data },
285+ };
286+ } else {
287+ error = request->getError();
288+ result = { { "error", request->getError() } };
289+ }
290+ qDebug() << "//BaseAssetScriptingInterface::downloadBytes" << error << result.keys();
291+ // forward thread-safe copies back to our thread
292+ deferred->handle(error, result);
293+ request->deleteLater();
294+ });
295+ assetRequest->start();
296+ return deferred;
297+}
298+
299+Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) {
300+ Promise deferred = makePromise(__FUNCTION__);
301+ QPointer<AssetUpload> upload = DependencyManager::get<AssetClient>()->createUpload(bytes);
302+
303+ QObject::connect(upload, &AssetUpload::finished, upload, [=](AssetUpload* upload, const QString& hash) {
304+ Q_ASSERT(QThread::currentThread() == upload->thread());
305+ // note: we are now on the "Resource Manager" thread
306+ QString error;
307+ QVariantMap result;
308+ if (upload->getError() == AssetUpload::NoError) {
309+ result = {
310+ { "hash", hash },
311+ { "url", AssetUtils::getATPUrl(hash).toString() },
312+ { "filename", upload->getFilename() },
313+ };
314+ } else {
315+ error = upload->getErrorString();
316+ result = { { "error", upload->getError() } };
317+ }
318+ // forward thread-safe copies back to our thread
319+ deferred->handle(error, result);
320+ upload->deleteLater();
321+ });
322+ upload->start();
323+ return deferred;
324+}
325+
326+Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) {
327+ auto deferred = makePromise(__FUNCTION__);
328+ auto url = AssetUtils::getATPUrl(asset);
329+ auto path = url.path();
330+ auto hash = AssetUtils::extractAssetHash(asset);
331+ if (AssetUtils::isValidHash(hash)) {
332+ // already a valid ATP hash -- nothing to do
333+ deferred->resolve(NoError, {
334+ { "hash", hash },
335+ { "path", path },
336+ { "url", url },
337+ });
338+ } else if (AssetUtils::isValidFilePath(path)) {
339+ auto assetClient = DependencyManager::get<AssetClient>();
340+ QPointer<GetMappingRequest> request = assetClient->createGetMappingRequest(path);
341+
342+ QObject::connect(request, &GetMappingRequest::finished, request, [=]() {
343+ Q_ASSERT(QThread::currentThread() == request->thread());
344+ // note: we are now on the "Resource Manager" thread
345+ QString error;
346+ QVariantMap result;
347+ if (request->getError() == GetMappingRequest::NoError) {
348+ result = {
349+ { "_hash", hash },
350+ { "_path", path },
351+ { "_url", url },
352+ { "hash", request->getHash() },
353+ { "wasRedirected", request->wasRedirected() },
354+ { "path", request->wasRedirected() ? request->getRedirectedPath() : path },
355+ };
356+ } else {
357+ error = request->getErrorString();
358+ result = { { "error", request->getError() } };
359+ }
360+ // forward thread-safe copies back to our thread
361+ deferred->handle(error, result);
362+ request->deleteLater();
363+ });
364+ request->start();
365+ } else {
366+ deferred->reject("invalid ATP file path: " + asset + "("+path+")", {});
367+ }
368+ return deferred;
369+}
370+
371+Promise BaseAssetScriptingInterface::symlinkAsset(QString hash, QString path) {
372+ auto deferred = makePromise(__FUNCTION__);
373+ auto assetClient = DependencyManager::get<AssetClient>();
374+ QPointer<SetMappingRequest> setMappingRequest = assetClient->createSetMappingRequest(path, hash);
375+
376+ connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [=](SetMappingRequest* request) {
377+ Q_ASSERT(QThread::currentThread() == request->thread());
378+ // we are now on the "Resource Manager" thread
379+ QString error;
380+ QVariantMap result;
381+ if (request->getError() == SetMappingRequest::NoError) {
382+ result = {
383+ { "_hash", hash },
384+ { "_path", path },
385+ { "hash", request->getHash() },
386+ { "path", request->getPath() },
387+ { "url", AssetUtils::getATPUrl(request->getPath()).toString() },
388+ };
389+ } else {
390+ error = request->getErrorString();
391+ result = { { "error", request->getError() } };
392+ }
393+ // forward results back to our thread
394+ deferred->handle(error, result);
395+ request->deleteLater();
396+ });
397+ setMappingRequest->start();
398+ return deferred;
399+}