History

expanded Assets scripting interface

humbletim committed 8 years ago

Commit 8c7a8f01 parent 15fcf6612 files · +1090 −47Browse files

Showing the changes to libraries/networking/src/BaseAssetScriptingInterface.cpp (1 of 12 files). Show all

Unified · Split

libraries/networking/src/BaseAssetScriptingInterface.cpp+399−0
@@ -0,0 +1,399 @@
1//
2// BaseAssetScriptingInterface.cpp
3// libraries/networking/src
4//
5// Copyright 2017 High Fidelity, Inc.
6//
7// Distributed under the Apache License, Version 2.0.
8// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
9
10#include "BaseAssetScriptingInterface.h"
11
12#include <QJsonDocument>
13#include <QJsonArray>
14#include <QMimeDatabase>
15#include <QThread>
16
17#include "AssetRequest.h"
18#include "AssetUpload.h"
19#include "AssetUtils.h"
20#include "MappingRequest.h"
21#include "NetworkLogging.h"
22
23#include <RegisteredMetaTypes.h>
24
25#include <shared/QtHelpers.h>
26#include "Gzip.h"
27
28using Promise = MiniPromise::Promise;
29
30QSharedPointer<AssetClient> BaseAssetScriptingInterface::assetClient() {
31 return DependencyManager::get<AssetClient>();
32}
33
34BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent), _cache(this) {
35}
36
37
38bool BaseAssetScriptingInterface::initializeCache() {
39 qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- current values" << _cache.cacheDirectory() << _cache.cacheSize() << _cache.maximumCacheSize();
40
41 // NOTE: *instances* of QNetworkDiskCache are not thread-safe -- however, different threads can effectively
42 // use the same underlying cache if configured with identical settings. Once AssetClient's disk cache settings
43 // become available we configure our instance to match.
44 auto assets = assetClient();
45 if (!assets) {
46 return false; // not yet possible to initialize the cache
47 }
48 if (_cache.cacheDirectory().size()) {
49 return true; // cache is ready
50 }
51
52 // attempt to initialize the cache
53 qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- invoking AssetClient::init" << assetClient().data();
54 QMetaObject::invokeMethod(assetClient().data(), "init");
55
56 qDebug() << "BaseAssetScriptingInterface::getCacheStatus querying cache status" << QThread::currentThread();
57 Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus");
58 deferred->then([&](QVariantMap result) {
59 qDebug() << "//queryCacheStatus" << QThread::currentThread();
60 auto cacheDirectory = result.value("cacheDirectory").toString();
61 auto cacheSize = result.value("cacheSize").toLongLong();
62 auto maximumCacheSize = result.value("maximumCacheSize").toLongLong();
63 qDebug() << "///queryCacheStatus" << cacheDirectory << cacheSize << maximumCacheSize;
64 _cache.setCacheDirectory(cacheDirectory);
65 _cache.setMaximumCacheSize(maximumCacheSize);
66 });
67 deferred->fail([&](QString error) {
68 qDebug() << "//queryCacheStatus ERROR" << QThread::currentThread() << error;
69 });
70 assets->cacheInfoRequest(deferred);
71 return false; // cache is not ready yet
72}
73
74QVariantMap BaseAssetScriptingInterface::getCacheStatus() {
75 //assetClient()->cacheInfoRequest(this, "onCacheInfoResponse");
76 return {
77 { "cacheDirectory", _cache.cacheDirectory() },
78 { "cacheSize", _cache.cacheSize() },
79 { "maximumCacheSize", _cache.maximumCacheSize() },
80 };
81}
82
83QVariantMap BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) {
84 QNetworkCacheMetaData metaData = _cache.metaData(url);
85 QVariantMap attributes, rawHeaders;
86
87 QHashIterator<QNetworkRequest::Attribute, QVariant> i(metaData.attributes());
88 while (i.hasNext()) {
89 i.next();
90 attributes[QString::number(i.key())] = i.value();
91 }
92 for (const auto& i : metaData.rawHeaders()) {
93 rawHeaders[i.first] = i.second;
94 }
95 return {
96 { "isValid", metaData.isValid() },
97 { "url", metaData.url() },
98 { "expirationDate", metaData.expirationDate() },
99 { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() },
100 { "saveToDisk", metaData.saveToDisk() },
101 { "attributes", attributes },
102 { "rawHeaders", rawHeaders },
103 };
104}
105
106QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) {
107 qDebug() << "loadFromCache" << url;
108 QVariantMap result = {
109 { "metadata", queryCacheMeta(url) },
110 { "data", QByteArray() },
111 };
112 // caller is responsible for the deletion of the ioDevice, hence the unique_ptr
113 if (auto ioDevice = std::unique_ptr<QIODevice>(_cache.data(url))) {
114 QByteArray data = ioDevice->readAll();
115 qCDebug(asset_client) << url.toDisplayString() << "loaded from disk cache (" << data.size() << " bytes)";
116 result["data"] = data;
117 }
118 return result;
119}
120
121namespace {
122 // parse RFC 1123 HTTP date format
123 QDateTime parseHttpDate(const QString& dateString) {
124 QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss");
125 dt.setTimeSpec(Qt::UTC);
126 return dt;
127 }
128}
129
130bool BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) {
131 QDateTime lastModified = headers.contains("last-modified") ?
132 parseHttpDate(headers["last-modified"].toString()) :
133 QDateTime::currentDateTimeUtc();
134 QDateTime expirationDate = headers.contains("expires") ?
135 parseHttpDate(headers["expires"].toString()) :
136 QDateTime(); // never expires
137 QNetworkCacheMetaData metaData;
138 metaData.setUrl(url);
139 metaData.setSaveToDisk(true);
140 metaData.setLastModified(lastModified);
141 metaData.setExpirationDate(expirationDate);
142 if (auto ioDevice = _cache.prepare(metaData)) {
143 ioDevice->write(data);
144 _cache.insert(ioDevice);
145 qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)";
146 return true;
147 }
148 qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache.";
149 return false;
150}
151
152Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) {
153 auto hash = AssetUtils::extractAssetHash(asset);
154 auto url = AssetUtils::getATPUrl(hash).toString();
155
156 QVariantMap metaData = {
157 { "_asset", asset },
158 { "_type", "download" },
159 { "hash", hash },
160 { "url", url },
161 { "responseType", responseType },
162 };
163
164 Promise fetched = makePromise("loadAsset::fetched"),
165 loaded = makePromise("loadAsset::loaded");
166
167 downloadBytes(hash)
168 ->mixin(metaData)
169 ->ready([=](QString error, QVariantMap result) {
170 Q_ASSERT(thread() == QThread::currentThread());
171 fetched->mixin(result);
172 if (decompress) {
173 qDebug() << "loadAsset::decompressBytes...";
174 decompressBytes(result.value("data").toByteArray())
175 ->mixin(result)
176 ->ready([=](QString error, QVariantMap result) {
177 fetched->handle(error, result);
178 });
179 } else {
180 fetched->handle(error, result);
181 }
182 });
183
184 fetched->ready([=](QString error, QVariantMap result) {
185 qDebug() << "loadAsset::fetched" << error;
186 if (responseType == "arraybuffer") {
187 loaded->resolve(NoError, result);
188 } else {
189 convertBytes(result.value("data").toByteArray(), responseType)
190 ->mixin(result)
191 ->ready([=](QString error, QVariantMap result) {
192 loaded->resolve(NoError, result);
193 });
194 }
195 });
196
197 return loaded;
198}
199
200Promise BaseAssetScriptingInterface::convertBytes(const QByteArray& dataByteArray, const QString& responseType) {
201 QVariantMap result;
202 Promise conversion = makePromise(__FUNCTION__);
203 if (dataByteArray.size() == 0) {
204 result["response"] = QString();
205 } else if (responseType == "text") {
206 result["response"] = QString::fromUtf8(dataByteArray);
207 } else if (responseType == "json") {
208 QJsonParseError status;
209 auto parsed = QJsonDocument::fromJson(dataByteArray, &status);
210 if (status.error == QJsonParseError::NoError) {
211 result["response"] = parsed.isArray() ?
212 QVariant(parsed.array().toVariantList()) :
213 QVariant(parsed.object().toVariantMap());
214 } else {
215 QVariantMap errorResult = {
216 { "error", status.error },
217 { "offset", status.offset },
218 };
219 return conversion->reject("JSON Parse Error: " + status.errorString(), errorResult);
220 }
221 } else if (responseType == "arraybuffer") {
222 result["response"] = dataByteArray;
223 }
224 return conversion->resolve(NoError, result);
225}
226
227Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteArray) {
228 QByteArray inflated;
229 auto start = usecTimestampNow();
230 if (gunzip(dataByteArray, inflated)) {
231 auto end = usecTimestampNow();
232 return makePromise(__FUNCTION__)->resolve(NoError, {
233 { "_compressedByteLength", dataByteArray.size() },
234 { "_compressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
235 { "_compressMS", (double)(end - start) / 1000.0 },
236 { "decompressed", true },
237 { "byteLength", inflated.size() },
238 { "contentType", QMimeDatabase().mimeTypeForData(inflated).name() },
239 { "data", inflated },
240 });
241 } else {
242 return makePromise(__FUNCTION__)->reject("gunzip error", {});
243 }
244}
245
246Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArray, int level) {
247 QByteArray deflated;
248 auto start = usecTimestampNow();
249 if (gzip(dataByteArray, deflated, level)) {
250 auto end = usecTimestampNow();
251 return makePromise(__FUNCTION__)->resolve(NoError, {
252 { "_uncompressedByteLength", dataByteArray.size() },
253 { "_uncompressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
254 { "_compressMS", (double)(end - start) / 1000.0 },
255 { "compressed", true },
256 { "byteLength", deflated.size() },
257 { "contentType", QMimeDatabase().mimeTypeForData(deflated).name() },
258 { "data", deflated },
259 });
260 } else {
261 return makePromise(__FUNCTION__)->reject("gzip error", {});
262 }
263}
264
265Promise BaseAssetScriptingInterface::downloadBytes(QString hash) {
266 auto assetClient = DependencyManager::get<AssetClient>();
267 QPointer<AssetRequest> assetRequest = assetClient->createRequest(hash);
268 Promise deferred = makePromise(__FUNCTION__);
269
270 QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) {
271 qDebug() << "...BaseAssetScriptingInterface::downloadBytes" << request->getErrorString();
272 // note: we are now on the "Resource Manager" thread
273 Q_ASSERT(QThread::currentThread() == request->thread());
274 Q_ASSERT(request->getState() == AssetRequest::Finished);
275 QString error;
276 QVariantMap result;
277 if (request->getError() == AssetRequest::Error::NoError) {
278 QByteArray data = request->getData();
279 result = {
280 { "url", request->getUrl() },
281 { "hash", request->getHash() },
282 { "cached", request->loadedFromCache() },
283 { "content-type", QMimeDatabase().mimeTypeForData(data).name() },
284 { "data", data },
285 };
286 } else {
287 error = request->getError();
288 result = { { "error", request->getError() } };
289 }
290 qDebug() << "//BaseAssetScriptingInterface::downloadBytes" << error << result.keys();
291 // forward thread-safe copies back to our thread
292 deferred->handle(error, result);
293 request->deleteLater();
294 });
295 assetRequest->start();
296 return deferred;
297}
298
299Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) {
300 Promise deferred = makePromise(__FUNCTION__);
301 QPointer<AssetUpload> upload = DependencyManager::get<AssetClient>()->createUpload(bytes);
302
303 QObject::connect(upload, &AssetUpload::finished, upload, [=](AssetUpload* upload, const QString& hash) {
304 Q_ASSERT(QThread::currentThread() == upload->thread());
305 // note: we are now on the "Resource Manager" thread
306 QString error;
307 QVariantMap result;
308 if (upload->getError() == AssetUpload::NoError) {
309 result = {
310 { "hash", hash },
311 { "url", AssetUtils::getATPUrl(hash).toString() },
312 { "filename", upload->getFilename() },
313 };
314 } else {
315 error = upload->getErrorString();
316 result = { { "error", upload->getError() } };
317 }
318 // forward thread-safe copies back to our thread
319 deferred->handle(error, result);
320 upload->deleteLater();
321 });
322 upload->start();
323 return deferred;
324}
325
326Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) {
327 auto deferred = makePromise(__FUNCTION__);
328 auto url = AssetUtils::getATPUrl(asset);
329 auto path = url.path();
330 auto hash = AssetUtils::extractAssetHash(asset);
331 if (AssetUtils::isValidHash(hash)) {
332 // already a valid ATP hash -- nothing to do
333 deferred->resolve(NoError, {
334 { "hash", hash },
335 { "path", path },
336 { "url", url },
337 });
338 } else if (AssetUtils::isValidFilePath(path)) {
339 auto assetClient = DependencyManager::get<AssetClient>();
340 QPointer<GetMappingRequest> request = assetClient->createGetMappingRequest(path);
341
342 QObject::connect(request, &GetMappingRequest::finished, request, [=]() {
343 Q_ASSERT(QThread::currentThread() == request->thread());
344 // note: we are now on the "Resource Manager" thread
345 QString error;
346 QVariantMap result;
347 if (request->getError() == GetMappingRequest::NoError) {
348 result = {
349 { "_hash", hash },
350 { "_path", path },
351 { "_url", url },
352 { "hash", request->getHash() },
353 { "wasRedirected", request->wasRedirected() },
354 { "path", request->wasRedirected() ? request->getRedirectedPath() : path },
355 };
356 } else {
357 error = request->getErrorString();
358 result = { { "error", request->getError() } };
359 }
360 // forward thread-safe copies back to our thread
361 deferred->handle(error, result);
362 request->deleteLater();
363 });
364 request->start();
365 } else {
366 deferred->reject("invalid ATP file path: " + asset + "("+path+")", {});
367 }
368 return deferred;
369}
370
371Promise BaseAssetScriptingInterface::symlinkAsset(QString hash, QString path) {
372 auto deferred = makePromise(__FUNCTION__);
373 auto assetClient = DependencyManager::get<AssetClient>();
374 QPointer<SetMappingRequest> setMappingRequest = assetClient->createSetMappingRequest(path, hash);
375
376 connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [=](SetMappingRequest* request) {
377 Q_ASSERT(QThread::currentThread() == request->thread());
378 // we are now on the "Resource Manager" thread
379 QString error;
380 QVariantMap result;
381 if (request->getError() == SetMappingRequest::NoError) {
382 result = {
383 { "_hash", hash },
384 { "_path", path },
385 { "hash", request->getHash() },
386 { "path", request->getPath() },
387 { "url", AssetUtils::getATPUrl(request->getPath()).toString() },
388 };
389 } else {
390 error = request->getErrorString();
391 result = { { "error", request->getError() } };
392 }
393 // forward results back to our thread
394 deferred->handle(error, result);
395 request->deleteLater();
396 });
397 setMappingRequest->start();
398 return deferred;
399}