History

merge require/module changes into clean branch

humbletim committed 9 years ago

Commit 9d860a81 parent cd91d6f4 files · +485 −0Browse files

Showing the changes to libraries/script-engine/src/ScriptEngine.cpp (1 of 4 files). Show all

Unified · Split

libraries/script-engine/src/ScriptEngine.cpp+464−0
@@ -70,6 +70,10 @@ · 69 unmodified lines
7070
7171 #include "MIDIEvent.h"
7272
73+const QString ScriptEngine::_SETTINGS_ENABLE_EXTENDED_MODULE_COMPAT {
74+ "com.highfidelity.experimental.enableExtendedModuleCompatbility"
75+};
76+
7377 static const QScriptEngine::QObjectWrapOptions DEFAULT_QOBJECT_WRAP_OPTIONS =
7478 QScriptEngine::ExcludeDeleteLater | QScriptEngine::ExcludeChildObjects;
7579 static const QScriptValue::PropertyFlags READONLY_PROP_FLAGS { QScriptValue::ReadOnly | QScriptValue::Undeletable };
@@ -532,6 +536,44 @@ static QScriptValue createScriptableResourcePrototype(QScriptEngine* engine) { · 456 unmodified lines
532536 return prototype;
533537 }
534538
539+void ScriptEngine::resetModuleCache(bool deleteScriptCache) {
540+ if (QThread::currentThread() != thread()) {
541+ QMetaObject::invokeMethod(this, "resetModuleCache");
542+ return;
543+ }
544+ {
545+ QMutexLocker locker(&_requireLock);
546+ auto jsRequire = globalObject().property("Script").property("require");
547+ auto cache = jsRequire.property("cache");
548+ auto cacheMeta = jsRequire.data();
549+
550+ if (deleteScriptCache) {
551+ QScriptValueIterator it(cache);
552+ while (it.hasNext()) {
553+ it.next();
554+ if (it.flags() & QScriptValue::SkipInEnumeration)
555+ continue;
556+ //scriptCache->deleteScript(it.name());
557+ qCDebug(scriptengine) << "resetModuleCache(true) -- staging " << it.name() << " for cache reset at next require";
558+ cacheMeta.setProperty(it.name(), true);
559+ }
560+ }
561+ //_debugDump("cacheMeta", cacheMeta);
562+ cache = newObject();
563+ if (!cacheMeta.isObject()) {
564+ cacheMeta = newObject();
565+ cacheMeta.setProperty("id", "Script.require.cacheMeta");
566+ cacheMeta.setProperty("type", "cacheMeta");
567+ jsRequire.setData(cacheMeta);
568+ }
569+ cache.setProperty("__created__", (double)QDateTime::currentMSecsSinceEpoch(), QScriptValue::SkipInEnumeration);
570+#if DEBUG_JS_MODULES
571+ cache.setProperty("__meta__", cacheMeta, HIDDEN_PROP_FLAGS);
572+#endif
573+ jsRequire.setProperty("cache", cache, QScriptValue::ReadOnly | QScriptValue::Undeletable);
574+ }
575+}
576+
535577 void ScriptEngine::init() {
536578 if (_isInitialized) {
537579 return; // only initialize once
@@ -595,6 +637,15 @@ void ScriptEngine::init() { · 57 unmodified lines
595637
596638 registerGlobalObject("Script", this);
597639
640+ {
641+ // set up Script.require.resolve and Script.require.cache
642+ auto Script = globalObject().property("Script");
643+ auto require = Script.property("require");
644+ auto resolve = Script.property("_requireResolve");
645+ require.setProperty("resolve", resolve, QScriptValue::ReadOnly | QScriptValue::Undeletable);
646+ resetModuleCache();
647+ }
648+
598649 registerGlobalObject("Audio", &AudioScriptingInterface::getInstance());
599650 registerGlobalObject("Entities", entityScriptingInterface.data());
600651 registerGlobalObject("Quat", &_quatLibrary);
@@ -1304,6 +1355,419 @@ void ScriptEngine::print(const QString& message) { · 703 unmodified lines
13041355 emit printedMessage(message);
13051356 }
13061357
1358+static const auto MAX_MODULE_IDENTIFIER_LEN { 4096 };
1359+static const auto MAX_MODULE_IDENTIFIER_LOG_LEN { 60 };
1360+
1361+// Script.require.resolve -- like resolvePath, but performs more validation and throws exceptions on invalid module identifiers (for consistency with Node.js)
1362+QString ScriptEngine::_requireResolve(const QString& moduleId, const QString& relativeTo) {
1363+ QUrl defaultScriptsLoc = defaultScriptsLocation();
1364+ QUrl url(moduleId);
1365+
1366+ // helper to generate an exception and return a null string
1367+ auto resolverException = [=](const QString& detail) -> QString {
1368+ currentContext()->throwError(
1369+ QString("Cannot find module '%1' (%2)")
1370+ .arg(moduleId.left(MAX_MODULE_IDENTIFIER_LOG_LEN))
1371+ .arg(detail));
1372+ return QString();
1373+ };
1374+
1375+ // de-fuzz the input a little by restricting to rational sizes
1376+ auto idLength = url.toString().length();
1377+ if (idLength < 1 || idLength > MAX_MODULE_IDENTIFIER_LEN) {
1378+ return resolverException(
1379+ QString("rejecting invalid module id size (%1 chars [1,%2])")
1380+ .arg(idLength).arg(MAX_MODULE_IDENTIFIER_LEN)
1381+ );
1382+ }
1383+
1384+ // this regex matches: absolute, dotted or path-like URLs
1385+ // (ie: the kind of stuff ScriptEngine::resolvePath already handles)
1386+ QRegularExpression qualified ("^\\w+:|^/|^[.]{1,2}(/|$)");
1387+
1388+ // this is for module.require (which is a bound version of require that's always relative to the module path)
1389+ if (!relativeTo.isEmpty()) {
1390+ url = QUrl(relativeTo).resolved(moduleId);
1391+ url = resolvePath(url.toString());
1392+ } else if (qualified.match(moduleId).hasMatch()) {
1393+ url = resolvePath(moduleId);
1394+ } else {
1395+ // check if the moduleId refers to a "system" module
1396+ QString defaultsPath = defaultScriptsLoc.path();
1397+ QString systemModulePath = QString("%1/modules/%2.js").arg(defaultsPath).arg(moduleId);
1398+ url = defaultScriptsLoc;
1399+ url.setPath(systemModulePath);
1400+ if (!QFileInfo(url.toLocalFile()).isFile()) {
1401+ return resolverException("system module not found");
1402+ }
1403+ }
1404+
1405+ if (url.isRelative()) {
1406+ return resolverException("could not resolve module id");
1407+ }
1408+
1409+ // if it looks like a local file, verify that it's an allowed path and really a file
1410+ if (url.isLocalFile()) {
1411+ QFileInfo file(url.toLocalFile());
1412+ QUrl canonical = url;
1413+ if (file.exists()) {
1414+ canonical.setPath(file.canonicalFilePath());
1415+ }
1416+
1417+ bool disallowOutsideFiles = !defaultScriptsLocation().isParentOf(canonical) && !currentSandboxURL.isLocalFile();
1418+ if (disallowOutsideFiles && !PathUtils::isDescendantOf(canonical, currentSandboxURL)) {
1419+ return resolverException(
1420+ QString("path '%1' outside of origin script '%2' '%3'")
1421+ .arg(PathUtils::stripFilename(url))
1422+ .arg(PathUtils::stripFilename(currentSandboxURL))
1423+ .arg(canonical.toString())
1424+ );
1425+ }
1426+ if (!file.exists()) {
1427+ return resolverException("path does not exist: " + url.toLocalFile());
1428+ }
1429+ if (!file.isFile()) {
1430+ return resolverException("path is not a file: " + url.toLocalFile());
1431+ }
1432+ }
1433+
1434+ return url.toString();
1435+}
1436+
1437+// retrieves the current parent module from the JS scope chain
1438+QScriptValue ScriptEngine::currentModule() {
1439+ auto jsRequire = globalObject().property("Script").property("require");
1440+ auto cache = jsRequire.property("cache");
1441+ auto candidate = QScriptValue();
1442+ for(auto ctx = currentContext(); ctx && !candidate.isObject(); ctx = ctx->parentContext()) {
1443+ QScriptContextInfo contextInfo { ctx };
1444+ candidate = cache.property(contextInfo.fileName());
1445+ }
1446+ if (!candidate.isObject()) {
1447+ return QScriptValue();
1448+ }
1449+ return candidate;
1450+}
1451+
1452+// replaces or adds "module" to "parent.children[]" array
1453+// (for consistency with Node.js and userscript cache invalidation without "cache busters")
1454+bool ScriptEngine::registerModuleWithParent(const QScriptValue& module, const QScriptValue& parent) {
1455+ auto children = parent.property("children");
1456+ if (children.isArray()) {
1457+ auto key = module.property("id");
1458+ auto length = children.property("length").toInt32();
1459+ for(int i=0; i < length; i++) {
1460+ if (children.property(i).property("id").strictlyEquals(key)) {
1461+ qCDebug(scriptengine_module) << key.toString() << " updating parent.children[" << i << "] = module";
1462+ children.setProperty(i, module);
1463+ return true;
1464+ }
1465+ }
1466+ qCDebug(scriptengine_module) << key.toString() << " appending parent.children[" << length << "] = module";
1467+ children.setProperty(length, module);
1468+ return true;
1469+ } else if (parent.isValid()) {
1470+ qCDebug(scriptengine_module) << "registerModuleWithParent -- unrecognized parent" << parent.toVariant().toString();
1471+ }
1472+ return false;
1473+}
1474+
1475+// creates a new JS "module" Object with default metadata properties
1476+QScriptValue ScriptEngine::newModule(const QString& modulePath, const QScriptValue& parent) {
1477+ auto closure = newObject();
1478+ auto exports = newObject();
1479+ auto module = newObject();
1480+ qCDebug(scriptengine_module) << "newModule" << modulePath << parent.property("filename").toString();
1481+
1482+ closure.setProperty("module", module, READONLY_PROP_FLAGS);
1483+
1484+ // note: this becomes the "exports" free variable, so should not be set read only
1485+ closure.setProperty("exports", exports);
1486+
1487+ // make the closure available to module instantiation
1488+ module.setProperty("__closure__", closure, HIDDEN_PROP_FLAGS);
1489+
1490+ // for consistency with Node.js Module
1491+ module.setProperty("id", modulePath, READONLY_PROP_FLAGS);
1492+ module.setProperty("filename", modulePath, READONLY_PROP_FLAGS);
1493+ module.setProperty("exports", exports); // not readonly
1494+ module.setProperty("loaded", false, READONLY_PROP_FLAGS);
1495+ module.setProperty("parent", parent, READONLY_PROP_FLAGS);
1496+ module.setProperty("children", newArray(), READONLY_PROP_FLAGS);
1497+
1498+ // module.require is a bound version of require that always resolves relative to that module's path
1499+ auto boundRequire = QScriptEngine::evaluate("(function(id) { return Script.require(Script.require.resolve(id, this.filename)); })", "(boundRequire)");
1500+ module.setProperty("require", boundRequire, READONLY_PROP_FLAGS);
1501+
1502+ return module;
1503+}
1504+
1505+// synchronously fetch a module's source code using BatchLoader
1506+QScriptValue ScriptEngine::fetchModuleSource(const QString& modulePath, const bool forceDownload) {
1507+ using UrlMap = QMap<QUrl, QString>;
1508+ auto scriptCache = DependencyManager::get<ScriptCache>();
1509+ QScriptValue req = newObject();
1510+ qCDebug(scriptengine_module) << "require.fetchModuleSource: " << QUrl(modulePath).fileName() << QThread::currentThread();
1511+
1512+ auto onload = [=, &req](const UrlMap& data, const UrlMap& _status) {
1513+ auto url = modulePath;
1514+ auto status = _status[url];
1515+ auto contents = data[url];
1516+ qCDebug(scriptengine_module) << "require.fetchModuleSource.onload: " << QUrl(url).fileName() << status << QThread::currentThread();
1517+ if (isStopping()) {
1518+ req.setProperty("status", "Stopped");
1519+ req.setProperty("success", false);
1520+ } else {
1521+ req.setProperty("url", url);
1522+ req.setProperty("status", status);
1523+ req.setProperty("success", ScriptCache::isSuccessStatus(status));
1524+ req.setProperty("contents", contents, HIDDEN_PROP_FLAGS);
1525+ }
1526+ };
1527+
1528+ if (forceDownload) {
1529+ qCDebug(scriptengine_module) << "require.requestScript -- clearing cache for" << modulePath;
1530+ scriptCache->deleteScript(modulePath);
1531+ }
1532+ BatchLoader* loader = new BatchLoader({ modulePath });
1533+ connect(loader, &BatchLoader::finished, this, onload);
1534+ connect(this, &QObject::destroyed, loader, &QObject::deleteLater);
1535+ // fail faster? (since require() blocks the engine thread while resolving dependencies)
1536+ const int MAX_RETRIES = 1;
1537+
1538+ loader->start(MAX_RETRIES);
1539+
1540+ if (!loader->isFinished()) {
1541+ QTimer monitor;
1542+ QEventLoop loop;
1543+ QObject::connect(loader, &BatchLoader::finished, this, [this, &monitor, &loop]{
1544+ monitor.stop();
1545+ loop.quit();
1546+ });
1547+
1548+ // this helps detect the case where stop() is invoked during the download
1549+ // but not seen in time to abort processing in onload()...
1550+ connect(&monitor, &QTimer::timeout, this, [this, &loop, &loader]{
1551+ if (isStopping()) {
1552+ loop.exit(-1);
1553+ }
1554+ });
1555+ monitor.start(500);
1556+ loop.exec();
1557+ }
1558+ loader->deleteLater();
1559+ return req;
1560+}
1561+
1562+// evaluate a pending module object using the fetched source code
1563+QScriptValue ScriptEngine::instantiateModule(const QScriptValue& module, const QString& sourceCode) {
1564+ QScriptValue result;
1565+ auto modulePath = module.property("filename").toString();
1566+ auto closure = module.property("__closure__");
1567+
1568+ qCDebug(scriptengine_module) << QString("require.instantiateModule: %1 / %2 bytes").arg(QUrl(modulePath).fileName()).arg(sourceCode.length());
1569+
1570+ {
1571+ ExceptionEmitter tryCatch(this, __FUNCTION__);
1572+
1573+ if (module.property("content-type").toString() == "application/json") {
1574+ qCDebug(scriptengine_module) << "... parsing as JSON";
1575+#ifdef DEV_OTHER_MODULE_JSON_PARSE
1576+ auto JSON = globalObject().property("JSON");
1577+ auto parse = JSON.property("parse");
1578+ if (!parse.isFunction()) {
1579+ currentContext()->throwValue(makeError("global JSON.parse is not a function", "EvalError"));
1580+ return nullValue();
1581+ }
1582+ result = parse.call(JSON, QScriptValueList({ sourceCode }));
1583+ closure.property("module").setProperty("exports", result);
1584+ return result;
1585+#endif
1586+ closure.setProperty("__json", sourceCode);
1587+ result = evaluateInClosure(closure, { "module.exports = JSON.parse(__json)", modulePath });
1588+ } else {
1589+ // scoped vars for consistency with Node.js
1590+ closure.setProperty("require", module.property("require"));
1591+ closure.setProperty("__filename", modulePath, HIDDEN_PROP_FLAGS);
1592+ closure.setProperty("__dirname", QString(modulePath).replace(QRegExp("/[^/]*$"), ""), HIDDEN_PROP_FLAGS);
1593+ result = evaluateInClosure(closure, { sourceCode, modulePath });
1594+ }
1595+ }
1596+ return result;
1597+}
1598+
1599+// CommonJS/Node.js like require/module support
1600+QScriptValue ScriptEngine::require(const QString& moduleId) {
1601+ qCDebug(scriptengine_module) << "ScriptEngine::require(" << moduleId.left(MAX_MODULE_IDENTIFIER_LOG_LEN) << ")";
1602+ if (QThread::currentThread() != thread()) {
1603+ qCDebug(scriptengine_module) << moduleId << " threads mismatch";
1604+ return nullValue();
1605+ }
1606+
1607+ // serialize require calls so the ordering/caching works correctly across multiple Entities
1608+ // note: this is a Recursive mutex so nested require's should not affected
1609+ QMutexLocker locker(&_requireLock);
1610+ // _requireDepth++;
1611+ // QObject autoDecrement;connect(&autoDecrement, &QObject::destroyed, this, [this](){ _requireDepth--; });
1612+
1613+ auto jsRequire = globalObject().property("Script").property("require");
1614+ auto cacheMeta = jsRequire.data();
1615+ auto cache = jsRequire.property("cache");
1616+ auto parent = currentModule();
1617+
1618+ auto throwModuleError = [this, &cache](const QString& modulePath, const QScriptValue& error) {
1619+ cache.setProperty(modulePath, nullValue());
1620+ if (!error.isNull()) {
1621+#ifdef DEBUG_JS_MODULES
1622+ qCWarning(scriptengine_module) << "throwing module error:" << error.toString() << modulePath << error.property("stack").toString();
1623+#endif
1624+ currentContext()->throwValue(makeError(error));
1625+ }
1626+ return nullValue();
1627+ };
1628+
1629+ // start by resolving the moduleId into a fully-qualified path/URL
1630+ QString modulePath = _requireResolve(moduleId);
1631+ if (hasUncaughtException()) {
1632+ // the resolver already threw an exception -- bail early
1633+ return nullValue();
1634+ }
1635+
1636+ // check the resolved path against the cache
1637+ auto module = cache.property(modulePath);
1638+
1639+ // modules get cached in `Script.require.cache` and (similar to Node.js) users can access it
1640+ // to inspect particular entries and invalidate them by deleting the key:
1641+ // `delete Script.require.cache[Script.require.resolve(moduleId)];`
1642+
1643+ // cacheMeta is just used right now to tell deleted keys apart from undefined ones
1644+ bool invalidateCache = module.isUndefined() && cacheMeta.property(moduleId).isValid();
1645+
1646+ // reset the cacheMeta record so invalidation won't apply next time, even if the module fails to load
1647+ cacheMeta.setProperty(modulePath, QScriptValue());
1648+
1649+ auto exports = module.property("exports");
1650+ if (!invalidateCache && exports.isObject()) {
1651+ // we have found a cacheed module -- just need to possibly register it with current parent
1652+ qCDebug(scriptengine_module) << QString("require - using cached module '%1' for '%2' (loaded: %3)")
1653+ .arg(modulePath).arg(moduleId).arg(module.property("loaded").toString());
1654+ registerModuleWithParent(module, parent);
1655+ return exports;
1656+ }
1657+
1658+ // bootstrap / register new empty module
1659+ module = newModule(modulePath, parent);
1660+ registerModuleWithParent(module, parent);
1661+
1662+ // add it to the cache (this is done early so any cyclic dependencies pick up)
1663+ cache.setProperty(modulePath, module);
1664+
1665+ // download the module source
1666+ auto req = fetchModuleSource(modulePath, invalidateCache);
1667+
1668+ if (!req.property("success").toBool()) {
1669+ auto error = QString("error retrieving script (%1)").arg(req.property("status").toString());
1670+ return throwModuleError(modulePath, error);
1671+ }
1672+
1673+#if DEBUG_JS_MODULES
1674+ qCDebug(scriptengine_module) << "require.loaded: " <<
1675+ QUrl(req.property("url").toString()).fileName() << req.property("status").toString();
1676+#endif
1677+
1678+ auto sourceCode = req.property("contents").toString();
1679+
1680+ if (QUrl(modulePath).fileName().endsWith(".json", Qt::CaseInsensitive)) {
1681+ module.setProperty("content-type", "application/json");
1682+ } else {
1683+ module.setProperty("content-type", "application/javascript");
1684+ }
1685+
1686+ {
1687+ // It seems that many JSON sources don't actually put .json in the URL...
1688+ // so for now as a workaround users wanting to indicate JSON parsing mode can
1689+ // do so by calling with a custom this context, eg:
1690+ //
1691+ // var ITEMS_URL = 'https://highfidelity.io/api/v1/marketplace/items';
1692+ // var thisObject = { 'content-type': 'application/json' };
1693+ // var items = Script.require.call(thisObject, ITEMS_URL + '?category=everything&sort=recent');
1694+
1695+ auto thisObject = currentContext()->thisObject();
1696+ bool calledWithCustomThis = thisObject.isObject() &&
1697+ !thisObject.strictlyEquals(globalObject()) &&
1698+ !thisObject.toQObject();
1699+
1700+ if (calledWithCustomThis) {
1701+#ifdef DEBUG_JS
1702+ _debugDump("this", thisObject);
1703+#endif
1704+ _applyUserOptions(module, thisObject);
1705+ }
1706+ }
1707+
1708+ // evaluate the module
1709+ auto result = instantiateModule(module, sourceCode);
1710+
1711+ if (result.isError() && !result.strictlyEquals(module.property("exports"))) {
1712+ qCWarning(scriptengine_module) << "-- result.isError --" << result.toString();
1713+ return throwModuleError(modulePath, result);
1714+ }
1715+
1716+ // mark as fully-loaded
1717+ module.setProperty("loaded", true, READONLY_PROP_FLAGS);
1718+
1719+ // set up a new reference point for detecting cache key deletion
1720+ cacheMeta.setProperty(modulePath, module);
1721+
1722+ qCDebug(scriptengine_module) << "//ScriptEngine::require(" << moduleId << ")";
1723+
1724+ return module.property("exports");
1725+}
1726+
1727+// User-configurable override options
1728+void ScriptEngine::_applyUserOptions(QScriptValue& module, QScriptValue& options) {
1729+ if (!options.isValid()) {
1730+ return;
1731+ }
1732+ // options['content-type'] === 'application/json'
1733+ // -- allows JSON modules to be used from URLs not ending in .json
1734+ if (options.property("content-type").isString()) {
1735+ module.setProperty("content-type", options.property("content-type"));
1736+ qCDebug(scriptengine_module) << "module['content-type'] =" << module.property("content-type").toString();
1737+ }
1738+
1739+ if (ScriptEngine::_enableExtendedModuleCompatbility.get()) {
1740+ auto closure = module.property("__closure__");
1741+
1742+ auto maybeSetToExports = [&](const QString& key) {
1743+ if (options.property(key).toString() == "exports") {
1744+ closure.setProperty(key, module.property("exports"));
1745+ qCDebug(scriptengine_module) << "module.closure[" << key << "] = exports";
1746+ }
1747+ };
1748+
1749+ // options[{key}] = 'exports'
1750+ // several "agnostic" modules in the wild are just one step away from being compatible --
1751+ // they just didn't know not to look specifically for this, self or global for attaching
1752+ // things onto.
1753+ maybeSetToExports("global");
1754+ maybeSetToExports("self");
1755+ maybeSetToExports("this");
1756+
1757+ // when options is an Object it will get used as the value of "this" during module evaluation
1758+ // (which is what one might expect when calling require.call(thisObject, ...))
1759+ if (options.isObject()) {
1760+ closure.setProperty("this", options);
1761+ }
1762+
1763+ // when options.global is an Object it'll get used as the global object (during evaluation only)
1764+ if (options.property("global").isObject()) {
1765+ closure.setProperty("global", options.property("global"));
1766+ qCDebug(scriptengine_module) << "module.closure['global'] = options.global";
1767+ }
1768+ }
1769+}
1770+
13071771 // If a callback is specified, the included files will be loaded asynchronously and the callback will be called
13081772 // when all of the files have finished loading.
13091773 // If no callback is specified, the included files will be loaded synchronously and will block execution until