History

merge require/module changes into clean branch

humbletim committed 9 years ago

Commit 9d860a81 parent cd91d6f4 files · +485 −0Browse files

Showing the changes to libraries/script-engine/src/ScriptEngine.cpp (1 of 4 files). Show all

Unified · Split

libraries/script-engine/src/ScriptEngine.cpp+464−0
@@ -70,6 +70,10 @@ · 69 unmodified lines
70 70
71#include "MIDIEvent.h"71#include "MIDIEvent.h"
72 72
73const QString ScriptEngine::_SETTINGS_ENABLE_EXTENDED_MODULE_COMPAT {
74 "com.highfidelity.experimental.enableExtendedModuleCompatbility"
75};
76
73static const QScriptEngine::QObjectWrapOptions DEFAULT_QOBJECT_WRAP_OPTIONS =77static const QScriptEngine::QObjectWrapOptions DEFAULT_QOBJECT_WRAP_OPTIONS =
74 QScriptEngine::ExcludeDeleteLater | QScriptEngine::ExcludeChildObjects;78 QScriptEngine::ExcludeDeleteLater | QScriptEngine::ExcludeChildObjects;
75static const QScriptValue::PropertyFlags READONLY_PROP_FLAGS { QScriptValue::ReadOnly | QScriptValue::Undeletable };79static const QScriptValue::PropertyFlags READONLY_PROP_FLAGS { QScriptValue::ReadOnly | QScriptValue::Undeletable };
@@ -532,6 +536,44 @@ static QScriptValue createScriptableResourcePrototype(QScriptEngine* engine) { · 456 unmodified lines
532 return prototype;536 return prototype;
533}537}
534 538
539void ScriptEngine::resetModuleCache(bool deleteScriptCache) {
540 if (QThread::currentThread() != thread()) {
541 QMetaObject::invokeMethod(this, "resetModuleCache");
542 return;
543 }
544 {
545 QMutexLocker locker(&_requireLock);
546 auto jsRequire = globalObject().property("Script").property("require");
547 auto cache = jsRequire.property("cache");
548 auto cacheMeta = jsRequire.data();
549
550 if (deleteScriptCache) {
551 QScriptValueIterator it(cache);
552 while (it.hasNext()) {
553 it.next();
554 if (it.flags() & QScriptValue::SkipInEnumeration)
555 continue;
556 //scriptCache->deleteScript(it.name());
557 qCDebug(scriptengine) << "resetModuleCache(true) -- staging " << it.name() << " for cache reset at next require";
558 cacheMeta.setProperty(it.name(), true);
559 }
560 }
561 //_debugDump("cacheMeta", cacheMeta);
562 cache = newObject();
563 if (!cacheMeta.isObject()) {
564 cacheMeta = newObject();
565 cacheMeta.setProperty("id", "Script.require.cacheMeta");
566 cacheMeta.setProperty("type", "cacheMeta");
567 jsRequire.setData(cacheMeta);
568 }
569 cache.setProperty("__created__", (double)QDateTime::currentMSecsSinceEpoch(), QScriptValue::SkipInEnumeration);
570#if DEBUG_JS_MODULES
571 cache.setProperty("__meta__", cacheMeta, HIDDEN_PROP_FLAGS);
572#endif
573 jsRequire.setProperty("cache", cache, QScriptValue::ReadOnly | QScriptValue::Undeletable);
574 }
575}
576
535void ScriptEngine::init() {577void ScriptEngine::init() {
536 if (_isInitialized) {578 if (_isInitialized) {
537 return; // only initialize once579 return; // only initialize once
@@ -595,6 +637,15 @@ void ScriptEngine::init() { · 57 unmodified lines
595 637
596 registerGlobalObject("Script", this);638 registerGlobalObject("Script", this);
597 639
640 {
641 // set up Script.require.resolve and Script.require.cache
642 auto Script = globalObject().property("Script");
643 auto require = Script.property("require");
644 auto resolve = Script.property("_requireResolve");
645 require.setProperty("resolve", resolve, QScriptValue::ReadOnly | QScriptValue::Undeletable);
646 resetModuleCache();
647 }
648
598 registerGlobalObject("Audio", &AudioScriptingInterface::getInstance());649 registerGlobalObject("Audio", &AudioScriptingInterface::getInstance());
599 registerGlobalObject("Entities", entityScriptingInterface.data());650 registerGlobalObject("Entities", entityScriptingInterface.data());
600 registerGlobalObject("Quat", &_quatLibrary);651 registerGlobalObject("Quat", &_quatLibrary);
@@ -1304,6 +1355,419 @@ void ScriptEngine::print(const QString& message) { · 703 unmodified lines
1304 emit printedMessage(message);1355 emit printedMessage(message);
1305}1356}
1306 1357
1358static const auto MAX_MODULE_IDENTIFIER_LEN { 4096 };
1359static const auto MAX_MODULE_IDENTIFIER_LOG_LEN { 60 };
1360
1361// Script.require.resolve -- like resolvePath, but performs more validation and throws exceptions on invalid module identifiers (for consistency with Node.js)
1362QString ScriptEngine::_requireResolve(const QString& moduleId, const QString& relativeTo) {
1363 QUrl defaultScriptsLoc = defaultScriptsLocation();
1364 QUrl url(moduleId);
1365
1366 // helper to generate an exception and return a null string
1367 auto resolverException = [=](const QString& detail) -> QString {
1368 currentContext()->throwError(
1369 QString("Cannot find module '%1' (%2)")
1370 .arg(moduleId.left(MAX_MODULE_IDENTIFIER_LOG_LEN))
1371 .arg(detail));
1372 return QString();
1373 };
1374
1375 // de-fuzz the input a little by restricting to rational sizes
1376 auto idLength = url.toString().length();
1377 if (idLength < 1 || idLength > MAX_MODULE_IDENTIFIER_LEN) {
1378 return resolverException(
1379 QString("rejecting invalid module id size (%1 chars [1,%2])")
1380 .arg(idLength).arg(MAX_MODULE_IDENTIFIER_LEN)
1381 );
1382 }
1383
1384 // this regex matches: absolute, dotted or path-like URLs
1385 // (ie: the kind of stuff ScriptEngine::resolvePath already handles)
1386 QRegularExpression qualified ("^\\w+:|^/|^[.]{1,2}(/|$)");
1387
1388 // this is for module.require (which is a bound version of require that's always relative to the module path)
1389 if (!relativeTo.isEmpty()) {
1390 url = QUrl(relativeTo).resolved(moduleId);
1391 url = resolvePath(url.toString());
1392 } else if (qualified.match(moduleId).hasMatch()) {
1393 url = resolvePath(moduleId);
1394 } else {
1395 // check if the moduleId refers to a "system" module
1396 QString defaultsPath = defaultScriptsLoc.path();
1397 QString systemModulePath = QString("%1/modules/%2.js").arg(defaultsPath).arg(moduleId);
1398 url = defaultScriptsLoc;
1399 url.setPath(systemModulePath);
1400 if (!QFileInfo(url.toLocalFile()).isFile()) {
1401 return resolverException("system module not found");
1402 }
1403 }
1404
1405 if (url.isRelative()) {
1406 return resolverException("could not resolve module id");
1407 }
1408
1409 // if it looks like a local file, verify that it's an allowed path and really a file
1410 if (url.isLocalFile()) {
1411 QFileInfo file(url.toLocalFile());
1412 QUrl canonical = url;
1413 if (file.exists()) {
1414 canonical.setPath(file.canonicalFilePath());
1415 }
1416
1417 bool disallowOutsideFiles = !defaultScriptsLocation().isParentOf(canonical) && !currentSandboxURL.isLocalFile();
1418 if (disallowOutsideFiles && !PathUtils::isDescendantOf(canonical, currentSandboxURL)) {
1419 return resolverException(
1420 QString("path '%1' outside of origin script '%2' '%3'")
1421 .arg(PathUtils::stripFilename(url))
1422 .arg(PathUtils::stripFilename(currentSandboxURL))
1423 .arg(canonical.toString())
1424 );
1425 }
1426 if (!file.exists()) {
1427 return resolverException("path does not exist: " + url.toLocalFile());
1428 }
1429 if (!file.isFile()) {
1430 return resolverException("path is not a file: " + url.toLocalFile());
1431 }
1432 }
1433
1434 return url.toString();
1435}
1436
1437// retrieves the current parent module from the JS scope chain
1438QScriptValue ScriptEngine::currentModule() {
1439 auto jsRequire = globalObject().property("Script").property("require");
1440 auto cache = jsRequire.property("cache");
1441 auto candidate = QScriptValue();
1442 for(auto ctx = currentContext(); ctx && !candidate.isObject(); ctx = ctx->parentContext()) {
1443 QScriptContextInfo contextInfo { ctx };
1444 candidate = cache.property(contextInfo.fileName());
1445 }
1446 if (!candidate.isObject()) {
1447 return QScriptValue();
1448 }
1449 return candidate;
1450}
1451
1452// replaces or adds "module" to "parent.children[]" array
1453// (for consistency with Node.js and userscript cache invalidation without "cache busters")
1454bool ScriptEngine::registerModuleWithParent(const QScriptValue& module, const QScriptValue& parent) {
1455 auto children = parent.property("children");
1456 if (children.isArray()) {
1457 auto key = module.property("id");
1458 auto length = children.property("length").toInt32();
1459 for(int i=0; i < length; i++) {
1460 if (children.property(i).property("id").strictlyEquals(key)) {
1461 qCDebug(scriptengine_module) << key.toString() << " updating parent.children[" << i << "] = module";
1462 children.setProperty(i, module);
1463 return true;
1464 }
1465 }
1466 qCDebug(scriptengine_module) << key.toString() << " appending parent.children[" << length << "] = module";
1467 children.setProperty(length, module);
1468 return true;
1469 } else if (parent.isValid()) {
1470 qCDebug(scriptengine_module) << "registerModuleWithParent -- unrecognized parent" << parent.toVariant().toString();
1471 }
1472 return false;
1473}
1474
1475// creates a new JS "module" Object with default metadata properties
1476QScriptValue ScriptEngine::newModule(const QString& modulePath, const QScriptValue& parent) {
1477 auto closure = newObject();
1478 auto exports = newObject();
1479 auto module = newObject();
1480 qCDebug(scriptengine_module) << "newModule" << modulePath << parent.property("filename").toString();
1481
1482 closure.setProperty("module", module, READONLY_PROP_FLAGS);
1483
1484 // note: this becomes the "exports" free variable, so should not be set read only
1485 closure.setProperty("exports", exports);
1486
1487 // make the closure available to module instantiation
1488 module.setProperty("__closure__", closure, HIDDEN_PROP_FLAGS);
1489
1490 // for consistency with Node.js Module
1491 module.setProperty("id", modulePath, READONLY_PROP_FLAGS);
1492 module.setProperty("filename", modulePath, READONLY_PROP_FLAGS);
1493 module.setProperty("exports", exports); // not readonly
1494 module.setProperty("loaded", false, READONLY_PROP_FLAGS);
1495 module.setProperty("parent", parent, READONLY_PROP_FLAGS);
1496 module.setProperty("children", newArray(), READONLY_PROP_FLAGS);
1497
1498 // module.require is a bound version of require that always resolves relative to that module's path
1499 auto boundRequire = QScriptEngine::evaluate("(function(id) { return Script.require(Script.require.resolve(id, this.filename)); })", "(boundRequire)");
1500 module.setProperty("require", boundRequire, READONLY_PROP_FLAGS);
1501
1502 return module;
1503}
1504
1505// synchronously fetch a module's source code using BatchLoader
1506QScriptValue ScriptEngine::fetchModuleSource(const QString& modulePath, const bool forceDownload) {
1507 using UrlMap = QMap<QUrl, QString>;
1508 auto scriptCache = DependencyManager::get<ScriptCache>();
1509 QScriptValue req = newObject();
1510 qCDebug(scriptengine_module) << "require.fetchModuleSource: " << QUrl(modulePath).fileName() << QThread::currentThread();
1511
1512 auto onload = [=, &req](const UrlMap& data, const UrlMap& _status) {
1513 auto url = modulePath;
1514 auto status = _status[url];
1515 auto contents = data[url];
1516 qCDebug(scriptengine_module) << "require.fetchModuleSource.onload: " << QUrl(url).fileName() << status << QThread::currentThread();
1517 if (isStopping()) {
1518 req.setProperty("status", "Stopped");
1519 req.setProperty("success", false);
1520 } else {
1521 req.setProperty("url", url);
1522 req.setProperty("status", status);
1523 req.setProperty("success", ScriptCache::isSuccessStatus(status));
1524 req.setProperty("contents", contents, HIDDEN_PROP_FLAGS);
1525 }
1526 };
1527
1528 if (forceDownload) {
1529 qCDebug(scriptengine_module) << "require.requestScript -- clearing cache for" << modulePath;
1530 scriptCache->deleteScript(modulePath);
1531 }
1532 BatchLoader* loader = new BatchLoader({ modulePath });
1533 connect(loader, &BatchLoader::finished, this, onload);
1534 connect(this, &QObject::destroyed, loader, &QObject::deleteLater);
1535 // fail faster? (since require() blocks the engine thread while resolving dependencies)
1536 const int MAX_RETRIES = 1;
1537
1538 loader->start(MAX_RETRIES);
1539
1540 if (!loader->isFinished()) {
1541 QTimer monitor;
1542 QEventLoop loop;
1543 QObject::connect(loader, &BatchLoader::finished, this, [this, &monitor, &loop]{
1544 monitor.stop();
1545 loop.quit();
1546 });
1547
1548 // this helps detect the case where stop() is invoked during the download
1549 // but not seen in time to abort processing in onload()...
1550 connect(&monitor, &QTimer::timeout, this, [this, &loop, &loader]{
1551 if (isStopping()) {
1552 loop.exit(-1);
1553 }
1554 });
1555 monitor.start(500);
1556 loop.exec();
1557 }
1558 loader->deleteLater();
1559 return req;
1560}
1561
1562// evaluate a pending module object using the fetched source code
1563QScriptValue ScriptEngine::instantiateModule(const QScriptValue& module, const QString& sourceCode) {
1564 QScriptValue result;
1565 auto modulePath = module.property("filename").toString();
1566 auto closure = module.property("__closure__");
1567
1568 qCDebug(scriptengine_module) << QString("require.instantiateModule: %1 / %2 bytes").arg(QUrl(modulePath).fileName()).arg(sourceCode.length());
1569
1570 {
1571 ExceptionEmitter tryCatch(this, __FUNCTION__);
1572
1573 if (module.property("content-type").toString() == "application/json") {
1574 qCDebug(scriptengine_module) << "... parsing as JSON";
1575#ifdef DEV_OTHER_MODULE_JSON_PARSE
1576 auto JSON = globalObject().property("JSON");
1577 auto parse = JSON.property("parse");
1578 if (!parse.isFunction()) {
1579 currentContext()->throwValue(makeError("global JSON.parse is not a function", "EvalError"));
1580 return nullValue();
1581 }
1582 result = parse.call(JSON, QScriptValueList({ sourceCode }));
1583 closure.property("module").setProperty("exports", result);
1584 return result;
1585#endif
1586 closure.setProperty("__json", sourceCode);
1587 result = evaluateInClosure(closure, { "module.exports = JSON.parse(__json)", modulePath });
1588 } else {
1589 // scoped vars for consistency with Node.js
1590 closure.setProperty("require", module.property("require"));
1591 closure.setProperty("__filename", modulePath, HIDDEN_PROP_FLAGS);
1592 closure.setProperty("__dirname", QString(modulePath).replace(QRegExp("/[^/]*$"), ""), HIDDEN_PROP_FLAGS);
1593 result = evaluateInClosure(closure, { sourceCode, modulePath });
1594 }
1595 }
1596 return result;
1597}
1598
1599// CommonJS/Node.js like require/module support
1600QScriptValue ScriptEngine::require(const QString& moduleId) {
1601 qCDebug(scriptengine_module) << "ScriptEngine::require(" << moduleId.left(MAX_MODULE_IDENTIFIER_LOG_LEN) << ")";
1602 if (QThread::currentThread() != thread()) {
1603 qCDebug(scriptengine_module) << moduleId << " threads mismatch";
1604 return nullValue();
1605 }
1606
1607 // serialize require calls so the ordering/caching works correctly across multiple Entities
1608 // note: this is a Recursive mutex so nested require's should not affected
1609 QMutexLocker locker(&_requireLock);
1610 // _requireDepth++;
1611 // QObject autoDecrement;connect(&autoDecrement, &QObject::destroyed, this, [this](){ _requireDepth--; });
1612
1613 auto jsRequire = globalObject().property("Script").property("require");
1614 auto cacheMeta = jsRequire.data();
1615 auto cache = jsRequire.property("cache");
1616 auto parent = currentModule();
1617
1618 auto throwModuleError = [this, &cache](const QString& modulePath, const QScriptValue& error) {
1619 cache.setProperty(modulePath, nullValue());
1620 if (!error.isNull()) {
1621#ifdef DEBUG_JS_MODULES
1622 qCWarning(scriptengine_module) << "throwing module error:" << error.toString() << modulePath << error.property("stack").toString();
1623#endif
1624 currentContext()->throwValue(makeError(error));
1625 }
1626 return nullValue();
1627 };
1628
1629 // start by resolving the moduleId into a fully-qualified path/URL
1630 QString modulePath = _requireResolve(moduleId);
1631 if (hasUncaughtException()) {
1632 // the resolver already threw an exception -- bail early
1633 return nullValue();
1634 }
1635
1636 // check the resolved path against the cache
1637 auto module = cache.property(modulePath);
1638
1639 // modules get cached in `Script.require.cache` and (similar to Node.js) users can access it
1640 // to inspect particular entries and invalidate them by deleting the key:
1641 // `delete Script.require.cache[Script.require.resolve(moduleId)];`
1642
1643 // cacheMeta is just used right now to tell deleted keys apart from undefined ones
1644 bool invalidateCache = module.isUndefined() && cacheMeta.property(moduleId).isValid();
1645
1646 // reset the cacheMeta record so invalidation won't apply next time, even if the module fails to load
1647 cacheMeta.setProperty(modulePath, QScriptValue());
1648
1649 auto exports = module.property("exports");
1650 if (!invalidateCache && exports.isObject()) {
1651 // we have found a cacheed module -- just need to possibly register it with current parent
1652 qCDebug(scriptengine_module) << QString("require - using cached module '%1' for '%2' (loaded: %3)")
1653 .arg(modulePath).arg(moduleId).arg(module.property("loaded").toString());
1654 registerModuleWithParent(module, parent);
1655 return exports;
1656 }
1657
1658 // bootstrap / register new empty module
1659 module = newModule(modulePath, parent);
1660 registerModuleWithParent(module, parent);
1661
1662 // add it to the cache (this is done early so any cyclic dependencies pick up)
1663 cache.setProperty(modulePath, module);
1664
1665 // download the module source
1666 auto req = fetchModuleSource(modulePath, invalidateCache);
1667
1668 if (!req.property("success").toBool()) {
1669 auto error = QString("error retrieving script (%1)").arg(req.property("status").toString());
1670 return throwModuleError(modulePath, error);
1671 }
1672
1673#if DEBUG_JS_MODULES
1674 qCDebug(scriptengine_module) << "require.loaded: " <<
1675 QUrl(req.property("url").toString()).fileName() << req.property("status").toString();
1676#endif
1677
1678 auto sourceCode = req.property("contents").toString();
1679
1680 if (QUrl(modulePath).fileName().endsWith(".json", Qt::CaseInsensitive)) {
1681 module.setProperty("content-type", "application/json");
1682 } else {
1683 module.setProperty("content-type", "application/javascript");
1684 }
1685
1686 {
1687 // It seems that many JSON sources don't actually put .json in the URL...
1688 // so for now as a workaround users wanting to indicate JSON parsing mode can
1689 // do so by calling with a custom this context, eg:
1690 //
1691 // var ITEMS_URL = 'https://highfidelity.io/api/v1/marketplace/items';
1692 // var thisObject = { 'content-type': 'application/json' };
1693 // var items = Script.require.call(thisObject, ITEMS_URL + '?category=everything&sort=recent');
1694
1695 auto thisObject = currentContext()->thisObject();
1696 bool calledWithCustomThis = thisObject.isObject() &&
1697 !thisObject.strictlyEquals(globalObject()) &&
1698 !thisObject.toQObject();
1699
1700 if (calledWithCustomThis) {
1701#ifdef DEBUG_JS
1702 _debugDump("this", thisObject);
1703#endif
1704 _applyUserOptions(module, thisObject);
1705 }
1706 }
1707
1708 // evaluate the module
1709 auto result = instantiateModule(module, sourceCode);
1710
1711 if (result.isError() && !result.strictlyEquals(module.property("exports"))) {
1712 qCWarning(scriptengine_module) << "-- result.isError --" << result.toString();
1713 return throwModuleError(modulePath, result);
1714 }
1715
1716 // mark as fully-loaded
1717 module.setProperty("loaded", true, READONLY_PROP_FLAGS);
1718
1719 // set up a new reference point for detecting cache key deletion
1720 cacheMeta.setProperty(modulePath, module);
1721
1722 qCDebug(scriptengine_module) << "//ScriptEngine::require(" << moduleId << ")";
1723
1724 return module.property("exports");
1725}
1726
1727// User-configurable override options
1728void ScriptEngine::_applyUserOptions(QScriptValue& module, QScriptValue& options) {
1729 if (!options.isValid()) {
1730 return;
1731 }
1732 // options['content-type'] === 'application/json'
1733 // -- allows JSON modules to be used from URLs not ending in .json
1734 if (options.property("content-type").isString()) {
1735 module.setProperty("content-type", options.property("content-type"));
1736 qCDebug(scriptengine_module) << "module['content-type'] =" << module.property("content-type").toString();
1737 }
1738
1739 if (ScriptEngine::_enableExtendedModuleCompatbility.get()) {
1740 auto closure = module.property("__closure__");
1741
1742 auto maybeSetToExports = [&](const QString& key) {
1743 if (options.property(key).toString() == "exports") {
1744 closure.setProperty(key, module.property("exports"));
1745 qCDebug(scriptengine_module) << "module.closure[" << key << "] = exports";
1746 }
1747 };
1748
1749 // options[{key}] = 'exports'
1750 // several "agnostic" modules in the wild are just one step away from being compatible --
1751 // they just didn't know not to look specifically for this, self or global for attaching
1752 // things onto.
1753 maybeSetToExports("global");
1754 maybeSetToExports("self");
1755 maybeSetToExports("this");
1756
1757 // when options is an Object it will get used as the value of "this" during module evaluation
1758 // (which is what one might expect when calling require.call(thisObject, ...))
1759 if (options.isObject()) {
1760 closure.setProperty("this", options);
1761 }
1762
1763 // when options.global is an Object it'll get used as the global object (during evaluation only)
1764 if (options.property("global").isObject()) {
1765 closure.setProperty("global", options.property("global"));
1766 qCDebug(scriptengine_module) << "module.closure['global'] = options.global";
1767 }
1768 }
1769}
1770
1307// If a callback is specified, the included files will be loaded asynchronously and the callback will be called1771// If a callback is specified, the included files will be loaded asynchronously and the callback will be called
1308// when all of the files have finished loading.1772// when all of the files have finished loading.
1309// If no callback is specified, the included files will be loaded synchronously and will block execution until1773// If no callback is specified, the included files will be loaded synchronously and will block execution until