diff options
author | Alessio Vanni <vannilla@firemail.cc> | 2019-05-15 15:40:42 +0200 |
---|---|---|
committer | Alessio Vanni <vannilla@firemail.cc> | 2019-05-15 15:40:42 +0200 |
commit | c83f575f798f795610aa1262776f5cabf24b12fb (patch) | |
tree | 7d5545f2a185db03c66a259e5878517cd39b7211 /js/assets.js | |
parent | 64c15c807f4f0387bf1e967354fab21dbdb9980c (diff) | |
download | ematrix-c83f575f798f795610aa1262776f5cabf24b12fb.tar.lz ematrix-c83f575f798f795610aa1262776f5cabf24b12fb.tar.xz ematrix-c83f575f798f795610aa1262776f5cabf24b12fb.zip |
Rewrite assets.js
It has come to this in the end.
Diffstat (limited to 'js/assets.js')
-rw-r--r-- | js/assets.js | 1739 |
1 files changed, 882 insertions, 857 deletions
diff --git a/js/assets.js b/js/assets.js index 5219a82..32090c7 100644 --- a/js/assets.js +++ b/js/assets.js @@ -27,888 +27,913 @@ ηMatrix.assets = (function() { -/******************************************************************************/ - -var reIsExternalPath = /^(?:[a-z-]+):\/\//, - errorCantConnectTo = vAPI.i18n('errorCantConnectTo'), - noopfunc = function(){}; - -var api = { -}; - -/******************************************************************************/ - -var observers = []; - -api.addObserver = function(observer) { - if ( observers.indexOf(observer) === -1 ) { - observers.push(observer); - } -}; - -api.removeObserver = function(observer) { - var pos; - while ( (pos = observers.indexOf(observer)) !== -1 ) { - observers.splice(pos, 1); - } -}; - -var fireNotification = function(topic, details) { - var result; - for ( var i = 0; i < observers.length; i++ ) { - if ( observers[i](topic, details) === false ) { - result = false; - } - } - return result; -}; - -/******************************************************************************/ - -api.fetchText = function(url, onLoad, onError) { - var actualUrl = reIsExternalPath.test(url) ? url : vAPI.getURL(url); - - if ( typeof onError !== 'function' ) { - onError = onLoad; - } - - // https://github.com/gorhill/uMatrix/issues/15 - var onResponseReceived = function() { - this.onload = this.onerror = this.ontimeout = null; - // xhr for local files gives status 0, but actually succeeds - var details = { - url: url, - content: '', - statusCode: this.status || 200, - statusText: this.statusText || '' - }; - if ( details.statusCode < 200 || details.statusCode >= 300 ) { - return onError.call(null, details); - } - // consider an empty result to be an error - if ( stringIsNotEmpty(this.responseText) === false ) { - return onError.call(null, details); - } - // we never download anything else than plain text: discard if response - // appears to be a HTML document: could happen when server serves - // some kind of error page I suppose - var text = this.responseText.trim(); - if ( text.startsWith('<') && text.endsWith('>') ) { - return onError.call(null, details); - } - details.content = this.responseText; - return onLoad.call(null, details); - }; - - var onErrorReceived = function() { - this.onload = this.onerror = this.ontimeout = null; - ηMatrix.logger.writeOne('', 'error', errorCantConnectTo.replace('{{url}}', actualUrl)); - onError.call(null, { url: url, content: '' }); - }; - - // Be ready for thrown exceptions: - // I am pretty sure it used to work, but now using a URL such as - // `file:///` on Chromium 40 results in an exception being thrown. - var xhr = new XMLHttpRequest(); - try { - xhr.open('get', actualUrl, true); - xhr.timeout = 30000; - xhr.onload = onResponseReceived; - xhr.onerror = onErrorReceived; - xhr.ontimeout = onErrorReceived; - xhr.responseType = 'text'; - xhr.send(); - } catch (e) { - onErrorReceived.call(xhr); - } -}; - -/******************************************************************************* - - TODO(seamless migration): - This block of code will be removed when I am confident all users have - moved to a version of uBO which does not require the old way of caching - assets. - - api.listKeyAliases: a map of old asset keys to new asset keys. - - migrate(): to seamlessly migrate the old cache manager to the new one: - - attempt to preserve and move content of cached assets to new locations; - - removes all traces of now obsolete cache manager entries in cacheStorage. - - This code will typically execute only once, when the newer version of uBO - is first installed and executed. - -**/ - -api.listKeyAliases = { - "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", - "assets/thirdparties/hosts-file.net/ad-servers": "hphosts", - "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-0", - "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-1", - "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", - "assets/thirdparties/someonewhocares.org/hosts/hosts": "dpollock-0", - "assets/thirdparties/winhelp2002.mvps.org/hosts.txt": "mvps-0" -}; - -var migrate = function(callback) { - var entries, - moveCount = 0, - toRemove = []; - - var countdown = function(change) { - moveCount -= (change || 0); - if ( moveCount !== 0 ) { return; } - vAPI.cacheStorage.remove(toRemove); - saveAssetCacheRegistry(); - callback(); - }; - - var onContentRead = function(oldKey, newKey, bin) { - var content = bin && bin['cached_asset_content://' + oldKey] || undefined; - if ( content ) { - assetCacheRegistry[newKey] = { - readTime: Date.now(), - writeTime: entries[oldKey] - }; - if ( reIsExternalPath.test(oldKey) ) { - assetCacheRegistry[newKey].remoteURL = oldKey; - } - bin = {}; - bin['cache/' + newKey] = content; - vAPI.cacheStorage.set(bin); - } - countdown(1); - }; - - var onEntries = function(bin) { - entries = bin && bin.cached_asset_entries; - if ( !entries ) { return callback(); } - if ( bin && bin.assetCacheRegistry ) { - assetCacheRegistry = bin.assetCacheRegistry; - } - var aliases = api.listKeyAliases; - for ( var oldKey in entries ) { - var newKey = aliases[oldKey]; - if ( !newKey && /^https?:\/\//.test(oldKey) ) { - newKey = oldKey; - } - if ( newKey ) { - vAPI.cacheStorage.get( - 'cached_asset_content://' + oldKey, - onContentRead.bind(null, oldKey, newKey) - ); - moveCount += 1; - } - toRemove.push('cached_asset_content://' + oldKey); - } - toRemove.push('cached_asset_entries', 'extensionLastVersion'); - countdown(); - }; - - vAPI.cacheStorage.get( - [ 'cached_asset_entries', 'assetCacheRegistry' ], - onEntries - ); -}; - -/******************************************************************************* + let api = {}; + let observers = []; + let externalPathRegex = /^(?:[a-z-]+):\/\//; + let connectionError = vAPI.i18n('errorCantConnectTo'); - The purpose of the asset source registry is to keep key detail information - about an asset: - - Where to load it from: this may consist of one or more URLs, either local - or remote. - - After how many days an asset should be deemed obsolete -- i.e. in need of - an update. - - The origin and type of an asset. - - The last time an asset was registered. - -**/ - -var assetSourceRegistryStatus, - assetSourceRegistry = Object.create(null); - -var registerAssetSource = function(assetKey, dict) { - var entry = assetSourceRegistry[assetKey] || {}; - for ( var prop in dict ) { - if ( dict.hasOwnProperty(prop) === false ) { continue; } - if ( dict[prop] === undefined ) { - delete entry[prop]; - } else { - entry[prop] = dict[prop]; - } + function stub(string) { + console.debug("stub! "+string); } - var contentURL = dict.contentURL; - if ( contentURL !== undefined ) { - if ( typeof contentURL === 'string' ) { - contentURL = entry.contentURL = [ contentURL ]; - } else if ( Array.isArray(contentURL) === false ) { - contentURL = entry.contentURL = []; - } - var remoteURLCount = 0; - for ( var i = 0; i < contentURL.length; i++ ) { - if ( reIsExternalPath.test(contentURL[i]) ) { - remoteURLCount += 1; - } - } - entry.hasLocalURL = remoteURLCount !== contentURL.length; - entry.hasRemoteURL = remoteURLCount !== 0; - } else if ( entry.contentURL === undefined ) { - entry.contentURL = []; - } - if ( typeof entry.updateAfter !== 'number' ) { - entry.updateAfter = 13; - } - if ( entry.submitter ) { - entry.submitTime = Date.now(); // To detect stale entries - } - assetSourceRegistry[assetKey] = entry; -}; - -var unregisterAssetSource = function(assetKey) { - assetCacheRemove(assetKey); - delete assetSourceRegistry[assetKey]; -}; - -var saveAssetSourceRegistry = (function() { - var timer; - var save = function() { - timer = undefined; - vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry }); - }; - return function(lazily) { - if ( timer !== undefined ) { - clearTimeout(timer); - } - if ( lazily ) { - timer = vAPI.setTimeout(save, 500); - } else { - save(); - } - }; -})(); - -var updateAssetSourceRegistry = function(json, silent) { - var newDict; - try { - newDict = JSON.parse(json); - } catch (ex) { - } - if ( newDict instanceof Object === false ) { return; } - - var oldDict = assetSourceRegistry, - assetKey; - - // Remove obsolete entries (only those which were built-in). - for ( assetKey in oldDict ) { - if ( - newDict[assetKey] === undefined && - oldDict[assetKey].submitter === undefined - ) { - unregisterAssetSource(assetKey); - } - } - // Add/update existing entries. Notify of new asset sources. - for ( assetKey in newDict ) { - if ( oldDict[assetKey] === undefined && !silent ) { - fireNotification( - 'builtin-asset-source-added', - { assetKey: assetKey, entry: newDict[assetKey] } - ); - } - registerAssetSource(assetKey, newDict[assetKey]); - } - saveAssetSourceRegistry(); -}; - -var getAssetSourceRegistry = function(callback) { - // Already loaded. - if ( assetSourceRegistryStatus === 'ready' ) { - callback(assetSourceRegistry); - return; - } - - // Being loaded. - if ( Array.isArray(assetSourceRegistryStatus) ) { - assetSourceRegistryStatus.push(callback); - return; - } - - // Not loaded: load it. - assetSourceRegistryStatus = [ callback ]; - - var registryReady = function() { - var callers = assetSourceRegistryStatus; - assetSourceRegistryStatus = 'ready'; - var fn; - while ( (fn = callers.shift()) ) { - fn(assetSourceRegistry); - } - }; - - // First-install case. - var createRegistry = function() { - api.fetchText( - ηMatrix.assetsBootstrapLocation || 'assets/assets.json', - function(details) { - updateAssetSourceRegistry(details.content, true); - registryReady(); - } - ); - }; - - vAPI.cacheStorage.get('assetSourceRegistry', function(bin) { - if ( !bin || !bin.assetSourceRegistry ) { - createRegistry(); - return; - } - assetSourceRegistry = bin.assetSourceRegistry; - registryReady(); - }); -}; - -api.registerAssetSource = function(assetKey, details) { - getAssetSourceRegistry(function() { - registerAssetSource(assetKey, details); - saveAssetSourceRegistry(true); - }); -}; - -api.unregisterAssetSource = function(assetKey) { - getAssetSourceRegistry(function() { - unregisterAssetSource(assetKey); - saveAssetSourceRegistry(true); - }); -}; - -/******************************************************************************* - - The purpose of the asset cache registry is to keep track of all assets - which have been persisted into the local cache. - -**/ - -var assetCacheRegistryStatus = undefined, - assetCacheRegistryStartTime = Date.now(), - assetCacheRegistry = {}; - -var getAssetCacheRegistry = function(callback) { - // Already loaded. - if ( assetCacheRegistryStatus === 'ready' ) { - callback(assetCacheRegistry); - return; - } - - // Being loaded. - if ( Array.isArray(assetCacheRegistryStatus) ) { - assetCacheRegistryStatus.push(callback); - return; - } - - // Not loaded: load it. - assetCacheRegistryStatus = [ callback ]; - - var registryReady = function() { - var callers = assetCacheRegistryStatus; - assetCacheRegistryStatus = 'ready'; - var fn; - while ( (fn = callers.shift()) ) { - fn(assetCacheRegistry); - } - }; - - var migrationDone = function() { - vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { - if ( bin && bin.assetCacheRegistry ) { - assetCacheRegistry = bin.assetCacheRegistry; - } - registryReady(); - }); - }; - - // Probably not needed anymore - // Commented out until something breaks - // migrate(migrationDone); -}; -var saveAssetCacheRegistry = (function() { - var timer; - var save = function() { - timer = undefined; - vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry }); - }; - return function(lazily) { - if ( timer !== undefined ) { clearTimeout(timer); } - if ( lazily ) { - timer = vAPI.setTimeout(save, 500); - } else { - save(); - } - }; -})(); + let notifyObservers = function (topic, details) { + let result; -var assetCacheRead = function(assetKey, callback) { - var internalKey = 'cache/' + assetKey; - - var reportBack = function(content, err) { - var details = { assetKey: assetKey, content: content }; - if ( err ) { details.error = err; } - callback(details); - }; - - var onAssetRead = function(bin) { - if ( !bin || !bin[internalKey] ) { - return reportBack('', 'E_NOTFOUND'); - } - var entry = assetCacheRegistry[assetKey]; - if ( entry === undefined ) { - return reportBack('', 'E_NOTFOUND'); - } - entry.readTime = Date.now(); - saveAssetCacheRegistry(true); - reportBack(bin[internalKey]); - }; + for (let i=0; i<observers.length; ++i) { + result = observers[i](topic, details); + } - var onReady = function() { - vAPI.cacheStorage.get(internalKey, onAssetRead); - }; - - getAssetCacheRegistry(onReady); -}; - -var assetCacheWrite = function(assetKey, details, callback) { - var internalKey = 'cache/' + assetKey; - var content = ''; - if ( typeof details === 'string' ) { - content = details; - } else if ( details instanceof Object ) { - content = details.content || ''; + return result; } - if ( content === '' ) { - return assetCacheRemove(assetKey, callback); + function isEmptyString(s) { + return (typeof s === 'string' && s === ''); } - var reportBack = function(content) { - var details = { assetKey: assetKey, content: content }; - if ( typeof callback === 'function' ) { - callback(details); - } - fireNotification('after-asset-updated', details); - }; - - var onReady = function() { - var entry = assetCacheRegistry[assetKey]; - if ( entry === undefined ) { - entry = assetCacheRegistry[assetKey] = {}; - } - entry.writeTime = entry.readTime = Date.now(); - if ( details instanceof Object && typeof details.url === 'string' ) { - entry.remoteURL = details.url; - } - var bin = { assetCacheRegistry: assetCacheRegistry }; - bin[internalKey] = content; - vAPI.cacheStorage.set(bin); - reportBack(content); - }; - getAssetCacheRegistry(onReady); -}; - -var assetCacheRemove = function(pattern, callback) { - var onReady = function() { - var cacheDict = assetCacheRegistry, - removedEntries = [], - removedContent = []; - for ( var assetKey in cacheDict ) { - if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { - continue; - } - if ( typeof pattern === 'string' && assetKey !== pattern ) { - continue; - } - removedEntries.push(assetKey); - removedContent.push('cache/' + assetKey); - delete cacheDict[assetKey]; - } - if ( removedContent.length !== 0 ) { - vAPI.cacheStorage.remove(removedContent); - var bin = { assetCacheRegistry: assetCacheRegistry }; - vAPI.cacheStorage.set(bin); - } - if ( typeof callback === 'function' ) { - callback(); - } - for ( var i = 0; i < removedEntries.length; i++ ) { - fireNotification('after-asset-updated', { assetKey: removedEntries[i] }); - } - }; - - getAssetCacheRegistry(onReady); -}; - -var assetCacheMarkAsDirty = function(pattern, exclude, callback) { - var onReady = function() { - var cacheDict = assetCacheRegistry, - cacheEntry, - mustSave = false; - for ( var assetKey in cacheDict ) { - if ( pattern instanceof RegExp ) { - if ( pattern.test(assetKey) === false ) { continue; } - } else if ( typeof pattern === 'string' ) { - if ( assetKey !== pattern ) { continue; } - } else if ( Array.isArray(pattern) ) { - if ( pattern.indexOf(assetKey) === -1 ) { continue; } - } - if ( exclude instanceof RegExp ) { - if ( exclude.test(assetKey) ) { continue; } - } else if ( typeof exclude === 'string' ) { - if ( assetKey === exclude ) { continue; } - } else if ( Array.isArray(exclude) ) { - if ( exclude.indexOf(assetKey) !== -1 ) { continue; } - } - cacheEntry = cacheDict[assetKey]; - if ( !cacheEntry.writeTime ) { continue; } - cacheDict[assetKey].writeTime = 0; - mustSave = true; - } - if ( mustSave ) { - var bin = { assetCacheRegistry: assetCacheRegistry }; - vAPI.cacheStorage.set(bin); - } - if ( typeof callback === 'function' ) { - callback(); - } - }; - if ( typeof exclude === 'function' ) { - callback = exclude; - exclude = undefined; - } - getAssetCacheRegistry(onReady); -}; - -/******************************************************************************/ - -var stringIsNotEmpty = function(s) { - return typeof s === 'string' && s !== ''; -}; - -/******************************************************************************/ - -api.get = function(assetKey, options, callback) { - if ( typeof options === 'function' ) { - callback = options; - options = {}; - } else if ( typeof callback !== 'function' ) { - callback = noopfunc; + function noOp() { + return; } - var assetDetails = {}, - contentURLs, - contentURL; - - var reportBack = function(content, err) { - var details = { assetKey: assetKey, content: content }; - if ( err ) { - details.error = assetDetails.lastError = err; - } else { - assetDetails.lastError = undefined; - } - callback(details); - }; + // Cache Registry + var cacheRegistry = new Object(); + let cacheRegistryReady = false; + let cacheRegistryCallbacks = undefined; + let cacheRegistryStart = Date.now(); + + let saveCacheRegistry = (function () { + let timer; + + function save() { + timer = undefined; + vAPI.cacheStorage.set({ + assetCacheRegistry: cacheRegistry, + }); + } + + return function (lazy) { + if (timer !== undefined) { + clearTimeout(timer); + } + + if (lazy === true) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; + })(); + + let getCacheRegistry = function (callback) { + if (cacheRegistryReady == true) { + callback(cacheRegistry); + return; + } + + if (cacheRegistryCallbacks !== undefined) { + // If it's not undefined it's always an array + // + // eMatrix: this block in particular is probably never + // called: originally, it was used because uMatrix called + // some code in a callback that could wait + // + // While waiting, more elements could've been pushed in + // the array + // + // Since the waiting callback is not used here, any + // further callback are likely to be handled by the + // condition above + // This block is left here just in case + cacheRegistryCallbacks.push(callback); + return; + } + + cacheRegistryCallbacks = [callback]; + cacheRegistryReady = true; + + let f; + while ((f = cacheRegistryCallbacks.shift())) { + f(cacheRegistry); + } + }; + + let readCache = function (key, callback) { + let report = function (content, error) { + let details = { + assetKey: key, + content: content, + }; + + if (error) { + details.error = error; + } + + callback(details); + }; + + let onRead = function (bin) { + if (!bin || !bin[key]) { + report('', 'E_NOTFOUND'); + return; + } + + let entry = cacheRegistry[key]; + if (entry === undefined) { + let onRead2 = function (bin) { + if (!bin || !bin['assetCacheRegistry']) { + cacheRegistry = {}; + } else { + cacheRegistry = bin['assetCacheRegistry']; + } + }; + + vAPI.cacheStorage.get('assetCacheRegistry', onRead2); + + entry = cacheRegistry[key]; + if (entry === undefined) { + report('', 'E_NOTFOUND'); + return; + } + } + + entry.readTime = Date.now(); + saveCacheRegistry(true); + + report(bin[key]); + }; + + let onReady = function () { + vAPI.cacheStorage.get(key, onRead); + }; + + getCacheRegistry(onReady); + }; + + let writeCache = function (key, details, callback) { + let content = ''; + + if (typeof details === 'string') { + content = details; + } else if (details instanceof Object) { + content = details.content || ''; + } + + if (content === '') { + removeCache(key, callback); + return; + } + + let report = function (content) { + let details = { + assetKey: key, + content: content, + }; + + if (typeof callback === 'function') { + callback(details); + } + + notifyObservers('after-asset-updated', details); + }; + + let onReady = function () { + let entry = cacheRegistry[key]; + if (entry === undefined) { + entry = cacheRegistry[key] = {}; + } + + entry.writeTime = entry.readTime = Date.now(); + if (details instanceof Object && typeof details.url === 'string') { + entry.remoteURL = details.url; + } + + let bin = { + assetCacheRegistry: cacheRegistry, + }; + + bin[key] = content; + vAPI.cacheStorage.set(bin); + report(content); + }; + + getCacheRegistry(onReady); + }; + + let markDirtyCache = function (pattern, exclude, callback) { + let onReady = function (registry) { + let entry; + let mustSave = false; + + for (let key in registry) { + if (pattern instanceof RegExp && pattern.test(key) === false) { + continue; + } else if (typeof pattern === 'string' && key !== pattern) { + continue; + } else if (Array.isArray(pattern) + && pattern.indexOf(key) === -1) { + continue; + } + + if (exclude instanceof RegExp && exclude.test(key)) { + continue; + } else if (typeof exclude === 'string' && key === exclude) { + continue; + } else if (Array.isArray(exclude) + && exclude.indexOf(key) !== -1) { + continue; + } + + entry = registry[key]; + if (!entry.writeTime) { + continue; + } + + registry[key].writeTime = 0; + mustSave = true; + } + + if (mustSave) { + let bin = { + assetCacheRegistry: registry, + }; + vAPI.cacheStorage.set(bin); + } + + if (typeof callback === 'function') { + callback(); + } + }; + + if (typeof exclude === 'function') { + callback = exclude; + exclude = undefined; + } + + getCacheRegistry(onReady); + }; + + let removeCache = function (key, callback) { + let onReady = function () { + let removedEntries = []; + let removedContent = []; + + for (let k in cacheRegistry) { + if (key instanceof RegExp && !key.test(k)) { + continue; + } + if (typeof key === 'string' && k !== key) { + continue; + } + + removedEntries.push(k); + removedContent.push(k); + + delete cacheRegistry[k]; + } + + if (removedContent.length > 0) { + vAPI.cacheStorage.remove(removedContent); + + let bin = { + assetCacheRegistry: cacheRegistry, + }; + + vAPI.cacheStorage.set(bin); + } + + if (typeof callback === 'function') { + callback(); + } + + for (let i=0; i<removedEntries.length; ++i) { + notifyObservers('after-asset-updated', { + assetKey: removedEntries[i], + }); + } + }; + + getCacheRegistry(onReady); + }; + + // Source Registry + var sourceRegistry = Object.create(null); + let sourceRegistryReady = false; + let sourceRegistryCallbacks = undefined; + + let saveSourceRegistry = (function () { + let timer; + + function save() { + timer = undefined; + vAPI.cacheStorage.set({ + assetSourceRegistry: sourceRegistry, + }); + } + + return function (lazy) { + if (timer !== undefined) { + clearTimeout(timer); + } + + if (lazy === true) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; + })(); + + let registerSource = function (key, details) { + let entry = sourceRegistry[key] || {}; + + for (let p in details) { + if (details.hasOwnProperty(p) === false) { + continue; + } + + if (details[p] !== undefined) { + entry[p] = details[p]; + } else { + delete entry[p]; + } + } + + let contentUrl = details.contentURL; + if (contentUrl) { + if (typeof contentUrl === 'string') { + contentUrl = entry.contentURL = [contentUrl]; + } else if (Array.isArray(contentUrl) === false) { + contentUrl = entry.contentURL = []; + } + + let remoteCount = 0; + + for (let i=0; i<contentUrl.length; ++i) { + if (externalPathRegex.test(contentUrl[i])) { + ++remoteCount; + } + } + + entry.hasLocalURL = (remoteCount !== contentUrl.length); + entry.hasRemoteURL = (remoteCount !== 0); + } else { + entry.contentURL = []; + } + + if (typeof entry.updateAfter !== 'number') { + entry.updateAfter = 13; + } + + if (entry.submitter) { + entry.submitTime = Date.now(); // Detects stale entries + } + + sourceRegistry[key] = entry; + }; + + let unregistrerSource = function (key) { + removeCache(key); + delete sourceRegistry[key]; + }; + + let updateSourceRegistry = function (string, silent) { + let json; + + try { + json = JSON.parse(string); + } catch (e) { + return; + } + + for (let key in sourceRegistry) { + if (json[key] === undefined + && sourceRegistry[key].submitter === undefined) { + unregisterSource(key); + } + } + + for (let key in json) { + if (sourceRegistry[key] === undefined && !silent) { + notifyObservers('builtin-asset-source-added', { + assetKey: key, + entry: json[key], + }); + } + + registerSource(key, json[key]); + } + + saveSourceRegistry(); + }; + + let getSourceRegistry = function (callback) { + if (sourceRegistryReady === true) { + callback(sourceRegistry); + return; + } + + if (sourceRegistryCallbacks !== undefined) { + // If it's not undefined it's always an array + sourceRegistryCallbacks.push(callback); + return; + } + + sourceRegistryCallbacks = [callback]; + + let onReady = function () { + sourceRegistryReady = true; + + let f; + while ((f = sourceRegistryCallbacks.shift())) { + f(sourceRegistry); + } + }; + + let createRegistry = function () { + api.fetchText + (ηMatrix.assetsBootstrapLocation || 'assets/assets.json', + function (details) { + updateSourceRegistry(details.content, true); + onReady(); + }); + }; + + let onRead = function (bin) { + if (!bin || !bin.assetSourceRegistry + || Object.keys(bin.assetSourceRegistry).length == 0) { + createRegistry(); + return; + } + + sourceRegistry = bin.assetSourceRegistry; + onReady(); + }; + + vAPI.cacheStorage.get('assetSourceRegistry', onRead); + }; + + // Remote + let getRemote = function (key, callback) { + let assetDetails = {}; + let contentUrl; + + let report = function (content, error) { + let details = { + assetKey: key, + content: content, + }; + if (error) { + details.error = assetDetails.lastError = error; + } + callback(details); + }; + + let tryLoad = function () { + let urls = []; + if (typeof assetDetails.contentURL === 'string') { + urls = [assetDetails.contentURL]; + } else if (Array.isArray(assetDetails.contentURL)) { + urls = assetDetails.contentURL.slice(0); + } + + while ((contentUrl = urls.shift())) { + if (externalPathRegex.test(contentUrl)) { + break; + } + } + + if (!contentUrl) { + report('', 'E_NOTFOUND'); + return; + } + + api.fetchText(contentUrl, onRemoteContentLoad, onRemoteContentError); + }; + + let onRemoteContentLoad = function (details) { + if (isEmptyString(details.content) === true) { + registerSource(key, { + error: { + time: Date.now(), + error: 'No content' + } + }); + tryLoad(); + } + + writeCache(key, { + content: details.content, + url: contentUrl, + }); + + registerSource(key, {error: undefined}); + report(details.content); + }; + + let onRemoteContentError = function (details) { + let text = detail.statusText; + if (details.statusCode === 0) { + text = 'network error'; + } + registerSource(key, { + error: { + time: Date.now(), + error: text, + } + }); + tryLoad(); + }; + + let onReady = function (registry) { + assetDetails = registry[key] || {}; + tryLoad(); + }; + + getSourceRegistry(onReady); + }; + + // Updater + let updateStatus = 'stop'; + let updateDefaultDelay = 120000; + let updateDelay = updateDefaultDelay; + let updateTimer; + let updated = []; + let updateFetch = new Set(); + + let updateStart = function () { + updateStatus = 'running'; + updateFetch.clear(); + updated = []; + notifyObservers('before-assets-updated'); + updateNext(); + }; + + let updateNext = function () { + let gcOne = function (key) { + let entry = cacheRegistry[key]; + if (entry && entry.readTime < cacheRegistryStart) { + cacheRemove(key); + } + }; + + let findOne = function () { + let now = Date.now(); + let sourceEntry; + let cacheEntry; + + for (let key in sourceRegistry) { + sourceEntry = sourceRegistry[key]; + if (sourceEntry.hasRemoteURL !== true) { + continue; + } + if (updateFetch.has(key)) { + continue; + } + + cacheEntry = cacheRegistry[key]; + if (cacheEntry + && (cacheEntry.writeTime + + sourceEntry.updateAfter*86400000) > now) { + continue; + } + if (notifyObservers('before-asset-updated', {assetKey: key})) { + return key; + } + + gcOne(key); + } + + return undefined; + }; + + let onUpdate = function (details) { + if (details.content !== '') { + updated.push(details.assetKey); + if (details.assetKey === 'asset.json') { + updateSourceRegistry(details.content); + } + } else { + notifyObservers('asset-update-failed', { + assetKey: details.assetKey, + }); + } + + if (findOne() !== undefined) { + vAPI.setTimeout(updateNext, updateDelay); + } else { + updateEnd(); + } + }; + + let updateOne = function () { + let key = findOne(); + if (key === undefined) { + updateEnd(); + return; + } + + updateFetch.add(key); + getRemote(key, onUpdate); + }; + + let onSourceReady = function (registry) { + updateOne(); + }; + + let onCacheReady = function (registry) { + getSourceRegistry(onSourceReady); + }; + + getCacheRegistry(onCacheReady); + }; + + let updateEnd = function () { + let keys = updated.slice(0); + updateFetch.clear(); + updateStatus = 'stop'; + updateDelay = updateDefaultDelay; + notifyObservers('after-asset-updated', { + assetKeys: keys, + }); + }; + + // Assets API + api.addObserver = function (observer) { + if (observers.indexOf(observer) === -1) { + observers.push(observer); + } + }; + + api.removeObserver = function (observer) { + let pos = observers.indexOf(observer); + if (pos !== -1) { + observers.splice(pos, 1); + } + }; + + api.fetchText = function (url, onLoad, onError) { + let iurl = externalPathRegex.test(url) ? url : vAPI.getURL(url); + + if (typeof onError !== 'function') { + onError = onLoad; + } + + let onResponseReceived = function () { + this.onload = this.onerror = this.ontimeout = null; + + let details = { + url: url, + content: '', + // On local files this.status is 0, but the request + // is successful + statusCode: this.status || 200, + statusText: this.statusText || '', + }; + + if (details.statusCode < 200 || details.statusCode >= 300) { + onError.call(null, details); + return; + } + + if (isEmptyString(this.responseText) === true) { + onError.call(null, details); + return; + } + + let t = this.responseText.trim(); + + // Discard HTML as it's probably an error + // (the request expects plain text as a response) + if (t.startsWith('<') && t.endsWith('>')) { + onError.call(null, details); + return; + } + + details.content = t; + onLoad.call(null, details); + }; + + let onErrorReceived = function () { + this.onload = this.onerror = this.ontimeout = null; + + ηMatrix.logger.writeOne('', 'error', + connectionError.replace('{{url}}', iurl)); + + onError.call(null, { + url: url, + content: '', + }); + }; + + let req = new XMLHttpRequest(); + req.open('GET', iurl, true); + req.timeout = 30000; + req.onload = onResponseReceived; + req.onerror = onErrorReceived; + req.ontimeout = onErrorReceived; + req.responseType = 'text'; + + try { + // This can throw in some cases + req.send(); + } catch (e) { + onErrorReceived.call(req); + } + }; + + api.registerAssetSource = function (key, details) { + getSourceRegistry(function () { + registerSource(key, details); + saveSourceRegistry(true); + }); + }; + + api.unregisterAssetSource = function (key) { + getSourceRegistry(function () { + unregisterSource(key); + saveSourceRegistry(true); + }); + }; + + api.get = function (key, options, callback) { + let cb; + let opt; + + if (typeof options === 'function') { + cb = options; + opt = {}; + } else if (typeof callback !== 'function') { + cb = noOp; + opt = options; + } else { + cb = callback; + opt = options; + } + + let assetDetails = {}; + let contentUrl = undefined; + + let report = function (content, error) { + let details = { + assetKey: key, + content: content, + }; + + if (error) { + details.error = assetDetails.error = error; + } else { + assetDetails.error = undefined; + } + + cb(details); + }; + + let onContentNotLoaded = function (details) { + let external; + let urls = []; + + console.debug(assetDetails); + + if (typeof assetDetails.contentURL === 'string') { + urls = [assetDetails.contentURL]; + } else if (Array.isArray(assetDetails.contentURL)) { + urls = assetDetails.contentURL.slice(0); + } + + while ((contentUrl = urls.shift())) { + external = externalPathRegex.test(contentUrl); + if (external === true && assetDetails.loaded !== true) { + break; + } + if (external === false || assetDetails.hasLocalURL !== true) { + break; + } + } + + if (!contentUrl) { + report('', 'E_NOTFOUND'); + return; + } + + api.fetchText(contentUrl, onContentLoaded, onContentNotLoaded); + }; + + let onContentLoaded = function (details) { + if (isEmptyString(details.content) === true) { + onContentNotLoaded(); + return; + } + + if (externalPathRegex.test(details.url) + && opt.dontCache !== true) { + writeCache(key, { + content: details.content, + url: contentUrl, + }); + } + + assetDetails.loaded = true; + + report(details.content); + }; + + let onCachedContentLoad = function (details) { + if (details.content !== '') { + report(details.content); + return; + } + + let onReady = function (registry) { + assetDetails = registry[key] || {}; + onContentNotLoaded(); + } + + getSourceRegistry(onReady); + }; + + readCache(key, onCachedContentLoad); + }; + + api.put = function (key, content, callback) { + writeCache(key, content, callback); + }; + + api.metadata = function (callback) { + let onSourceReady = function (registry) { + let source = JSON.parse(JSON.stringify(registry)); + let cache = cacheRegistry; + let sourceEntry; + let cacheEntry; + let now = Date.now(); + let obsoleteAfter; + + for (let key in source) { + sourceEntry = source[key]; + cacheEntry = cache[key]; + + if (cacheEntry) { + sourceEntry.cached = true; + sourceEntry.writeTime = cacheEntry.writeTime; + obsoleteAfter = cacheEntry.writeTime + + sourceEntry.updateAfter * 86400000; + sourceEntry.obsolete = obsoleteAfter < now; + sourceEntry.remoteURL = cacheEntry.remoteURL; + } else { + sourceEntry.writeTime = 0; + obsoleteAfter = 0; + sourceEntry.obsolete = true; + } + } - var onContentNotLoaded = function() { - var isExternal; - while ( (contentURL = contentURLs.shift()) ) { - isExternal = reIsExternalPath.test(contentURL); - if ( isExternal === false || assetDetails.hasLocalURL !== true ) { - break; - } - } - if ( !contentURL ) { - return reportBack('', 'E_NOTFOUND'); - } - api.fetchText(contentURL, onContentLoaded, onContentNotLoaded); + callback(source); + } + + let onCacheReady = function () { + getSourceRegistry(onSourceReady); + } + + getCacheRegistry(onCacheReady); + }; + + api.purge = function (pattern, exclude, callback) { + markDirtyCache(pattern, exclude, callback); }; - var onContentLoaded = function(details) { - if ( stringIsNotEmpty(details.content) === false ) { - onContentNotLoaded(); - return; - } - if ( reIsExternalPath.test(contentURL) && options.dontCache !== true ) { - assetCacheWrite(assetKey, { - content: details.content, - url: contentURL - }); - } - reportBack(details.content); + api.remove = function (pattern, callback) { + cacheRemove(pattern, callback); }; - var onCachedContentLoaded = function(details) { - if ( details.content !== '' ) { - return reportBack(details.content); - } - getAssetSourceRegistry(function(registry) { - assetDetails = registry[assetKey] || {}; - if ( typeof assetDetails.contentURL === 'string' ) { - contentURLs = [ assetDetails.contentURL ]; - } else if ( Array.isArray(assetDetails.contentURL) ) { - contentURLs = assetDetails.contentURL.slice(0); - } else { - contentURLs = []; - } - onContentNotLoaded(); - }); + api.rmrf = function () { + cacheRemove(/./); }; - assetCacheRead(assetKey, onCachedContentLoaded); -}; - -/******************************************************************************/ + api.updateStart = function (details) { + let oldDelay = updateDelay; + let newDelay = details.delay || updateDefaultDelay; -var getRemote = function(assetKey, callback) { - var assetDetails = {}, - contentURLs, - contentURL; - - var reportBack = function(content, err) { - var details = { assetKey: assetKey, content: content }; - if ( err ) { - details.error = assetDetails.lastError = err; - } else { - assetDetails.lastError = undefined; - } - callback(details); - }; + updateDelay = Math.min(oldDelay, newDelay); - var onRemoteContentLoaded = function(details) { - if ( stringIsNotEmpty(details.content) === false ) { - registerAssetSource(assetKey, { error: { time: Date.now(), error: 'No content' } }); - tryLoading(); - return; - } - assetCacheWrite(assetKey, { - content: details.content, - url: contentURL - }); - registerAssetSource(assetKey, { error: undefined }); - reportBack(details.content); - }; + if (updateStatus === 'running') { + if (newDelay < oldDelay) { + clearTimeout(updateTimer); + updateTimer = vAPI.setTimeout(updateNext, updateDelay); + } + return; + } - var onRemoteContentError = function(details) { - var text = details.statusText; - if ( details.statusCode === 0 ) { - text = 'network error'; - } - registerAssetSource(assetKey, { error: { time: Date.now(), error: text } }); - tryLoading(); + updateStart(); }; - var tryLoading = function() { - while ( (contentURL = contentURLs.shift()) ) { - if ( reIsExternalPath.test(contentURL) ) { break; } - } - if ( !contentURL ) { - return reportBack('', 'E_NOTFOUND'); - } - api.fetchText(contentURL, onRemoteContentLoaded, onRemoteContentError); + api.updateStop = function () { + if (updateTimer) { + clearTimeout(updateTimer); + updateTimer = undefined; + } + if (updateStatus === 'running') { + updateEnd(); + } }; - getAssetSourceRegistry(function(registry) { - assetDetails = registry[assetKey] || {}; - if ( typeof assetDetails.contentURL === 'string' ) { - contentURLs = [ assetDetails.contentURL ]; - } else if ( Array.isArray(assetDetails.contentURL) ) { - contentURLs = assetDetails.contentURL.slice(0); - } else { - contentURLs = []; - } - tryLoading(); - }); -}; - -/******************************************************************************/ - -api.put = function(assetKey, content, callback) { - assetCacheWrite(assetKey, content, callback); -}; - -/******************************************************************************/ - -api.metadata = function(callback) { - var assetRegistryReady = false, - cacheRegistryReady = false; - - var onReady = function() { - var assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)), - cacheDict = assetCacheRegistry, - assetEntry, cacheEntry, - now = Date.now(), obsoleteAfter; - for ( var assetKey in assetDict ) { - assetEntry = assetDict[assetKey]; - cacheEntry = cacheDict[assetKey]; - if ( cacheEntry ) { - assetEntry.cached = true; - assetEntry.writeTime = cacheEntry.writeTime; - obsoleteAfter = cacheEntry.writeTime + assetEntry.updateAfter * 86400000; - assetEntry.obsolete = obsoleteAfter < now; - assetEntry.remoteURL = cacheEntry.remoteURL; - } else { - assetEntry.writeTime = 0; - obsoleteAfter = 0; - assetEntry.obsolete = true; - } - } - callback(assetDict); - }; - - getAssetSourceRegistry(function() { - assetRegistryReady = true; - if ( cacheRegistryReady ) { onReady(); } - }); - - getAssetCacheRegistry(function() { - cacheRegistryReady = true; - if ( assetRegistryReady ) { onReady(); } - }); -}; - -/******************************************************************************/ - -api.purge = assetCacheMarkAsDirty; - -api.remove = function(pattern, callback) { - assetCacheRemove(pattern, callback); -}; - -api.rmrf = function() { - assetCacheRemove(/./); -}; - -/******************************************************************************/ - -// Asset updater area. -var updaterStatus, - updaterTimer, - updaterAssetDelayDefault = 120000, - updaterAssetDelay = updaterAssetDelayDefault, - updaterUpdated = [], - updaterFetched = new Set(); - -var updateFirst = function() { - updaterStatus = 'updating'; - updaterFetched.clear(); - updaterUpdated = []; - fireNotification('before-assets-updated'); - updateNext(); -}; - -var updateNext = function() { - var assetDict, cacheDict; - - // This will remove a cached asset when it's no longer in use. - var garbageCollectOne = function(assetKey) { - var cacheEntry = cacheDict[assetKey]; - if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { - assetCacheRemove(assetKey); - } - }; - - var findOne = function() { - var now = Date.now(), - assetEntry, cacheEntry; - for ( var assetKey in assetDict ) { - assetEntry = assetDict[assetKey]; - if ( assetEntry.hasRemoteURL !== true ) { continue; } - if ( updaterFetched.has(assetKey) ) { continue; } - cacheEntry = cacheDict[assetKey]; - if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { - continue; - } - if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { - return assetKey; - } - garbageCollectOne(assetKey); - } - }; - - var updatedOne = function(details) { - if ( details.content !== '' ) { - updaterUpdated.push(details.assetKey); - if ( details.assetKey === 'assets.json' ) { - updateAssetSourceRegistry(details.content); - } - } else { - fireNotification('asset-update-failed', { assetKey: details.assetKey }); - } - if ( findOne() !== undefined ) { - vAPI.setTimeout(updateNext, updaterAssetDelay); - } else { - updateDone(); - } - }; - - var updateOne = function() { - var assetKey = findOne(); - if ( assetKey === undefined ) { - return updateDone(); - } - updaterFetched.add(assetKey); - getRemote(assetKey, updatedOne); - }; - - getAssetSourceRegistry(function(dict) { - assetDict = dict; - if ( !cacheDict ) { return; } - updateOne(); - }); - - getAssetCacheRegistry(function(dict) { - cacheDict = dict; - if ( !assetDict ) { return; } - updateOne(); - }); -}; - -var updateDone = function() { - var assetKeys = updaterUpdated.slice(0); - updaterFetched.clear(); - updaterUpdated = []; - updaterStatus = undefined; - updaterAssetDelay = updaterAssetDelayDefault; - fireNotification('after-assets-updated', { assetKeys: assetKeys }); -}; - -api.updateStart = function(details) { - var oldUpdateDelay = updaterAssetDelay, - newUpdateDelay = details.delay || updaterAssetDelayDefault; - updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); - if ( updaterStatus !== undefined ) { - if ( newUpdateDelay < oldUpdateDelay ) { - clearTimeout(updaterTimer); - updaterTimer = vAPI.setTimeout(updateNext, updaterAssetDelay); - } - return; - } - updateFirst(); -}; - -api.updateStop = function() { - if ( updaterTimer ) { - clearTimeout(updaterTimer); - updaterTimer = undefined; - } - if ( updaterStatus !== undefined ) { - updateDone(); - } -}; - -/******************************************************************************/ - -return api; - -/******************************************************************************/ - + return api; })(); /******************************************************************************/ |