[adapters] 增加资源管线的多线程支持

This commit is contained in:
SmallMain 2024-10-21 20:23:37 +08:00
parent 6aec2d7cfa
commit 29427f6bf6
No known key found for this signature in database
18 changed files with 1772 additions and 60 deletions

View File

@ -0,0 +1,144 @@
const { getUserDataPath, readJsonSync, makeDirSync, writeFileSync, copyFile, downloadFile, writeFile, deleteFile, rmdirSync, unzip, isOutOfStorage } = window.fsUtils;
var cacheManager = {
// 以下为单向属性,修改会同步到 Worker 中
_cacheDir: 'gamecaches',
get cacheDir() {
return this._cacheDir;
},
set cacheDir(v) {
worker.cacheManager.set_cacheDir([this._cacheDir = v]);
},
_cachedFileName: 'cacheList.json',
get cachedFileName() {
return this._cachedFileName;
},
set cachedFileName(v) {
worker.cacheManager.set_cachedFileName([this._cachedFileName = v]);
},
_cacheEnabled: true,
get cacheEnabled() {
return this._cacheEnabled;
},
set cacheEnabled(v) {
worker.cacheManager.set_cacheEnabled([this._cacheEnabled = v]);
},
_autoClear: true,
get autoClear() {
return this._autoClear;
},
set autoClear(v) {
worker.cacheManager.set_autoClear([this._autoClear = v]);
},
_cacheInterval: 500,
get cacheInterval() {
return this._cacheInterval;
},
set cacheInterval(v) {
worker.cacheManager.set_cacheInterval([this._cacheInterval = v]);
},
_deleteInterval: 500,
get deleteInterval() {
return this._deleteInterval;
},
set deleteInterval(v) {
worker.cacheManager.set_deleteInterval([this._deleteInterval = v]);
},
// 以下属性未暴露,仅在 Worker 中保留
// writeFileInterval: 2000,
// outOfStorage: false,
// tempFiles: null,
// cacheQueue: {},
// 以下为只读属性,并且在变动时从 Worker 中同步,需注意 lastTime 可能不是最新的
cachedFiles: null,
// 以下为只读属性
version: '1.0',
// 增加 download 函数以在 Worker 执行下载逻辑
download(url, func, options, onFileProgress, onComplete) {
// 暂未实现 onFileProgress 回调
worker.cacheManager.download(
[url, options.reload, options.header, options.cacheEnabled, options.__cacheBundleRoot__],
([errMsg, path]) => {
if (errMsg) {
onComplete(new Error(errMsg), null);
return;
}
func(path, options, (err, data) => {
if (err) {
this.removeCache(url);
}
onComplete(err, data);
});
},
);
},
// 增加 handleZip 函数以在 Worker 执行处理逻辑
handleZip(url, options, onComplete) {
// 暂未实现 options.onFileProgress 回调
worker.cacheManager.handleZip(
[url, options.header, options.__cacheBundleRoot__],
([errMsg, path]) => {
if (errMsg) {
onComplete(new Error(errMsg), null);
} else {
onComplete(null, path);
}
},
);
},
getCache(url) {
return this.cachedFiles.has(url) ? this.cachedFiles.get(url).url : '';
},
// getTemp 改为异步函数
getTempAsync(url, callback) {
worker.cacheManager.getTemp([url], ([url]) => {
callback(url);
});
},
init() {
this._cacheDir = getUserDataPath() + '/' + this.cacheDir;
worker.cacheManager.init(null, ([cachedFiles]) => {
this.cachedFiles = new cc.AssetManager.Cache(cachedFiles);
});
},
clearCache() {
worker.cacheManager.clearCache(null, () => {
this.cachedFiles.clear();
});
},
clearLRU() {
worker.cacheManager.clearLRU(null, ([deletedFiles]) => {
for (let i = 0, l = deletedFiles.length; i < l; i++) {
this.cachedFiles.remove(deletedFiles[i]);
}
});
},
removeCache(url) {
worker.cacheManager.removeCache([url], () => {
this.cachedFiles.remove(url);
});
},
makeBundleFolder(bundleName) {
makeDirSync(this.cacheDir + '/' + bundleName, true);
},
};
cc.assetManager.cacheManager = module.exports = cacheManager;

View File

@ -1,4 +1,4 @@
const cacheManager = require('../cache-manager'); const cacheManager = CC_WORKER_ASSET_PIPELINE ? require('../cache-manager-proxy') : require('../cache-manager');
const { fs, downloadFile, readText, readArrayBuffer, readJson, loadSubpackage, getUserDataPath, exists } = window.fsUtils; const { fs, downloadFile, readText, readArrayBuffer, readJson, loadSubpackage, getUserDataPath, exists } = window.fsUtils;
const REGEX = /^https?:\/\/.*/; const REGEX = /^https?:\/\/.*/;
@ -29,7 +29,11 @@ function downloadScript (url, options, onComplete) {
} }
} }
function handleZip (url, options, onComplete) { var handleZip = CC_WORKER_ASSET_PIPELINE
? function handleZip(url, options, onComplete) {
cacheManager.handleZip(url, options, onComplete);
}
: function handleZip(url, options, onComplete) {
let cachedUnzip = cacheManager.cachedFiles.get(url); let cachedUnzip = cacheManager.cachedFiles.get(url);
if (cachedUnzip) { if (cachedUnzip) {
cacheManager.updateLastTime(url); cacheManager.updateLastTime(url);
@ -68,7 +72,11 @@ function downloadDomAudio (url, options, onComplete) {
onComplete && onComplete(null, dom); onComplete && onComplete(null, dom);
} }
function download (url, func, options, onFileProgress, onComplete) { var download = CC_WORKER_ASSET_PIPELINE
? function download(url, func, options, onFileProgress, onComplete) {
cacheManager.download(url, func, options, onFileProgress, onComplete);
}
: function download(url, func, options, onFileProgress, onComplete) {
var result = transformUrl(url, options); var result = transformUrl(url, options);
if (result.inLocal) { if (result.inLocal) {
func(result.url, options, onComplete); func(result.url, options, onComplete);
@ -132,6 +140,10 @@ var loadFont = !isSubDomain ? function (url, options, onComplete) {
} }
function doNothing(content, options, onComplete) { function doNothing(content, options, onComplete) {
if (CC_WORKER_ASSET_PIPELINE) {
onComplete(null, content);
return;
}
exists(content, (existence) => { exists(content, (existence) => {
if (existence) { if (existence) {
onComplete(null, content); onComplete(null, content);
@ -402,6 +414,9 @@ parser.register({
}); });
var transformUrl = !isSubDomain ? function (url, options) { var transformUrl = !isSubDomain ? function (url, options) {
if (CC_WORKER_ASSET_PIPELINE) {
console.error('transformUrl should not be called when the macro CC_WORKER_ASSET_PIPELINE is enabled.');
}
var inLocal = false; var inLocal = false;
var inCache = false; var inCache = false;
var isInUserDataPath = url.startsWith(getUserDataPath()); var isInUserDataPath = url.startsWith(getUserDataPath());

View File

@ -1,3 +1,4 @@
const initWorker = require('./worker_adapter/index.js');
require('adapter-js-path'); require('adapter-js-path');
__globalAdapter.init(); __globalAdapter.init();
require('cocos2d-js-path'); require('cocos2d-js-path');
@ -17,4 +18,6 @@ if (cc.sys.platform !== cc.sys.WECHAT_GAME_SUB) {
cc.macro.CLEANUP_IMAGE_CACHE = true; cc.macro.CLEANUP_IMAGE_CACHE = true;
} }
initWorker(() => {
window.boot(); window.boot();
});

View File

@ -13,5 +13,9 @@
"version": "2.4.15", "version": "2.4.15",
"path": "cocos" "path": "cocos"
} }
},
"workers": {
"path": "workers",
"isSubpackage": false
} }
} }

View File

@ -0,0 +1,27 @@
var assetManagerWorkerAdapter = {
// 返回当前 cc.assetManager.bundles 的 [name, base]
getAllBundles(args, cmdId, callback) {
var bundles = [];
cc.assetManager.bundles.forEach((v, k) => {
bundles.push([v.name, v.base]);
});
callback(cmdId, [bundles]);
},
// 删除缓存文件记录
removeCachedFiles(args, cmdId, callback) {
const deletedFiles = args[0];
for (let i = 0, l = deletedFiles.length; i < l; i++) {
cc.assetManager.cacheManager.cachedFiles.remove(deletedFiles[i]);
}
},
// 添加缓存文件记录
addCachedFiles(args, cmdId, callback) {
const addedFiles = args[0];
for (let i = 0, l = addedFiles.length; i < l; i++) {
const [id, cacheBundleRoot, localPath, time] = addedFiles[i];
cc.assetManager.cacheManager.cachedFiles.add(id, { bundle: cacheBundleRoot, url: localPath, lastTime: time });
}
},
};
module.exports = assetManagerWorkerAdapter;

View File

@ -0,0 +1,4 @@
if (CC_WORKER_ASSET_PIPELINE) {
const assetManagerWorkerAdapter = require("./asset-manager.js");
ipcMain.registerHandler("assetManager", assetManagerWorkerAdapter);
}

View File

@ -0,0 +1,15 @@
require("./macro.js");
require("./ipc-main.js");
require("./handlers.js");
module.exports = function init(callback) {
if (CC_USE_WORKER) {
var t = Date.now();
ipcMain.init(() => {
console.log("worker init cost:", Date.now() - t);
callback();
});
} else {
callback();
}
}

View File

@ -0,0 +1,220 @@
const ipcMain = {
worker: null,
cmdId: 0,
callbacks: {},
_cmd: 0,
handlers: {},
_inited: false,
_initCallback: null,
init(callback) {
this._initCallback = callback;
this.worker = wx.createWorker("workers/index.js", { useExperimentalWorker: true });
this.worker.onProcessKilled(() => {
console.warn("worker has been killed");
this.worker.terminate();
this.worker = null;
// TODO 这里还未正确处理,还需要重新 init cacheManager 等等,把这边对属性的修改同步过来
});
this.worker.onMessage(
CC_WORKER_SCHEDULER
? msgs => {
for (let index = 0; index < msgs.length; index++) {
const msg = msgs[index];
this._handleWorkerMessage(msg);
}
}
: this._handleWorkerMessage.bind(this)
);
if (CC_WORKER_SCHEDULER) {
sendScheduler.init(this);
}
this._init();
},
_handleWorkerMessage(msg) {
// 格式:[id, cmd, [args] | null]
// 如果 cmd 是正数,则是返回到主线程的 Callback
// 反之,是 Worker 的调用
// [-, 0, handlers] 为初始化调用
const id = msg[0];
const cmd = msg[1];
const args = msg[2];
if (cmd > 0) {
if (CC_WORKER_DEBUG) {
console.log("main thread recv callback:", msg);
}
const callback = this.callbacks[id];
if (callback) {
callback(args);
delete this.callbacks[id];
}
} else if (cmd < 0) {
const handler = this.handlers[cmd];
if (handler) {
const { func, name, key, callback } = handler;
if (CC_WORKER_DEBUG) {
console.log(`main thread recv call (${name}.${key}):`, msg);
}
func(args, id, callback);
} else {
console.error("main thread recv unknown call:", msg);
}
} else {
if (CC_WORKER_DEBUG) {
console.log("main thread recv init:", msg);
}
this._initFromWorker(args);
}
},
_init() {
const _handlers = [];
for (const cmd in this.handlers) {
const { name, key } = this.handlers[cmd];
_handlers.push({ name, cmd: Number(cmd), key });
}
this.callToWorker(0, [
_handlers,
CC_WORKER_FS_SYNC,
CC_WORKER_ASSET_PIPELINE,
]);
},
_initFromWorker(wrappers) {
for (const wrapper of wrappers) {
const { name, key, cmd } = wrapper;
if (!worker[name]) {
worker[name] = {};
}
worker[name][key] = (args, callback) => {
this.callToWorker(cmd, args, callback);
};
}
this._inited = true;
if (this._initCallback) this._initCallback();
},
callbackToWorker(id, cmd, args) {
const msg = [id, cmd, args];
if (CC_WORKER_DEBUG) {
console.log("main thread send callback:", msg);
}
if (CC_WORKER_SCHEDULER) {
sendScheduler.send(msg);
} else {
this.worker.postMessage(msg);
}
},
callToWorker(cmd, args, callback) {
const id = ++this.cmdId;
if (callback) {
this.callbacks[id] = callback;
}
const msg = [id, cmd, args];
if (CC_WORKER_DEBUG) {
console.log("main thread send call:", msg);
}
if (CC_WORKER_SCHEDULER) {
sendScheduler.send(msg);
} else {
this.worker.postMessage(msg);
}
},
registerHandler(name, obj) {
const descs = Object.getOwnPropertyDescriptors(obj);
for (const key in descs) {
const desc = descs[key];
if (typeof desc.value === "function") {
const cmd = ++this._cmd;
this.handlers[cmd] = {
name,
key,
func: obj[key].bind(obj),
callback: (id, args) => this.callbackToWorker(id, cmd, args),
};
} else {
// getter/setter
const cmd1 = ++this._cmd;
this.handlers[cmd1] = {
name,
key: "get_" + key,
func: (args, id, callback) => {
this.callbackToWorker(id, cmd1, [obj[key]]);
},
callback: null,
};
const cmd2 = ++this._cmd;
this.handlers[cmd2] = {
name,
key: "set_" + key,
func: (args, id, callback) => {
obj[key] = args[0];
}
};
const cmd3 = ++this._cmd;
this.handlers[cmd3] = {
name,
key: "write_" + key,
func: (args, id, callback) => {
obj[key] = args[0];
this.callbackToWorker(id, cmd3, null);
}
};
}
}
},
};
const sendScheduler = {
queue: [],
ipc: null,
init(ipc) {
this.ipc = ipc;
setInterval(() => {
if (this.queue.length > 0) {
this.ipc.worker.postMessage(this.queue);
this.queue = [];
}
}, 0);
},
send(msg) {
this.queue.push(msg);
},
};
const worker = {};
globalThis.ipcMain = ipcMain;
globalThis.worker = worker;

View File

@ -0,0 +1,23 @@
const isSubContext = wx.getOpenDataContext === undefined;
const sysinfo = wx.getSystemInfoSync();
const platform = sysinfo.platform.toLowerCase();
const isIOS = platform === "ios";
const sdkVersion = sysinfo.SDKVersion.split('.').map(Number);
// >= 2.20.2
const hasWorker = sdkVersion[0] > 2 || (sdkVersion[0] === 2 && (sdkVersion[1] > 20 || (sdkVersion[1] === 20 && sdkVersion[2] >= 2)));
// 是否启用 Worker 驱动资源管线(下载、缓存)
globalThis.CC_WORKER_ASSET_PIPELINE = false;
// 是否启用 Worker
globalThis.CC_USE_WORKER = (CC_WORKER_ASSET_PIPELINE) && hasWorker && !isSubContext;
// 是否启用 Worker 调试模式
globalThis.CC_WORKER_DEBUG = true;
// 是否启用 Worker 调度模式,这也许能减少通信次数带来的性能消耗(必须一致)
globalThis.CC_WORKER_SCHEDULER = true;
// 是否启用 Worker 使用同步版本的文件系统 API
// NOTE: IOS 不支持 async 文件系统 APIAndroid 不支持部分 sync 文件系统 API
globalThis.CC_WORKER_FS_SYNC = isIOS;

View File

@ -0,0 +1,347 @@
const { getUserDataPath, readJsonSync, deleteFileSync, makeDirSync, writeFileSync, copyFile, downloadFile, writeFile, deleteFile, rmdirSync, unzip, isOutOfStorage } = require("./fs-utils.js");
const { extname } = require("./path.js");
const { main } = require("./ipc-worker.js");
var checkNextPeriod = false;
var writeCacheFileTimer = null;
var suffix = 0;
const REGEX = /^https?:\/\/.*/;
function isEmptyObject(obj) {
for (var key in obj) {
return false;
}
return true;
}
var cacheManager_worker = {
cacheDir: 'gamecaches',
cachedFileName: 'cacheList.json',
// whether or not cache asset into user's storage space
cacheEnabled: true,
// whether or not auto clear cache when storage ran out
autoClear: true,
// cache one per cycle
cacheInterval: 500,
deleteInterval: 500,
writeFileInterval: 2000,
// whether or not storage space has run out
outOfStorage: false,
tempFiles: null,
cachedFiles: null,
cacheQueue: {},
version: '1.0',
init(callback) {
this.cacheDir = getUserDataPath() + '/' + this.cacheDir;
var cacheFilePath = this.cacheDir + '/' + this.cachedFileName;
var result = readJsonSync(cacheFilePath);
if (result instanceof Error || !result.version) {
if (!(result instanceof Error)) rmdirSync(this.cacheDir, true);
this.cachedFiles = {};
makeDirSync(this.cacheDir, true);
writeFileSync(cacheFilePath, JSON.stringify({ files: this.cachedFiles, version: this.version }), 'utf8');
}
else {
this.cachedFiles = result.files;
}
this.tempFiles = {};
callback(this.cachedFiles);
},
transformUrl(url, reload) {
var inLocal = false;
var inCache = false;
var isInUserDataPath = url.startsWith(getUserDataPath());
if (isInUserDataPath) {
inLocal = true;
} else if (REGEX.test(url)) {
if (!reload) {
var cache = this.cachedFiles[url];
if (cache) {
inCache = true;
url = cache.url;
} else {
var tempUrl = this.tempFiles[url];
if (tempUrl) {
inLocal = true;
url = tempUrl;
}
}
}
} else {
inLocal = true;
}
return { url, inLocal, inCache };
},
download(
callback,
url,
options_reload,
options_header,
options_cacheEnabled,
options___cacheBundleRoot__,
) {
var result = this.transformUrl(url, options_reload);
if (result.inLocal) {
callback(null, result.url);
} else if (result.inCache) {
this.updateLastTime(url);
callback(null, result.url);
}
else {
downloadFile(url, null, options_header, null, (err, path) => {
if (err) {
callback(err.message, null);
return;
}
this.tempFiles[url] = path;
this.cacheFile(null, url, path, options_cacheEnabled, options___cacheBundleRoot__, true);
callback(null, path);
});
}
},
handleZip(
callback,
url,
options_header,
options___cacheBundleRoot__,
) {
let cachedUnzip = this.cachedFiles[url];
if (cachedUnzip) {
this.updateLastTime(url);
callback(null, cachedUnzip.url);
} else if (REGEX.test(url)) {
downloadFile(url, null, options_header, null, (err, downloadedZipPath) => {
if (err) {
callback(err.message, null);
return;
}
this.unzipAndCacheBundle(url, downloadedZipPath, options___cacheBundleRoot__, callback);
});
} else {
this.unzipAndCacheBundle(url, url, options___cacheBundleRoot__, callback);
}
},
getTemp(callback, url) {
callback(this.tempFiles.has(url) ? this.tempFiles.get(url) : '');
},
updateLastTime(url) {
if (this.cachedFiles[url]) {
var cache = this.cachedFiles[url];
cache.lastTime = Date.now();
}
},
writeCacheFile() {
if (!writeCacheFileTimer) {
writeCacheFileTimer = setTimeout(() => {
writeCacheFileTimer = null;
writeFile(this.cacheDir + '/' + this.cachedFileName, JSON.stringify({ files: this.cachedFiles, version: this.version }), "utf8", () => { });
}, this.writeFileInterval);
}
},
writeCacheFileSync() {
writeFileSync(this.cacheDir + '/' + this.cachedFileName, JSON.stringify({ files: this.cachedFiles, version: this.version }), "utf8");
},
_cache() {
var self = this;
for (var id in this.cacheQueue) {
var { srcUrl, isCopy, cacheBundleRoot, callback: _callback } = this.cacheQueue[id];
var time = Date.now().toString();
var localPath = '';
if (cacheBundleRoot) {
localPath = `${this.cacheDir}/${cacheBundleRoot}/${time}${suffix++}${extname(id)}`;
}
else {
localPath = `${this.cacheDir}/${time}${suffix++}${extname(id)}`;
}
function callback(err) {
checkNextPeriod = false;
if (err) {
if (isOutOfStorage(err.message)) {
self.outOfStorage = true;
self.autoClear && self.clearLRU();
return;
}
} else {
self.cachedFiles[id] = { bundle: cacheBundleRoot, url: localPath, lastTime: time };
delete self.cacheQueue[id];
self.writeCacheFile();
// TODO main.assetManager.addCachedFiles([[id, cacheBundleRoot, localPath, time]]);
if (_callback) _callback(id, cacheBundleRoot, localPath, time);
}
if (!isEmptyObject(self.cacheQueue)) {
checkNextPeriod = true;
setTimeout(self._cache.bind(self), self.cacheInterval);
}
}
if (!isCopy) {
downloadFile(srcUrl, localPath, null, callback);
}
else {
copyFile(srcUrl, localPath, callback);
}
return;
}
checkNextPeriod = false;
},
cacheFile(callback, id, srcUrl, cacheEnabled, cacheBundleRoot, isCopy) {
cacheEnabled = cacheEnabled !== undefined ? cacheEnabled : this.cacheEnabled;
if (!cacheEnabled || this.cacheQueue[id] || this.cachedFiles[id]) {
if (callback) callback(null);
return;
}
this.cacheQueue[id] = { srcUrl, cacheBundleRoot, isCopy, callback };
if (!checkNextPeriod) {
checkNextPeriod = true;
if (!this.outOfStorage) {
setTimeout(this._cache.bind(this), this.cacheInterval);
}
else {
checkNextPeriod = false;
}
}
},
clearCache(callback) {
main.assetManager.getAllBundles(bundles => {
this.cachedFiles = {};
this.writeCacheFileSync();
rmdirSync(this.cacheDir, true);
makeDirSync(this.cacheDir, true);
this.outOfStorage = false;
bundles.forEach(bundle => {
const [name, base] = bundle;
if (REGEX.test(base)) this.makeBundleFolder(name);
});
callback();
});
},
clearLRU(callback) {
main.assetManager.getAllBundles(bundles => {
var caches = [];
var self = this;
for (const key in this.cachedFiles) {
const val = this.cachedFiles[key];
if (val.bundle === 'internal') continue;
if (self._isZipFile(key) && bundles.find(bundle => bundle[1].indexOf(val.url) !== -1)) continue;
caches.push({ originUrl: key, url: val.url, lastTime: val.lastTime });
}
caches.sort(function (a, b) {
return a.lastTime - b.lastTime;
});
caches.length = Math.floor(caches.length / 3);
if (caches.length === 0) {
if (callback) {
callback([]);
}
return;
}
for (var i = 0, l = caches.length; i < l; i++) {
const item = caches[i];
delete this.cachedFiles[item.originUrl];
if (self._isZipFile(item.originUrl)) {
rmdirSync(item.url, true);
} else {
deleteFileSync(item.url);
}
}
this.outOfStorage = false;
self.writeCacheFileSync();
if (callback) {
callback(caches.map(v => v.originUrl));
} else {
main.assetManager.removeCachedFiles(caches.map(v => v.originUrl));
}
});
},
removeCache(callback, url) {
if (this.cachedFiles[url]) {
var self = this;
var path = this.cachedFiles[url].url;
delete this.cachedFiles[url];
if (self._isZipFile(url)) {
rmdirSync(path, true);
} else {
deleteFileSync(path);
}
this.outOfStorage = false;
self.writeCacheFileSync();
}
if (callback) callback();
},
makeBundleFolder(bundleName) {
makeDirSync(this.cacheDir + '/' + bundleName, true);
},
unzipAndCacheBundle(id, zipFilePath, cacheBundleRoot, onComplete) {
let time = Date.now().toString();
let targetPath = `${this.cacheDir}/${cacheBundleRoot}/${time}${suffix++}`;
let self = this;
makeDirSync(targetPath, true);
unzip(zipFilePath, targetPath, function (err) {
if (err) {
rmdirSync(targetPath, true);
if (isOutOfStorage(err.message)) {
self.outOfStorage = true;
self.autoClear && self.clearLRU();
}
onComplete && onComplete(err);
return;
}
self.cachedFiles[id] = { bundle: cacheBundleRoot, url: targetPath, lastTime: time };
self.writeCacheFile();
main.assetManager.addCachedFiles([[id, cacheBundleRoot, targetPath, time]]);
onComplete && onComplete(null, targetPath);
});
},
_isZipFile(url) {
return url.slice(-4) === '.zip';
},
};
module.exports = cacheManager_worker;

View File

@ -0,0 +1,261 @@
/****************************************************************************
Copyright (c) 2017-2019 Xiamen Yaji Software Co., Ltd.
https://www.cocos.com/
Permission is hereby granted, free of charge, to any person obtaining a copy
of fsUtils software and associated engine source code (the "Software"), a limited,
worldwide, royalty-free, non-assignable, revocable and non-exclusive license
to use Cocos Creator solely to develop games on your target platforms. You shall
not use Cocos Creator software for developing other software or tools that's
used for developing games. You are not granted to publish, distribute,
sublicense, and/or sell copies of Cocos Creator.
The software or tools in fsUtils License Agreement are licensed, not sold.
Xiamen Yaji Software Co., Ltd. reserves all rights not expressly granted to you.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
****************************************************************************/
var fs = worker.getFileSystemManager();
var outOfStorageRegExp = /the maximum size of the file storage/;
var fsUtils = {
fs,
isOutOfStorage(errMsg) {
return outOfStorageRegExp.test(errMsg);
},
getUserDataPath() {
return worker.env.USER_DATA_PATH;
},
checkFsValid() {
if (!fs) {
console.warn('can not get the file system!');
return false;
}
return true;
},
deleteFile(filePath, onComplete) {
fs.unlink({
filePath: filePath,
success: function () {
onComplete && onComplete(null);
},
fail: function (res) {
console.warn(`Delete file failed: path: ${filePath} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg));
}
});
},
deleteFileSync(filePath, onComplete) {
try {
fs.unlinkSync(filePath);
onComplete && onComplete(null);
} catch (error) {
console.warn(`Delete file failed: path: ${filePath} message: ${error.message}`);
onComplete && onComplete(error);
}
},
downloadFile(remoteUrl, filePath, header, onProgress, onComplete) {
var options = {
url: remoteUrl,
success: function (res) {
if (res.statusCode === 200) {
onComplete && onComplete(null, res.tempFilePath || res.filePath);
}
else {
if (res.filePath) {
fsUtils.deleteFile(res.filePath);
}
console.warn(`Download file failed: path: ${remoteUrl} message: ${res.statusCode}`);
onComplete && onComplete(new Error(res.statusCode), null);
}
},
fail: function (res) {
console.warn(`Download file failed: path: ${remoteUrl} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg), null);
}
}
if (filePath) options.filePath = filePath;
if (header) options.header = header;
var task = worker.downloadFile(options);
onProgress && task.onProgressUpdate(onProgress);
},
saveFile(srcPath, destPath, onComplete) {
fs.saveFile({
tempFilePath: srcPath,
filePath: destPath,
success: function (res) {
onComplete && onComplete(null);
},
fail: function (res) {
console.warn(`Save file failed: path: ${srcPath} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg));
}
});
},
copyFile(srcPath, destPath, onComplete) {
fs.copyFile({
srcPath: srcPath,
destPath: destPath,
success: function () {
onComplete && onComplete(null);
},
fail: function (res) {
console.warn(`Copy file failed: path: ${srcPath} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg));
}
});
},
writeFile(path, data, encoding, onComplete) {
fs.writeFile({
filePath: path,
encoding: encoding,
data: data,
success: function () {
onComplete && onComplete(null);
},
fail: function (res) {
console.warn(`Write file failed: path: ${path} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg));
}
});
},
writeFileSync(path, data, encoding) {
try {
fs.writeFileSync(path, data, encoding);
return null;
}
catch (e) {
console.warn(`Write file failed: path: ${path} message: ${e.message}`);
return new Error(e.message);
}
},
readFile(filePath, encoding, onComplete) {
fs.readFile({
filePath: filePath,
encoding: encoding,
success: function (res) {
onComplete && onComplete(null, res.data);
},
fail: function (res) {
console.warn(`Read file failed: path: ${filePath} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg), null);
}
});
},
readDir(filePath, onComplete) {
fs.readdir({
dirPath: filePath,
success: function (res) {
onComplete && onComplete(null, res.files);
},
fail: function (res) {
console.warn(`Read directory failed: path: ${filePath} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg), null);
}
});
},
readText(filePath, onComplete) {
fsUtils.readFile(filePath, 'utf8', onComplete);
},
readArrayBuffer(filePath, onComplete) {
fsUtils.readFile(filePath, '', onComplete);
},
readJson(filePath, onComplete) {
fsUtils.readFile(filePath, 'utf8', function (err, text) {
var out = null;
if (!err) {
try {
out = JSON.parse(text);
}
catch (e) {
console.warn(`Read json failed: path: ${filePath} message: ${e.message}`);
err = new Error(e.message);
}
}
onComplete && onComplete(err, out);
});
},
readJsonSync(path) {
try {
var str = fs.readFileSync(path, 'utf8');
return JSON.parse(str);
}
catch (e) {
console.warn(`Read json failed: path: ${path} message: ${e.message}`);
return new Error(e.message);
}
},
makeDirSync(path, recursive) {
try {
fs.mkdirSync(path, recursive);
return null;
}
catch (e) {
console.warn(`Make directory failed: path: ${path} message: ${e.message}`);
return new Error(e.message);
}
},
rmdirSync(dirPath, recursive) {
try {
fs.rmdirSync(dirPath, recursive);
}
catch (e) {
console.warn(`rm directory failed: path: ${dirPath} message: ${e.message}`);
return new Error(e.message);
}
},
exists(filePath, onComplete) {
fs.access({
path: filePath,
success: function () {
onComplete && onComplete(true);
},
fail: function () {
onComplete && onComplete(false);
}
});
},
unzip(zipFilePath, targetPath, onComplete) {
fs.unzip({
zipFilePath,
targetPath,
success() {
onComplete && onComplete(null);
},
fail(res) {
console.warn(`unzip failed: path: ${zipFilePath} message: ${res.errMsg}`);
onComplete && onComplete(new Error('unzip failed: ' + res.errMsg));
},
})
},
};
module.exports = fsUtils;

View File

@ -0,0 +1,235 @@
/****************************************************************************
Copyright (c) 2017-2019 Xiamen Yaji Software Co., Ltd.
https://www.cocos.com/
Permission is hereby granted, free of charge, to any person obtaining a copy
of fsUtils software and associated engine source code (the "Software"), a limited,
worldwide, royalty-free, non-assignable, revocable and non-exclusive license
to use Cocos Creator solely to develop games on your target platforms. You shall
not use Cocos Creator software for developing other software or tools that's
used for developing games. You are not granted to publish, distribute,
sublicense, and/or sell copies of Cocos Creator.
The software or tools in fsUtils License Agreement are licensed, not sold.
Xiamen Yaji Software Co., Ltd. reserves all rights not expressly granted to you.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
****************************************************************************/
var fs = worker.getFileSystemManager();
var outOfStorageRegExp = /the maximum size of the file storage/;
var fsUtils = {
fs,
isOutOfStorage(errMsg) {
return outOfStorageRegExp.test(errMsg);
},
getUserDataPath() {
return worker.env.USER_DATA_PATH;
},
checkFsValid() {
if (!fs) {
console.warn('can not get the file system!');
return false;
}
return true;
},
deleteFile(filePath, onComplete) {
try {
fs.unlinkSync(filePath);
onComplete && onComplete(null);
} catch (error) {
console.warn(`Delete file failed: path: ${filePath} message: ${error.message}`);
onComplete && onComplete(error);
}
},
deleteFileSync(filePath, onComplete) {
try {
fs.unlinkSync(filePath);
onComplete && onComplete(null);
} catch (error) {
console.warn(`Delete file failed: path: ${filePath} message: ${error.message}`);
onComplete && onComplete(error);
}
},
downloadFile(remoteUrl, filePath, header, onProgress, onComplete) {
var options = {
url: remoteUrl,
success: function (res) {
if (res.statusCode === 200) {
onComplete && onComplete(null, res.tempFilePath || res.filePath);
}
else {
if (res.filePath) {
fsUtils.deleteFile(res.filePath);
}
console.warn(`Download file failed: path: ${remoteUrl} message: ${res.statusCode}`);
onComplete && onComplete(new Error(res.statusCode), null);
}
},
fail: function (res) {
console.warn(`Download file failed: path: ${remoteUrl} message: ${res.errMsg}`);
onComplete && onComplete(new Error(res.errMsg), null);
}
}
if (filePath) options.filePath = filePath;
if (header) options.header = header;
var task = worker.downloadFile(options);
onProgress && task.onProgressUpdate(onProgress);
},
saveFile(srcPath, destPath, onComplete) {
try {
fs.saveFileSync(srcPath, destPath);
onComplete && onComplete(null);
} catch (error) {
console.warn(`Save file failed: path: ${srcPath} message: ${error.message}`);
onComplete && onComplete(error);
}
},
copyFile(srcPath, destPath, onComplete) {
try {
fs.copyFileSync(srcPath, destPath);
onComplete && onComplete(null);
} catch (error) {
console.warn(`Copy file failed: path: ${srcPath} message: ${error.message}`);
onComplete && onComplete(error);
}
},
writeFile(path, data, encoding, onComplete) {
try {
fs.writeFileSync(path, data, encoding);
onComplete && onComplete(null);
} catch (error) {
console.warn(`Write file failed: path: ${path} message: ${error.message}`);
onComplete && onComplete(error);
}
},
writeFileSync(path, data, encoding) {
try {
fs.writeFileSync(path, data, encoding);
return null;
}
catch (e) {
console.warn(`Write file failed: path: ${path} message: ${e.message}`);
return new Error(e.message);
}
},
readFile(filePath, encoding, onComplete) {
try {
var data = fs.readFileSync(filePath, encoding);
onComplete && onComplete(null, data);
} catch (error) {
console.warn(`Read file failed: path: ${filePath} message: ${error.message}`);
onComplete && onComplete(error, null);
}
},
readDir(filePath, onComplete) {
try {
var files = fs.readdirSync(filePath);
onComplete && onComplete(null, files);
} catch (error) {
console.warn(`Read directory failed: path: ${filePath} message: ${error.message}`);
onComplete && onComplete(error, null);
}
},
readText(filePath, onComplete) {
fsUtils.readFile(filePath, 'utf8', onComplete);
},
readArrayBuffer(filePath, onComplete) {
fsUtils.readFile(filePath, '', onComplete);
},
readJson(filePath, onComplete) {
fsUtils.readFile(filePath, 'utf8', function (err, text) {
var out = null;
if (!err) {
try {
out = JSON.parse(text);
}
catch (e) {
console.warn(`Read json failed: path: ${filePath} message: ${e.message}`);
err = new Error(e.message);
}
}
onComplete && onComplete(err, out);
});
},
readJsonSync(path) {
try {
var str = fs.readFileSync(path, 'utf8');
return JSON.parse(str);
}
catch (e) {
console.warn(`Read json failed: path: ${path} message: ${e.message}`);
return new Error(e.message);
}
},
makeDirSync(path, recursive) {
try {
fs.mkdirSync(path, recursive);
return null;
}
catch (e) {
console.warn(`Make directory failed: path: ${path} message: ${e.message}`);
return new Error(e.message);
}
},
rmdirSync(dirPath, recursive) {
try {
fs.rmdirSync(dirPath, recursive);
}
catch (e) {
console.warn(`rm directory failed: path: ${dirPath} message: ${e.message}`);
return new Error(e.message);
}
},
exists(filePath, onComplete) {
try {
fs.accessSync(filePath);
onComplete && onComplete(true);
} catch (error) {
onComplete && onComplete(false);
}
},
unzip(zipFilePath, targetPath, onComplete) {
fs.unzip({
zipFilePath,
targetPath,
success() {
onComplete && onComplete(null);
},
fail(res) {
console.warn(`unzip failed: path: ${zipFilePath} message: ${res.errMsg}`);
onComplete && onComplete(new Error('unzip failed: ' + res.errMsg));
},
})
},
};
module.exports = fsUtils;

View File

@ -0,0 +1 @@
module.exports = globalThis.CC_WORKER_FS_SYNC ? require("./fs-utils-sync.js") : require("./fs-utils-async.js");

View File

@ -0,0 +1,6 @@
const { registerHandler } = require("./ipc-worker.js");
if (globalThis.CC_WORKER_ASSET_PIPELINE) {
const cacheManager = require("./cache-manager-worker.js");
registerHandler("cacheManager", cacheManager);
}

View File

@ -0,0 +1,6 @@
require("./macro.js");
const { init } = require("./ipc-worker.js");
init(() => {
require("./handlers.js");
});

View File

@ -0,0 +1,232 @@
//
// IPC 说明
//
// - 从主线程调用 Worker
//
// 注册:
// 1.在 worker 端调用 registerHandler(name, obj) 注册处理对象。
// 2.所有非函数属性会生成 `get_xxx()`、`set_xxx(v)`、`write_xxx(v)` 三个函数,
// 其中,`write_` 函数会在设置完毕后回调到主线程。
// 3.所有函数属性请确保第一个参数是 callback用于回调到主线程
// 用法是 callback(...args)。
//
// 注册好函数后,在主线程通过 worker.name.key(args | null, (args)=>{}) 调用。
// 注意在主线程调用时传入和返回的都是参数数组。
//
// - 从 Worker 调用 主线程:
//
// 注册:
// 1.在 main 端调用 registerHandler(name, obj) 注册处理对象。
// 2.所有非函数属性会生成 `get_xxx()`、`set_xxx(v)`、`write_xxx(v)` 三个函数,
// 其中,`write_` 函数会在设置完毕后回调到 Worker。
// 3.所有函数属性请确保参数是 [args, cmdId, callback],用于回调到 Worker
// 用法是 callback(cmdId, args)。
// 注意在主线程回调时传入的是参数数组。
//
// 注册好函数后,在 Worker 通过 main.name.key(...args, (...args)=>{}) 调用。
// 最后一个参数如果是函数的话则会当作 callback 处理。
//
const { CC_WORKER_SCHEDULER, CC_WORKER_DEBUG } = globalThis;
var _inited = false;
var _initCallback = null;
var _cmdId = 0;
var callbacks = {};
var _cmd = 0;
var handlers = {};
function callToMain(cmd, args) {
const id = ++_cmdId;
args.unshift(id, cmd);
if (typeof args[args.length - 1] === "function") {
const callback = args.pop();
callbacks[id] = callback;
}
if (CC_WORKER_DEBUG) {
console.log("worker send call:", args);
}
if (CC_WORKER_SCHEDULER) {
sendScheduler.send(args);
} else {
worker.postMessage(args);
}
}
function callbackToMain(id, cmd, args) {
const msg = [id, cmd, args];
if (CC_WORKER_DEBUG) {
console.log("worker send callback:", msg);
}
if (CC_WORKER_SCHEDULER) {
sendScheduler.send(msg);
} else {
worker.postMessage(msg);
}
}
function registerHandler(name, obj) {
const descs = Object.getOwnPropertyDescriptors(obj);
for (const key in descs) {
const desc = descs[key];
if (typeof desc.value === "function") {
const cmd = ++_cmd;
handlers[cmd] = {
name,
key,
func: (id, cmd, args) => {
obj[key](
(...args) => {
callbackToMain(id, cmd, args);
},
...(args ? args : []),
);
},
};
} else {
// getter/setter
let cmd = ++_cmd;
handlers[cmd] = {
name,
key: "get_" + key,
func: (id, cmd, args) => {
callbackToMain(id, cmd, [obj[key]]);
}
};
cmd = ++_cmd;
handlers[cmd] = {
name,
key: "set_" + key,
func: (id, cmd, args) => {
obj[key] = args ? args[0] : undefined;
}
};
cmd = ++_cmd;
handlers[cmd] = {
name,
key: "write_" + key,
func: (id, cmd, args) => {
obj[key] = args ? args[0] : undefined;
callbackToMain(id, cmd, null);
}
};
}
}
}
function init(callback) {
_initCallback = callback;
if (CC_WORKER_SCHEDULER) {
sendScheduler.init();
}
worker.onMessage(CC_WORKER_SCHEDULER
? msgs => {
for (let index = 0; index < msgs.length; index++) {
const msg = msgs[index];
handleMainMessage(msg);
}
}
: handleMainMessage
);
}
function _initFromWorker(id, meta) {
const [
wrappers,
CC_WORKER_FS_SYNC,
CC_WORKER_ASSET_PIPELINE,
] = meta;
for (const wrapper of wrappers) {
const { name, key, cmd } = wrapper;
if (!main[name]) {
main[name] = {};
}
main[name][key] = (...args) => {
callToMain(cmd, args);
};
}
globalThis.CC_WORKER_FS_SYNC = CC_WORKER_FS_SYNC;
globalThis.CC_WORKER_ASSET_PIPELINE = CC_WORKER_ASSET_PIPELINE;
_inited = true;
if (_initCallback) _initCallback();
const _handlers = [];
for (const cmd in handlers) {
const { name, key } = handlers[cmd];
_handlers.push({ name, cmd: Number(cmd), key });
}
callbackToMain(id, 0, _handlers);
}
function handleMainMessage(msg) {
// 格式:[id, cmd, [args]]
// 如果 cmd 是正数,则是主线程的调用
// 反之,是返回到 Worker 的 Callback
// [0, 0, meta] 为初始化调用
const id = msg[0];
const cmd = msg[1];
const args = msg[2];
if (cmd > 0) {
const handler = handlers[cmd];
if (handler) {
const { func, name, key } = handler;
if (CC_WORKER_DEBUG) {
console.log(`worker recv call (${name}.${key}):`, msg);
}
func(id, cmd, args);
} else {
console.error("worker recv unknown call:", msg);
}
} else if (cmd < 0) {
if (CC_WORKER_DEBUG) {
console.log("worker recv callback:", msg);
}
if (callbacks[id]) {
callbacks[id](msg.slice(2));
delete callbacks[id];
}
} else {
if (CC_WORKER_DEBUG) {
console.log("worker recv init:", msg);
}
_initFromWorker(id, args);
}
}
const sendScheduler = {
queue: [],
init() {
setInterval(() => {
if (this.queue.length > 0) {
worker.postMessage(this.queue);
this.queue = [];
}
}, 0);
},
send(msg) {
this.queue.push(msg);
},
};
const main = {};
module.exports = { init, registerHandler, main };

View File

@ -0,0 +1,13 @@
// 是否启用 Worker 调度模式,这会减少通信次数(必须一致)
globalThis.CC_WORKER_SCHEDULER = true;
// 是否启用 Worker 调试模式
globalThis.CC_WORKER_DEBUG = true;
// --- 以下从主线程同步值 ---
// 是否启用 Worker 使用同步版本的文件系统 API
globalThis.CC_WORKER_FS_SYNC = null;
// 是否启用 Worker 驱动资源管线(下载、缓存)
globalThis.CC_WORKER_ASSET_PIPELINE = null;

View File

@ -0,0 +1,156 @@
var EXTNAME_RE = /(\.[^\.\/\?\\]*)(\?.*)?$/;
var DIRNAME_RE = /((.*)(\/|\\|\\\\))?(.*?\..*$)?/;
var NORMALIZE_RE = /[^\.\/]+\/\.\.\//;
/**
* !#en The module provides utilities for working with file and directory paths
* !#zh 用于处理文件与目录的路径的模块
* @class path
* @static
*/
var path = /** @lends cc.path# */{
/**
* !#en Join strings to be a path.
* !#zh 拼接字符串为 Path
* @method join
* @example {@link cocos2d/core/utils/CCPath/join.js}
* @returns {String}
*/
join: function () {
var l = arguments.length;
var result = "";
for (var i = 0; i < l; i++) {
result = (result + (result === "" ? "" : "/") + arguments[i]).replace(/(\/|\\\\)$/, "");
}
return result;
},
/**
* !#en Get the ext name of a path including '.', like '.png'.
* !#zh 返回 Path 的扩展名包括 '.'例如 '.png'
* @method extname
* @example {@link cocos2d/core/utils/CCPath/extname.js}
* @param {String} pathStr
* @returns {*}
*/
extname: function (pathStr) {
var temp = EXTNAME_RE.exec(pathStr);
return temp ? temp[1] : '';
},
/**
* !#en Get the main name of a file name
* !#zh 获取文件名的主名称
* @method mainFileName
* @param {String} fileName
* @returns {String}
* @deprecated
*/
mainFileName: function (fileName) {
if (fileName) {
var idx = fileName.lastIndexOf(".");
if (idx !== -1)
return fileName.substring(0, idx);
}
return fileName;
},
/**
* !#en Get the file name of a file path.
* !#zh 获取文件路径的文件名
* @method basename
* @example {@link cocos2d/core/utils/CCPath/basename.js}
* @param {String} pathStr
* @param {String} [extname]
* @returns {*}
*/
basename: function (pathStr, extname) {
var index = pathStr.indexOf("?");
if (index > 0) pathStr = pathStr.substring(0, index);
var reg = /(\/|\\)([^\/\\]+)$/g;
var result = reg.exec(pathStr.replace(/(\/|\\)$/, ""));
if (!result) return pathStr;
var baseName = result[2];
if (extname && pathStr.substring(pathStr.length - extname.length).toLowerCase() === extname.toLowerCase())
return baseName.substring(0, baseName.length - extname.length);
return baseName;
},
/**
* !#en Get dirname of a file path.
* !#zh 获取文件路径的目录名
* @method dirname
* @example {@link cocos2d/core/utils/CCPath/dirname.js}
* @param {String} pathStr
* @returns {*}
*/
dirname: function (pathStr) {
var temp = DIRNAME_RE.exec(pathStr);
return temp ? temp[2] : '';
},
/**
* !#en Change extname of a file path.
* !#zh 更改文件路径的扩展名
* @method changeExtname
* @example {@link cocos2d/core/utils/CCPath/changeExtname.js}
* @param {String} pathStr
* @param {String} [extname]
* @returns {String}
*/
changeExtname: function (pathStr, extname) {
extname = extname || "";
var index = pathStr.indexOf("?");
var tempStr = "";
if (index > 0) {
tempStr = pathStr.substring(index);
pathStr = pathStr.substring(0, index);
}
index = pathStr.lastIndexOf(".");
if (index < 0) return pathStr + extname + tempStr;
return pathStr.substring(0, index) + extname + tempStr;
},
/**
* !#en Change file name of a file path.
* !#zh 更改文件路径的文件名
* @example {@link cocos2d/core/utils/CCPath/changeBasename.js}
* @param {String} pathStr
* @param {String} basename
* @param {Boolean} [isSameExt]
* @returns {String}
*/
changeBasename: function (pathStr, basename, isSameExt) {
if (basename.indexOf(".") === 0) return this.changeExtname(pathStr, basename);
var index = pathStr.indexOf("?");
var tempStr = "";
var ext = isSameExt ? this.extname(pathStr) : "";
if (index > 0) {
tempStr = pathStr.substring(index);
pathStr = pathStr.substring(0, index);
}
index = pathStr.lastIndexOf("/");
index = index <= 0 ? 0 : index + 1;
return pathStr.substring(0, index) + basename + ext + tempStr;
},
//todo make public after verification
_normalize: function (url) {
var oldUrl = url = String(url);
//removing all ../
do {
oldUrl = url;
url = url.replace(NORMALIZE_RE, "");
} while (oldUrl.length !== url.length);
return url;
},
// The platform-specific file separator. '\\' or '/'.
sep: '/',
// @param {string} path
stripSep(path) {
return path.replace(/[\/\\]$/, '');
}
};
module.exports = path;