refactor: reorganize package structure and decouple framework packages (#338)

* refactor: reorganize package structure and decouple framework packages

## Package Structure Reorganization
- Reorganized 55 packages into categorized subdirectories:
  - packages/framework/ - Generic framework (Laya/Cocos compatible)
  - packages/engine/ - ESEngine core modules
  - packages/rendering/ - Rendering modules (WASM dependent)
  - packages/physics/ - Physics modules
  - packages/streaming/ - World streaming
  - packages/network-ext/ - Network extensions
  - packages/editor/ - Editor framework and plugins
  - packages/rust/ - Rust WASM engine
  - packages/tools/ - Build tools and SDK

## Framework Package Decoupling
- Decoupled behavior-tree and blueprint packages from ESEngine dependencies
- Created abstracted interfaces (IBTAssetManager, IBehaviorTreeAssetContent)
- ESEngine-specific code moved to esengine/ subpath exports
- Framework packages now usable with Cocos/Laya without ESEngine

## CI Configuration
- Updated CI to only type-check and lint framework packages
- Added type-check:framework and lint:framework scripts

## Breaking Changes
- Package import paths changed due to directory reorganization
- ESEngine integrations now use subpath imports (e.g., '@esengine/behavior-tree/esengine')

* fix: update es-engine file path after directory reorganization

* docs: update README to focus on framework over engine

* ci: only build framework packages, remove Rust/WASM dependencies

* fix: remove esengine subpath from behavior-tree and blueprint builds

ESEngine integration code will only be available in full engine builds.
Framework packages are now purely engine-agnostic.

* fix: move network-protocols to framework, build both in CI

* fix: update workflow paths from packages/core to packages/framework/core

* fix: exclude esengine folder from type-check in behavior-tree and blueprint

* fix: update network tsconfig references to new paths

* fix: add test:ci:framework to only test framework packages in CI

* fix: only build core and math npm packages in CI

* fix: exclude test files from CodeQL and fix string escaping security issue
This commit is contained in:
YHH
2025-12-26 14:50:35 +08:00
committed by GitHub
parent a84ff902e4
commit 155411e743
1936 changed files with 4147 additions and 11578 deletions

View File

@@ -0,0 +1,41 @@
/**
* Asset System Editor
* 资产系统编辑器模块
*
* Editor-side asset management:
* - Meta files (.meta) management
* - Asset packing and bundling
* - Import settings
*
* 编辑器端资产管理:
* - 元数据文件 (.meta) 管理
* - 资产打包和捆绑
* - 导入设置
*/
// Meta file management | 元数据文件管理
export {
AssetMetaManager,
type IAssetMeta,
type IImportSettings,
type ISpriteSettings,
type IMetaFileSystem,
getMetaFilePath,
inferAssetType,
getDefaultImportSettings,
createAssetMeta,
serializeAssetMeta,
parseAssetMeta
} from './meta/AssetMetaFile';
// Re-export utilities from asset-system | 从 asset-system 重导出工具函数
export { generateGUID, isValidGUID } from '@esengine/asset-system';
// Asset packing
export {
AssetPacker,
collectSceneAssets,
type IPackingResult,
type IPackedBundle,
type IAssetFileReader
} from './packing/AssetPacker';

View File

@@ -0,0 +1,466 @@
/**
* Asset Meta File (.meta) Management
* 资产元数据文件 (.meta) 管理
*
* Each asset file has a companion .meta file that stores:
* - GUID: Persistent unique identifier
* - Import settings: How to process the asset
* - Labels: User-defined tags
*
* 每个资产文件都有一个配套的 .meta 文件,存储:
* - GUID持久化唯一标识符
* - 导入设置:如何处理资产
* - 标签:用户定义的标签
*/
import {
AssetGUID,
AssetType,
generateGUID
} from '@esengine/asset-system';
/**
* Meta file content structure
* 元数据文件内容结构
*/
export interface IAssetMeta {
/** Persistent unique identifier | 持久化唯一标识符 */
guid: AssetGUID;
/** Asset type | 资产类型 */
type: AssetType;
/**
* Explicit loader type override
* 显式指定的加载器类型覆盖
*
* When set, this type will be used instead of extension-based detection.
* Useful when file extension doesn't match the actual content type.
*
* 设置后,将使用此类型而非基于扩展名的检测。
* 适用于文件扩展名与实际内容类型不匹配的情况。
*/
loaderType?: string;
/** Import settings | 导入设置 */
importSettings?: IImportSettings;
/** User-defined labels | 用户定义的标签 */
labels?: string[];
/** Meta file version | 元数据文件版本 */
version: number;
/** Last modified timestamp | 最后修改时间戳 */
lastModified?: number;
}
/**
* Sprite settings for textures
* 纹理的 Sprite 设置
*/
export interface ISpriteSettings {
/**
* Nine-patch slice border [top, right, bottom, left]
* 九宫格切片边距
*
* Defines the non-stretchable borders for nine-patch rendering.
* 定义九宫格渲染时不可拉伸的边框区域。
*/
sliceBorder?: [number, number, number, number];
/**
* Sprite pivot point (0-1 normalized)
* Sprite 锚点0-1 归一化)
*
* Default is [0.5, 0.5] (center)
* 默认为 [0.5, 0.5](中心)
*/
pivot?: [number, number];
/**
* Pixels per unit for world-space rendering
* 世界空间渲染的像素单位比
*/
pixelsPerUnit?: number;
}
/**
* Import settings for different asset types
* 不同资产类型的导入设置
*/
export interface IImportSettings {
// Texture settings | 纹理设置
maxSize?: number;
compression?: 'none' | 'dxt' | 'etc2' | 'astc' | 'webp';
generateMipmaps?: boolean;
filterMode?: 'point' | 'bilinear' | 'trilinear';
wrapMode?: 'clamp' | 'repeat' | 'mirror';
premultiplyAlpha?: boolean;
// Sprite settings | Sprite 设置
spriteSettings?: ISpriteSettings;
// Audio settings | 音频设置
audioFormat?: 'mp3' | 'ogg' | 'wav';
sampleRate?: number;
channels?: 1 | 2;
normalize?: boolean;
// General settings | 通用设置
[key: string]: unknown;
}
/**
* Get meta file path for an asset
* 获取资产的元数据文件路径
*/
export function getMetaFilePath(assetPath: string): string {
return `${assetPath}.meta`;
}
/**
* Infer asset type from file extension
* 根据文件扩展名推断资产类型
*/
export function inferAssetType(path: string): AssetType {
const ext = path.split('.').pop()?.toLowerCase() || '';
const typeMap: Record<string, AssetType> = {
// Textures
png: 'texture',
jpg: 'texture',
jpeg: 'texture',
gif: 'texture',
webp: 'texture',
bmp: 'texture',
svg: 'texture',
// Audio
mp3: 'audio',
wav: 'audio',
ogg: 'audio',
m4a: 'audio',
flac: 'audio',
// Data
json: 'json',
txt: 'text',
xml: 'text',
csv: 'text',
// Scenes and prefabs
ecs: 'scene',
prefab: 'prefab',
// Fonts
ttf: 'font',
otf: 'font',
woff: 'font',
woff2: 'font',
// Shaders
glsl: 'shader',
vert: 'shader',
frag: 'shader',
// Custom types (plugins)
tilemap: 'tilemap',
tileset: 'tileset',
btree: 'behavior-tree',
bp: 'blueprint',
mat: 'material',
particle: 'particle',
// FairyGUI
fui: 'fui'
};
return typeMap[ext] || 'binary';
}
/**
* Get default import settings for asset type
* 获取资产类型的默认导入设置
*/
export function getDefaultImportSettings(type: AssetType): IImportSettings {
switch (type) {
case 'texture':
return {
maxSize: 2048,
compression: 'none',
generateMipmaps: false,
filterMode: 'bilinear',
wrapMode: 'clamp',
premultiplyAlpha: false
};
case 'audio':
return {
audioFormat: 'mp3',
sampleRate: 44100,
channels: 2,
normalize: false
};
default:
return {};
}
}
/**
* Create a new meta file content
* 创建新的元数据文件内容
*/
export function createAssetMeta(assetPath: string, overrides?: Partial<IAssetMeta>): IAssetMeta {
const type = overrides?.type || inferAssetType(assetPath);
return {
guid: overrides?.guid || generateGUID(),
type,
importSettings: overrides?.importSettings || getDefaultImportSettings(type),
labels: overrides?.labels || [],
version: 1,
lastModified: Date.now()
};
}
/**
* Serialize meta to JSON string
* 将元数据序列化为 JSON 字符串
*/
export function serializeAssetMeta(meta: IAssetMeta): string {
return JSON.stringify(meta, null, 2);
}
/**
* Parse meta from JSON string
* 从 JSON 字符串解析元数据
*/
export function parseAssetMeta(json: string): IAssetMeta {
const meta = JSON.parse(json) as IAssetMeta;
// Validate required fields
if (!meta.guid || typeof meta.guid !== 'string') {
throw new Error('Invalid meta file: missing or invalid guid');
}
if (!meta.type || typeof meta.type !== 'string') {
throw new Error('Invalid meta file: missing or invalid type');
}
// Set defaults for optional fields
meta.version = meta.version || 1;
meta.labels = meta.labels || [];
meta.importSettings = meta.importSettings || {};
return meta;
}
/**
* Asset Meta File Manager
* 资产元数据文件管理器
*
* Handles reading/writing .meta files through a file system interface.
*/
export class AssetMetaManager {
private _cache = new Map<string, IAssetMeta>();
private _guidToPath = new Map<AssetGUID, string>();
/**
* File system interface for reading/writing files
* 用于读写文件的文件系统接口
*/
private _fs: IMetaFileSystem | null = null;
/**
* Set file system interface
* 设置文件系统接口
*/
setFileSystem(fs: IMetaFileSystem): void {
this._fs = fs;
}
/**
* Get or create meta for an asset
* 获取或创建资产的元数据
*/
async getOrCreateMeta(assetPath: string): Promise<IAssetMeta> {
// Check cache first
const cached = this._cache.get(assetPath);
if (cached) {
return cached;
}
const metaPath = getMetaFilePath(assetPath);
// Try to read existing meta file
if (this._fs) {
try {
if (await this._fs.exists(metaPath)) {
const content = await this._fs.readText(metaPath);
const meta = parseAssetMeta(content);
this._cache.set(assetPath, meta);
this._guidToPath.set(meta.guid, assetPath);
return meta;
}
} catch (e) {
console.warn(`Failed to read meta file: ${metaPath}`, e);
}
}
// Create new meta
const meta = createAssetMeta(assetPath);
this._cache.set(assetPath, meta);
this._guidToPath.set(meta.guid, assetPath);
// Save to file system
if (this._fs) {
try {
await this._fs.writeText(metaPath, serializeAssetMeta(meta));
} catch (e) {
console.warn(`Failed to write meta file: ${metaPath}`, e);
}
}
return meta;
}
/**
* Get meta by GUID
* 根据 GUID 获取元数据
*/
getMetaByGUID(guid: AssetGUID): IAssetMeta | undefined {
const path = this._guidToPath.get(guid);
return path ? this._cache.get(path) : undefined;
}
/**
* Get asset path by GUID
* 根据 GUID 获取资产路径
*/
getPathByGUID(guid: AssetGUID): string | undefined {
return this._guidToPath.get(guid);
}
/**
* Get GUID by asset path
* 根据资产路径获取 GUID
*/
async getGUIDByPath(assetPath: string): Promise<AssetGUID> {
const meta = await this.getOrCreateMeta(assetPath);
return meta.guid;
}
/**
* Update meta and save
* 更新元数据并保存
*/
async updateMeta(assetPath: string, updates: Partial<IAssetMeta>): Promise<void> {
const meta = await this.getOrCreateMeta(assetPath);
// Apply updates
Object.assign(meta, updates);
meta.lastModified = Date.now();
meta.version++;
// Update cache
this._cache.set(assetPath, meta);
// Handle GUID change (rare, but possible)
if (updates.guid && updates.guid !== meta.guid) {
this._guidToPath.delete(meta.guid);
this._guidToPath.set(updates.guid, assetPath);
}
// Save to file system
if (this._fs) {
const metaPath = getMetaFilePath(assetPath);
await this._fs.writeText(metaPath, serializeAssetMeta(meta));
}
}
/**
* Handle asset rename
* 处理资产重命名
*/
async handleAssetRename(oldPath: string, newPath: string): Promise<void> {
const meta = this._cache.get(oldPath);
if (meta) {
// Update cache with new path
this._cache.delete(oldPath);
this._cache.set(newPath, meta);
this._guidToPath.set(meta.guid, newPath);
// Move meta file
if (this._fs) {
const oldMetaPath = getMetaFilePath(oldPath);
const newMetaPath = getMetaFilePath(newPath);
if (await this._fs.exists(oldMetaPath)) {
const content = await this._fs.readText(oldMetaPath);
await this._fs.writeText(newMetaPath, content);
await this._fs.delete(oldMetaPath);
}
}
}
}
/**
* Handle asset delete
* 处理资产删除
*/
async handleAssetDelete(assetPath: string): Promise<void> {
const meta = this._cache.get(assetPath);
if (meta) {
this._cache.delete(assetPath);
this._guidToPath.delete(meta.guid);
// Delete meta file
if (this._fs) {
const metaPath = getMetaFilePath(assetPath);
if (await this._fs.exists(metaPath)) {
await this._fs.delete(metaPath);
}
}
}
}
/**
* Invalidate cache for a specific asset path
* 使特定资产路径的缓存失效
*
* Call this when a .meta file is modified externally.
* 当 .meta 文件被外部修改时调用此方法。
*/
invalidateCache(assetPath: string): void {
const meta = this._cache.get(assetPath);
if (meta) {
this._guidToPath.delete(meta.guid);
this._cache.delete(assetPath);
}
}
/**
* Clear cache
* 清除缓存
*/
clear(): void {
this._cache.clear();
this._guidToPath.clear();
}
/**
* Get all cached metas
* 获取所有缓存的元数据
*/
getAllMetas(): Map<string, IAssetMeta> {
return new Map(this._cache);
}
}
/**
* File system interface for meta file operations
* 元数据文件操作的文件系统接口
*/
export interface IMetaFileSystem {
exists(path: string): Promise<boolean>;
readText(path: string): Promise<string>;
writeText(path: string, content: string): Promise<void>;
delete(path: string): Promise<void>;
}

View File

@@ -0,0 +1,390 @@
/**
* Asset Packer
* 资产打包器
*
* Collects and packs assets into bundles for runtime loading.
* 收集并将资产打包成运行时加载的包。
*/
import {
AssetGUID,
AssetType,
IBundleManifest,
IBundleAssetInfo,
IRuntimeCatalog,
IRuntimeBundleInfo,
IRuntimeAssetLocation,
IAssetToPack,
IBundlePackOptions,
hashBuffer
} from '@esengine/asset-system';
import { IAssetMeta } from '../meta/AssetMetaFile';
/**
* Packing result
* 打包结果
*/
export interface IPackingResult {
/** Generated bundles | 生成的包 */
bundles: IPackedBundle[];
/** Runtime catalog | 运行时目录 */
catalog: IRuntimeCatalog;
/** Total size in bytes | 总大小 */
totalSize: number;
/** Number of assets packed | 打包的资产数量 */
assetCount: number;
/** Packing duration in ms | 打包耗时 */
duration: number;
}
/**
* Packed bundle
* 已打包的包
*/
export interface IPackedBundle {
/** Bundle name | 包名称 */
name: string;
/** Bundle data | 包数据 */
data: ArrayBuffer;
/** Bundle manifest | 包清单 */
manifest: IBundleManifest;
}
/**
* Asset file reader interface
* 资产文件读取器接口
*/
export interface IAssetFileReader {
readBinary(path: string): Promise<ArrayBuffer>;
readText(path: string): Promise<string>;
exists(path: string): Promise<boolean>;
}
/**
* Asset Packer
* 资产打包器
*/
export class AssetPacker {
private _fileReader: IAssetFileReader | null = null;
private _assets: IAssetToPack[] = [];
private _metas = new Map<AssetGUID, IAssetMeta>();
/**
* Set file reader for loading asset data
* 设置用于加载资产数据的文件读取器
*/
setFileReader(reader: IAssetFileReader): void {
this._fileReader = reader;
}
/**
* Add asset to pack
* 添加要打包的资产
*/
addAsset(asset: IAssetToPack, meta?: IAssetMeta): void {
this._assets.push(asset);
if (meta) {
this._metas.set(asset.guid, meta);
}
}
/**
* Add multiple assets
* 添加多个资产
*/
addAssets(assets: IAssetToPack[]): void {
for (const asset of assets) {
this.addAsset(asset);
}
}
/**
* Clear all added assets
* 清除所有已添加的资产
*/
clear(): void {
this._assets = [];
this._metas.clear();
}
/**
* Pack assets into bundles
* 将资产打包成包
*/
async pack(options: IBundlePackOptions = { name: 'main' }): Promise<IPackingResult> {
const startTime = Date.now();
// Group assets for bundling
const groups = this._groupAssets(options);
// Pack each group into a bundle
const bundles: IPackedBundle[] = [];
const catalogAssets: Record<AssetGUID, IRuntimeAssetLocation> = {};
const catalogBundles: Record<string, IRuntimeBundleInfo> = {};
for (const [bundleName, assets] of groups) {
const packed = await this._packBundle(bundleName, assets, options);
bundles.push(packed);
// Add to catalog
catalogBundles[bundleName] = {
url: `assets/${bundleName}.bundle`,
size: packed.data.byteLength,
hash: await hashBuffer(packed.data),
// 预加载核心资产包(可通过配置扩展) | Preload core bundles (extensible via config)
preload: options.preloadBundles?.includes(bundleName) ??
(bundleName === 'core' || bundleName === 'main')
};
// Add asset locations
for (const assetInfo of packed.manifest.assets) {
catalogAssets[assetInfo.guid] = {
bundle: bundleName,
offset: assetInfo.offset,
size: assetInfo.size,
type: assetInfo.type,
name: assetInfo.name
};
}
}
// Create catalog
const catalog: IRuntimeCatalog = {
version: '1.0',
createdAt: Date.now(),
bundles: catalogBundles,
assets: catalogAssets
};
const totalSize = bundles.reduce((sum, b) => sum + b.data.byteLength, 0);
return {
bundles,
catalog,
totalSize,
assetCount: this._assets.length,
duration: Date.now() - startTime
};
}
/**
* Pack assets by type (textures.bundle, audio.bundle, etc.)
* 按类型打包资产
*/
async packByType(): Promise<IPackingResult> {
return this.pack({
name: 'main',
groupByType: true
});
}
/**
* Group assets for bundling
* 分组资产以便打包
*/
private _groupAssets(options: IBundlePackOptions): Map<string, IAssetToPack[]> {
const groups = new Map<string, IAssetToPack[]>();
if (options.groupByType) {
// Group by asset type
for (const asset of this._assets) {
const bundleName = this._getBundleNameForType(asset.type);
const group = groups.get(bundleName) || [];
group.push(asset);
groups.set(bundleName, group);
}
} else {
// Single bundle
groups.set(options.name, [...this._assets]);
}
// Handle max size splitting
if (options.maxSize) {
const splitGroups = new Map<string, IAssetToPack[]>();
for (const [name, assets] of groups) {
let currentSize = 0;
let partIndex = 0;
let currentGroup: IAssetToPack[] = [];
for (const asset of assets) {
const assetSize = asset.data?.byteLength || 0;
if (currentSize + assetSize > options.maxSize && currentGroup.length > 0) {
splitGroups.set(`${name}_${partIndex}`, currentGroup);
partIndex++;
currentGroup = [];
currentSize = 0;
}
currentGroup.push(asset);
currentSize += assetSize;
}
if (currentGroup.length > 0) {
const finalName = partIndex > 0 ? `${name}_${partIndex}` : name;
splitGroups.set(finalName, currentGroup);
}
}
return splitGroups;
}
return groups;
}
/**
* Get bundle name for asset type
* 获取资产类型的包名称
*/
private _getBundleNameForType(type: AssetType): string {
const typeGroups: Record<string, string[]> = {
textures: ['texture'],
audio: ['audio'],
data: ['json', 'text', 'binary', 'scene', 'prefab'],
fonts: ['font'],
shaders: ['shader', 'material'],
tilemaps: ['tilemap', 'tileset'],
scripts: ['behavior-tree', 'blueprint']
};
for (const [bundleName, types] of Object.entries(typeGroups)) {
if (types.includes(type)) {
return bundleName;
}
}
return 'misc';
}
/**
* Pack a single bundle
* 打包单个包
*/
private async _packBundle(
name: string,
assets: IAssetToPack[],
_options: IBundlePackOptions
): Promise<IPackedBundle> {
const assetInfos: IBundleAssetInfo[] = [];
const dataChunks: ArrayBuffer[] = [];
let currentOffset = 0;
// Load and pack each asset
for (const asset of assets) {
let data = asset.data;
// Load data if not provided
if (!data && this._fileReader) {
try {
data = await this._fileReader.readBinary(asset.path);
} catch (e) {
console.warn(`[AssetPacker] Failed to load asset: ${asset.path}`, e);
continue;
}
}
if (!data) {
console.warn(`[AssetPacker] No data for asset: ${asset.guid}`);
continue;
}
// Align to 4 bytes
const padding = (4 - (data.byteLength % 4)) % 4;
const paddedSize = data.byteLength + padding;
assetInfos.push({
guid: asset.guid,
name: asset.name,
type: asset.type,
offset: currentOffset,
size: data.byteLength
});
// Add data with padding
dataChunks.push(data);
if (padding > 0) {
dataChunks.push(new ArrayBuffer(padding));
}
currentOffset += paddedSize;
}
// Combine all data
const totalSize = dataChunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
const bundleData = new Uint8Array(totalSize);
let offset = 0;
for (const chunk of dataChunks) {
bundleData.set(new Uint8Array(chunk), offset);
offset += chunk.byteLength;
}
// Create manifest
const manifest: IBundleManifest = {
name,
version: '1.0',
hash: await hashBuffer(bundleData.buffer),
compression: 'none',
size: bundleData.byteLength,
assets: assetInfos,
dependencies: [],
createdAt: Date.now()
};
return {
name,
data: bundleData.buffer,
manifest
};
}
}
/**
* Collect assets referenced by a scene
* 收集场景引用的资产
*/
export async function collectSceneAssets(
sceneData: unknown,
_metaManager: { getPathByGUID: (guid: AssetGUID) => string | undefined }
): Promise<AssetGUID[]> {
const guids = new Set<AssetGUID>();
// Recursively find all GUID references
function findGUIDs(obj: unknown): void {
if (!obj || typeof obj !== 'object') return;
if (Array.isArray(obj)) {
for (const item of obj) {
findGUIDs(item);
}
return;
}
const record = obj as Record<string, unknown>;
// Check for GUID fields
for (const [key, value] of Object.entries(record)) {
if (key.endsWith('Guid') || key.endsWith('GUID') || key === 'guid') {
if (typeof value === 'string' && isValidGUID(value)) {
guids.add(value);
}
} else if (typeof value === 'object') {
findGUIDs(value);
}
}
}
findGUIDs(sceneData);
return Array.from(guids);
}
/**
* Validate GUID format
* 验证 GUID 格式
*/
function isValidGUID(guid: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(guid);
}