diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f34de607..80f4e4a11 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -67,6 +67,8 @@ jobs: else parallel --tty -j+0 yarn ::: lint:ci build test fi + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - name: Publish if: ${{ github.event_name == 'push' }} run: node -r @hydrooj/register build/publish.ts diff --git a/README.md b/README.md index edf3cc724..df7a93513 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,38 @@ Hydro 用户群：1085853538

更新日志（点击展开）

 +### Hydro 4.13.0 / UI 4.51.0 +- ui: 添加 sentry +- core&ui: 支持显示当前编译器版本 +- core: 数据库索引优化 +- core: 文件复制使用软链接 +- a11y: 添加性能测试工具 +- utils: 支持识别大写文件扩展名 +- ui: 优化下拉菜单样式 +- core: 优化页面标题 +- register: 支持从文件读取原 sourcemap +- ui: 修复站内消息推送 +- ui: ranking 页面添加说明 +- ui: 添加生成测试数据提示 +- framework: 从 core 解离 +- core: loader: 支持多 profile 切换 +- ui: 修复比赛计分板选手组显示 +- core: 重设比赛分数时自动重算分数 + +本版本同时引入下述插件 API 修改： + +- 移除了 app/load/${category} 钩子 +- 移除了 ctx.app (请使用 ctx.root) +- 移除了 ctx.options (请使用 ctx.root.config) +- serializer: 移除 showDisplayName 参数 +- 移除了 loader.addScript, loader.addon 函数 +- 移除了 Hydro.module.render (请使用 ctx.server.registerRenderer) +- 移除了 lib/paginate 和 lib/rank (请使用 db.paginate 和 db.ranked) + ### Hydro 4.12.3 / UI 4.50.2 - core: 将 API 模块移入 service - core: 比赛时提交被 hack 不触发整体重测 -- core: 将添加 pid match 的逻辑移入搜索中 +- core: 将添加 pid match 的逻辑移入搜索中 - ui: 修正 cssFilter - judge: builtin: 维护 callback 顺序 - core: 优化邮件地址处理 @@ -634,7 +662,8 @@ Hydro 用户群：1085853538 ## 开源许可 -本项目中的 examples/ install/ packages/ui-default/ 下的内容仅采用 AGPL-3.0 进行授权。 +本项目中 framework/ examples/ install/ 下的内容采用 MIT 协议授权，您可自由使用。 +本项目中 packages/ui-default/ 下的内容仅采用 AGPL-3.0 进行授权。 项目其余部分使用双重许可： 1. 您可以在遵守 AGPL-3.0 许可证和下述附加条款章节的前提下免费使用这些代码： diff --git a/build/prepare.js b/build/prepare.js index 6e00c7c00..52cd75024 100644 --- a/build/prepare.js +++ b/build/prepare.js @@ -72,7 +72,7 @@ for (const name of ['plugins', 'modules']) { const modules = [ 'packages/hydrooj', - ...['packages', 'framework', 'modules'].flatMap((i) => fs.readdirSync(path.resolve(process.cwd(), i)).map((j) => `${i}/${j}`)), + ...['packages', 'framework'].flatMap((i) => fs.readdirSync(path.resolve(process.cwd(), i)).map((j) => `${i}/${j}`)), ].filter((i) => !i.includes('/.') && !i.includes('ui-default')).filter((i) => fs.statSync(path.resolve(process.cwd(), i)).isDirectory()); const UIConfig = { @@ -80,22 +80,17 @@ const UIConfig = { 'packages/ui-default/public', '**/node_modules', ], - include: ['ts', 'tsx'] - .flatMap((ext) => ['plugins', 'modules'] + include: ['ts', 'tsx', 'vue', 'json'] + .flatMap((ext) => ['plugins'] .flatMap((name) => [`${name}/**/public/**/*.${ext}`, `${name}/**/frontend/**/*.${ext}`]) .concat(`packages/ui-default/**/*.${ext}`)), compilerOptions: { - experimentalDecorators: true, - esModuleInterop: true, - resolveJsonModule: true, - jsx: 'react', - module: 'commonjs', + ...compilerOptionsBase, + module: 'ESNext', skipLibCheck: true, allowSyntheticDefaultImports: true, - target: 'es2020', baseUrl: '.', outDir: path.join(baseOutDir, 'ui'), - moduleResolution: 'node', paths: { 'vj/*': [ './packages/ui-default/*', @@ -120,17 +115,7 @@ const pluginsConfig = { ], exclude, compilerOptions: { - target: 'es2020', - module: 'commonjs', - esModuleInterop: true, - moduleResolution: 'node', - jsx: 'react', - sourceMap: false, - composite: true, - strictBindCallApply: true, - resolveJsonModule: true, - experimentalDecorators: true, - incremental: true, + ...compilerOptionsBase, rootDir: '.', baseUrl: '.', outDir: path.join(baseOutDir, 'plugins'), diff --git a/changelog b/changelog deleted file mode 100644 index 7947fd9a1..000000000 --- a/changelog +++ /dev/null @@ -1,6 +0,0 @@ -移除了 app/listen, app/load/${category} 钩子 -移除了 ctx.app (请使用 ctx.root) -移除了 ctx.options (请使用 ctx.root.config) -serializer: 移除 showDisplayName 参数 -移除了 loader.addScript, loader.addon 函数 -移除了 Hydro.module.render (请使用 ctx.server.registerRenderer) \ No newline at end of file diff --git a/packages/eslint-config/base.yaml b/framework/eslint-config/base.yaml similarity index 100% rename from packages/eslint-config/base.yaml rename to framework/eslint-config/base.yaml diff --git a/packages/eslint-config/package.json b/framework/eslint-config/package.json similarity index 50% rename from packages/eslint-config/package.json rename to framework/eslint-config/package.json index a5a0e2185..23df59fd1 100644 --- a/packages/eslint-config/package.json +++ b/framework/eslint-config/package.json @@ -1,20 +1,20 @@ { "name": "@hydrooj/eslint-config", - "version": "1.0.10", + "version": "1.0.11", "license": "MIT", "main": "react.yaml", "dependencies": { - "@stylistic/eslint-plugin": "^2.1.0", - "@typescript-eslint/eslint-plugin": "^7.12.0", - "@typescript-eslint/parser": "^7.12.0", + "@stylistic/eslint-plugin": "^2.3.0", + "@typescript-eslint/eslint-plugin": "^7.15.0", + "@typescript-eslint/parser": "^7.15.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-airbnb-typescript": "^18.0.0", - "eslint-plugin-github": "^4.10.2", + "eslint-plugin-github": "^5.0.1", "eslint-plugin-import": "2.29.1", - "eslint-plugin-jsx-a11y": "^6.8.0", - "eslint-plugin-react": "^7.34.2", + "eslint-plugin-jsx-a11y": "^6.9.0", + "eslint-plugin-react": "^7.34.3", "eslint-plugin-react-hooks": "^4.6.2", - "eslint-plugin-simple-import-sort": "^12.1.0" + "eslint-plugin-simple-import-sort": "^12.1.1" }, "peerDependencies": { "eslint": ">= 8" diff --git a/packages/eslint-config/react.yaml b/framework/eslint-config/react.yaml similarity index 100% rename from packages/eslint-config/react.yaml rename to framework/eslint-config/react.yaml diff --git a/framework/framework/package.json b/framework/framework/package.json index 12ed76220..80e7f873b 100644 --- a/framework/framework/package.json +++ b/framework/framework/package.json @@ -1,7 +1,8 @@ { "name": "@hydrooj/framework", - "version": "0.0.2", + "version": "0.0.4", "main": "index.ts", + "license": "MIT", "peerDependencies": { "cordis": "*" }, @@ -15,18 +16,18 @@ "koa-router": "^12.0.1", "lodash": "^4.17.21", "moment-timezone": "^0.5.45", - "path-to-regexp": "^6.2.2", + "path-to-regexp": "^7.0.0", "sanitize-filename": "^1.6.3", "saslprep": "^1.0.3", "shorty.js": "^1.0.1", - "ws": "^8.17.0" + "ws": "^8.18.0" }, "devDependencies": { "@types/fs-extra": "^11.0.4", "@types/koa": "^2.15.0", "@types/koa-compress": "^4.0.6", "@types/koa-router": "^7.4.8", - "@types/lodash": "^4.17.4", + "@types/lodash": "^4.17.6", "@types/saslprep": "^1.0.3", "@types/ws": "^8.5.10" } diff --git a/framework/framework/server.ts b/framework/framework/server.ts index 09897f36b..8b5f7c630 100644 --- a/framework/framework/server.ts +++ b/framework/framework/server.ts @@ -8,7 +8,7 @@ import Koa from 'koa'; import Body from 'koa-body'; import Compress from 'koa-compress'; import { Shorty } from 'shorty.js'; -import WebSocket from 'ws'; +import { WebSocketServer } from 'ws'; import { Counter, errorMessage, isClass, Logger, parseMemoryMB, } from '@hydrooj/utils/lib/utils'; @@ -21,6 +21,8 @@ import { import { Router } from './router'; import serializer from './serializer'; +export { WebSocket, WebSocketServer } from 'ws'; + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent export function encodeRFC5987ValueChars(str: string) { return ( @@ -91,7 +93,7 @@ export const koa = new Koa({ }); export const router = new Router(); export const httpServer = http.createServer(koa.callback()); -export const wsServer = new WebSocket.Server({ server: httpServer }); +export const wsServer = new WebSocketServer({ server: httpServer }); koa.on('error', (error) => { if (error.code !== 'EPIPE' && error.code !== 'ECONNRESET' && !error.message.includes('Parse Error')) { logger.error('Koa app-level error', { error }); @@ -322,6 +324,7 @@ export class WebService extends Service { HandlerCommon = HandlerCommon; Handler = Handler; ConnectionHandler = ConnectionHandler; + handlerCtxBase = this.ctx; constructor(ctx: Context, public config: WebServiceConfig) { super(ctx, 'server', true); @@ -420,6 +423,9 @@ ${c.response.status} ${endTime - startTime}ms ${c.response.length}`); } socket.close(); }); + this.ctx.inject(['server'], (c) => { + this.handlerCtxBase = c; + }); } async listen() { @@ -438,7 +444,7 @@ ${c.response.status} ${endTime - startTime}ms ${c.response.length}`); private async handleHttp(ctx: KoaContext, HandlerClass, checker) { const { args } = ctx.HydroContext; Object.assign(args, ctx.params); - const h = new HandlerClass(ctx, this.ctx); + const h = new HandlerClass(ctx, this.handlerCtxBase); ctx.handler = h; const method = ctx.method.toLowerCase(); try { @@ -511,7 +517,7 @@ ${c.response.status} ${endTime - startTime}ms ${c.response.length}`); private async handleWS(ctx: KoaContext, HandlerClass, checker, conn, layer) { const { args } = ctx.HydroContext; - const h = new HandlerClass(ctx, this.ctx); + const h = new HandlerClass(ctx, this.handlerCtxBase); await this.ctx.parallel('connection/create', h); ctx.handler = h; h.conn = conn; diff --git a/packages/register/index.js b/framework/register/index.js similarity index 90% rename from packages/register/index.js rename to framework/register/index.js index 6d70e161e..d7ee5f518 100644 --- a/packages/register/index.js +++ b/framework/register/index.js @@ -50,6 +50,12 @@ require.extensions['.js'] = function loader(module, filename) { } try { const content = fs.readFileSync(filename, 'utf-8'); + const lastLine = content.trim().split('\n').pop(); + if (lastLine.startsWith('//# sourceMappingURL=data:application/json;base64,')) { + const info = lastLine.split('//# sourceMappingURL=data:application/json;base64,')[1]; + const payload = JSON.parse(Buffer.from(info, 'base64').toString()); + map[filename] = payload; + } return module._compile(content, filename); } catch (e) { // ESM return module._compile(transform(filename), filename); diff --git a/packages/register/package.json b/framework/register/package.json similarity index 92% rename from packages/register/package.json rename to framework/register/package.json index a8a60a18e..48fb75e48 100644 --- a/packages/register/package.json +++ b/framework/register/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/register", - "version": "1.0.1", + "version": "1.0.2", "license": "MIT", "main": "index.js", "engines": { diff --git a/package.json b/package.json index 018ce99eb..da0a7c30b 100644 --- a/package.json +++ b/package.json @@ -49,14 +49,14 @@ "eslint-import-resolver-typescript": "^3.6.1", "eslint-import-resolver-webpack": "^0.13.8", "fs-extra": "^11.2.0", - "globby": "^14.0.1", + "globby": "^14.0.2", "inspectpack": "^4.7.1", - "mongodb-memory-server": "^9.3.0", - "nyc": "^15.1.0", + "mongodb-memory-server": "^9.4.0", + "nyc": "^17.0.0", "ora": "^8.0.1", "package-json": "^10.0.0", "semver": "^7.6.2", - "simple-git": "^3.24.0", + "simple-git": "^3.25.0", "supertest": "^7.0.0", "typescript": "5.3.3", "wtfnode": "^0.9.2" diff --git a/packages/a11y/index.ts b/packages/a11y/index.ts index 4d9fc0fcf..0e582a027 100644 --- a/packages/a11y/index.ts +++ b/packages/a11y/index.ts @@ -1,8 +1,10 @@ -import { Context, UserModel } from 'hydrooj'; +import { Context, Schema, UserModel } from 'hydrooj'; +import { startPerformanceTest } from './performance-test'; export async function apply(ctx: Context) { // Auto authorize user 2 as super admin ctx.on('handler/after/UserRegisterWithCode#post', async (that) => { if (that.session.uid === 2) await UserModel.setSuperAdmin(2); }); + ctx.addScript('performance-test', 'test', Schema.object({ enable5: Schema.boolean().default(false) }), startPerformanceTest); } diff --git a/packages/a11y/package.json b/packages/a11y/package.json index afde7a87d..7d16ff5fd 100644 --- a/packages/a11y/package.json +++ b/packages/a11y/package.json @@ -1,4 +1,4 @@ { "name": "@hydrooj/a11y", - "version": "0.0.1" + "version": "0.0.2" } diff --git a/packages/a11y/performance-test.ts b/packages/a11y/performance-test.ts new file mode 100644 index 000000000..36edd86ad --- /dev/null +++ b/packages/a11y/performance-test.ts @@ -0,0 +1,207 @@ +/* eslint-disable no-await-in-loop */ +import { + ProblemModel, RecordModel, sleep, STATUS, yaml, +} from 'hydrooj'; + +// These tests are from https://loj.ac/d/425 +const head = '#include\nusing namespace std;\n'; +const TESTS = { + ' 1 ': ` +int main(){ + int a=1000000000,b=1; + while(a)b<<=1,a--; + printf("%d\\n",b);`, + + ' 2 ': ` +const int MX=50000000; +int p[MX],m[MX],pc; +int main(){ + for(int i=2;iG[j][i]+G[i][k])G[j][k]=G[j][i]+G[i][k]; + int ans=0; + for(int i=0;i +#include +const int MX=1000000; +int sed=0; +inline int rand(){return sed=(sed*sed*73+sed*233+19260817);} +int main(){ + setS; + for(int i=0;i::iterator it=S.begin();it!=S.end();it++)ans^=*it; + printf("%d\\n",ans);`, + + // skipping test 5 by default as it contains a lot of ram + ' 5 ': ` +const int MX=20000000; +int *it[MX]; +int main(){ + for(int i=0;i { + if (key === ' 5 ' && !args.enable5) return; + const id = await RecordModel.add('system', docId, 1, 'cc.cc14o2', `// TEST ${key}\n${head}${TESTS[key]}\nreturn 0;}`, true); + while ([ + STATUS.STATUS_WAITING, STATUS.STATUS_JUDGING, STATUS.STATUS_FETCHED, STATUS.STATUS_COMPILING, + ].includes((await RecordModel.get('system', id))?.status)) { + await sleep(500); + } + const result = await RecordModel.get('system', id); + if (result.status !== STATUS.STATUS_ACCEPTED && result.status !== STATUS.STATUS_WRONG_ANSWER) { + report({ message: `Test ${key} failed (${id}) ${result.status}` }); + } else { + results[key] = result.testCases.map((i) => i.time); + } + })); + const formatL = (t: number, width = 4) => Math.floor(t).toString().padEnd(width); + const formatR = (t: number, width = 4) => Math.floor(t).toString().padStart(width); + for (const key of Object.keys(TESTS)) { + const timeArr = results[key]; + if (!timeArr) continue; + const avgTime = Math.sum(...timeArr) / 20; + await report({ + message: [ + `-------Test ${key}-------`, + ` Avg: ${formatL(avgTime)} D: ${formatR(Math.sum(...timeArr.map((i) => (i - avgTime) ** 2)) / 20, 5)} `, + ` Max: ${formatL(Math.max(...timeArr))} Min: ${formatR(Math.min(...timeArr))} `, + ].join('\n'), + }); + } + return true; +} diff --git a/packages/elastic/package.json b/packages/elastic/package.json index 805e19eaa..cda59af1d 100644 --- a/packages/elastic/package.json +++ b/packages/elastic/package.json @@ -1,11 +1,11 @@ { "name": "@hydrooj/elastic-search", - "version": "1.1.4", + "version": "1.1.5", "main": "index.ts", "repository": "https://github.com/hydro-dev/Hydro", "author": "undefined ", "license": "AGPL-3.0-or-later", "dependencies": { - "@elastic/elasticsearch": "^8.13.1" + "@elastic/elasticsearch": "^8.14.0" } } diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 6dc24e72d..741bbe263 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -1,20 +1,20 @@ { "name": "@hydrooj/hydrojudge", "bin": "bin/hydrojudge.js", - "version": "3.3.13", + "version": "3.3.15", "main": "./src/index.ts", "author": "undefined ", "repository": "https://github.com/hydro-dev/Hydro.git", "dependencies": { "@hydrooj/utils": "workspace:^", - "adm-zip": "0.5.5", + "adm-zip": "0.5.14", "cac": "^6.7.14", "p-queue": "^8.0.1", "schemastery": "^3.14.6", "semver": "^7.6.2", "shell-quote": "^1.8.1", "superagent": "^9.0.2", - "ws": "^8.17.0" + "ws": "^8.18.0" }, "preferUnplugged": true, "license": "AGPL-3.0-or-later", diff --git a/packages/hydrojudge/src/compiler.ts b/packages/hydrojudge/src/compiler.ts new file mode 100644 index 000000000..bcf5050ac --- /dev/null +++ b/packages/hydrojudge/src/compiler.ts @@ -0,0 +1,18 @@ +import { LangConfig } from '@hydrooj/utils/lib/lang'; +import { runQueued } from './sandbox'; + +export async function compilerVersions(langs: Record) { + const result = {}; + for (const lang in langs) { + const version = langs[lang].version; + if (!version) continue; + // eslint-disable-next-line no-await-in-loop + const res = await runQueued(version, { + copyIn: {}, + time: 10000, + memory: 256 * 1024 * 1024, + }, 5); + result[lang] = `${res.stdout}\n${res.stderr}`.trim(); + } + return result; +} diff --git a/packages/hydrojudge/src/hosts/builtin.ts b/packages/hydrojudge/src/hosts/builtin.ts index 455132618..889ce399b 100644 --- a/packages/hydrojudge/src/hosts/builtin.ts +++ b/packages/hydrojudge/src/hosts/builtin.ts @@ -2,10 +2,13 @@ /* eslint-disable no-await-in-loop */ import path from 'path'; import { fs } from '@hydrooj/utils'; +import * as sysinfo from '@hydrooj/utils/lib/sysinfo'; import { - JudgeHandler, JudgeResultBody, ObjectId, RecordModel, SettingModel, - StorageModel, SystemModel, TaskModel, + db, JudgeHandler, JudgeResultBody, ObjectId, RecordModel, + SettingModel, StorageModel, SystemModel, TaskModel, } from 'hydrooj'; +import { langs } from 'hydrooj/src/model/setting'; +import { compilerVersions } from '../compiler'; import { getConfig } from '../config'; import { FormatError, SystemError } from '../error'; import { Context } from '../judge/interface'; @@ -82,6 +85,7 @@ export async function postInit(ctx) { c.check.addChecker('Judge', (_ctx, log, warn, error) => versionCheck(warn, error)); }); await fs.ensureDir(getConfig('tmp_dir')); + const info = await sysinfo.get(); const handle = async (t) => { const rdoc = await RecordModel.get(t.domainId, t.rid); if (!rdoc) { @@ -92,9 +96,19 @@ export async function postInit(ctx) { }; const parallelism = Math.max(getConfig('parallelism'), 2); const taskConsumer = TaskModel.consume({ type: 'judge' }, handle, true, parallelism); + function collectCompilerInfo() { + const coll = db.collection('status'); + compilerVersions(langs).then((compilers) => coll.updateOne( + { mid: info.mid, type: 'server' }, + { $set: { compilers } }, + { upsert: true }, + )).catch((e) => logger.error(e)); + } ctx.on('system/setting', () => { taskConsumer.setConcurrency(Math.max(getConfig('parallelism'), 2)); + collectCompilerInfo(); }); + collectCompilerInfo(); TaskModel.consume({ type: 'judge', priority: { $gt: -50 } }, handle); TaskModel.consume({ type: 'generate' }, handle); } diff --git a/packages/hydrojudge/src/hosts/hydro.ts b/packages/hydrojudge/src/hosts/hydro.ts index 9bcad2147..eaf5e81c4 100644 --- a/packages/hydrojudge/src/hosts/hydro.ts +++ b/packages/hydrojudge/src/hosts/hydro.ts @@ -7,6 +7,7 @@ import { fs, pipeRequest } from '@hydrooj/utils'; import { LangConfig } from '@hydrooj/utils/lib/lang'; import * as sysinfo from '@hydrooj/utils/lib/sysinfo'; import type { JudgeResultBody } from 'hydrooj'; +import { compilerVersions } from '../compiler'; import { getConfig } from '../config'; import { FormatError, SystemError } from '../error'; import { Session } from '../interface'; @@ -173,13 +174,21 @@ export default class Hydro implements Session { const content = Object.keys(config).length ? JSON.stringify({ key: 'config', ...config }) : '{"key":"ping"}'; + let compilers = {}; + let compilerVersionCallback = () => { }; this.ws.on('message', (data) => { if (data.toString() === 'ping') { this.ws.send('pong'); return; } const request = JSON.parse(data.toString()); - if (request.language) this.language = request.language; + if (request.language) { + this.language = request.language; + compilerVersions(this.language).then((res) => { + compilers = res; + compilerVersionCallback(); + }); + } if (request.task) queue.add(() => new JudgeTask(this, request.task).handle().catch((e) => log.error(e))); }); this.ws.on('close', (data, reason) => { @@ -196,10 +205,13 @@ export default class Hydro implements Session { this.ws.send('{"key":"start"}'); if (!this.config.noStatus) { const info = await sysinfo.get(); - this.ws.send(JSON.stringify({ key: 'status', info })); + this.ws.send(JSON.stringify({ key: 'status', info: { ...info } })); + compilerVersionCallback = () => { + this.ws.send(JSON.stringify({ key: 'status', info: { ...info, compilers } })); + }; setInterval(async () => { const [mid, inf] = await sysinfo.update(); - this.ws.send(JSON.stringify({ key: 'status', info: { mid, ...inf } })); + this.ws.send(JSON.stringify({ key: 'status', info: { mid, ...inf, compilers } })); }, 1200000); } resolve(null); diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 7725cc6ae..9ca9a8c07 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "4.12.3", + "version": "4.13.1", "bin": "bin/hydrooj.js", "main": "src/plugin-api", "module": "src/plugin-api", @@ -12,38 +12,37 @@ }, "preferUnplugged": true, "dependencies": { - "@aws-sdk/client-s3": "3.591.0", - "@aws-sdk/lib-storage": "3.591.0", - "@aws-sdk/s3-presigned-post": "3.591.0", - "@aws-sdk/s3-request-presigner": "3.591.0", + "@aws-sdk/client-s3": "3.609.0", + "@aws-sdk/lib-storage": "3.609.0", + "@aws-sdk/s3-presigned-post": "3.609.0", + "@aws-sdk/s3-request-presigner": "3.609.0", "@graphql-tools/schema": "^10.0.4", "@hydrooj/framework": "workspace:^", "@hydrooj/utils": "workspace:^", "@simplewebauthn/server": "7.4.0", "@undefined-moe/require-resolve-hook": "^1.1.0", - "adm-zip": "0.5.5", + "adm-zip": "0.5.14", "cac": "^6.7.14", "cordis": "3.16.0", "detect-browser": "^5.3.0", "emoji-regex": "^10.3.0", "emojis-list": "2.1.0", "fs-extra": "^11.2.0", - "graphql": "^16.8.1", + "graphql": "^16.9.0", "graphql-scalars": "^1.23.0", "js-yaml": "^4.1.0", "koa-proxies": "^0.12.4", "koa-static-cache": "^5.1.4", "lodash": "^4.17.21", - "lru-cache": "^10.2.2", + "lru-cache": "^10.3.1", "mime-types": "^2.1.35", "moment-timezone": "^0.5.45", "mongodb": "^5.9.2", "mongodb-uri": "^0.9.7", "nanoid": "^5.0.7", - "nodemailer": "^6.9.13", + "nodemailer": "^6.9.14", "notp": "^2.0.3", "p-queue": "^8.0.1", - "path-to-regexp": "^6.2.2", "sanitize-filename": "^1.6.3", "saslprep": "^1.0.3", "schemastery": "^3.14.6", @@ -51,15 +50,14 @@ "shorty.js": "^1.0.1", "superagent": "^9.0.2", "tar": "6.2.1", - "thirty-two": "^1.0.2", - "ws": "^8.17.0" + "thirty-two": "^1.0.2" }, "devDependencies": { "@types/adm-zip": "^0.4.34", "@types/fs-extra": "^11.0.4", "@types/js-yaml": "^4.0.9", "@types/koa-static-cache": "^5.1.4", - "@types/lodash": "^4.17.4", + "@types/lodash": "^4.17.6", "@types/mime-types": "^2.1.4", "@types/mongodb-uri": "^0.9.4", "@types/nodemailer": "^6.4.15", diff --git a/packages/hydrooj/setting.yaml b/packages/hydrooj/setting.yaml index 5ff8a6ede..e9e3b6501 100644 --- a/packages/hydrooj/setting.yaml +++ b/packages/hydrooj/setting.yaml @@ -23,12 +23,12 @@ langs: - ~ All keys starting with _ will be ignored - ~ Entries named like [foo], [foo].[bar] [foo].[baz] - will be merged into one category. - - ~ A entry named [foo].[bar] will inhert all options + - ~ A entry named [foo].[bar] will inhert all options - defined in entry [foo] if not overwritten - ~ code_file option defaults to foo.[name] - ~ highlight option defaults to [name] - ~ if highlight option was set, monaco option defaults - - to [highlight], otherwise defaults to [name] + - to [highlight], otherwise defaults to [name] - ~ Compiling be be ignored if compile option wasn't set - ~ Key marked with a ? means the option is optional - (name): @@ -49,11 +49,13 @@ langs: code_file: foo.sh execute: /bin/bash foo.sh highlight: bash + version: /bin/bash --version c: compile: /usr/bin/gcc -Wall --std=c99 -o foo foo.c -lm highlight: c monaco: c display: C + version: /usr/bin/gcc --version cc: compile: /usr/bin/g++ -Wall -std=c++14 -o foo foo.cc -lm -I/include code_file: foo.cc @@ -62,6 +64,7 @@ langs: display: C++ address_space_limit: true process_limit: 1 + version: /usr/bin/g++ --version cc.cc98: compile: /usr/bin/g++ -Wall -std=c++98 -o foo foo.cc -lm -I/include display: C++98 @@ -95,6 +98,7 @@ langs: compile: /usr/bin/fpc -O2 -o/w/foo foo.pas highlight: pascal display: Pascal + version: /usr/bin/fpc -iW java: compile: /usr/bin/bash -c "javac -d /w -encoding utf8 ./Main.java && jar cvf Main.jar *.class >/dev/null" code_file: Main.java @@ -105,6 +109,7 @@ langs: monaco: java display: Java compile_time_limit: 30000 + version: /usr/bin/javac -version kt: compile: echo "Invalid language selection" code_file: foo.kt @@ -119,6 +124,7 @@ langs: execute: /usr/bin/java -jar foo.jar display: Kotlin/JVM compile_time_limit: 30000 + version: /usr/bin/kotlinc -version py: compile: /usr/bin/python -c "import py_compile; py_compile.compile('/w/foo.py', '/w/foo', doraise=True)" code_file: foo.py @@ -132,38 +138,47 @@ langs: compile: /usr/bin/python3 -c "import py_compile; py_compile.compile('/w/foo.py', '/w/foo', doraise=True)" execute: /usr/bin/python3 foo display: Python 3 + version: /usr/bin/python3 --version py.pypy3: compile: /bin/bash -c "/usr/bin/pypy3 -c \"import py_compile; py_compile.compile('/w/foo.py', '/w/foo', doraise=True)\" && mv foo.py foo" execute: /bin/bash -c 'mv foo foo.py && /usr/bin/pypy3 -B foo.py' display: PyPy3 + version: /usr/bin/pypy3 --version php: execute: /usr/bin/php foo.php display: PHP + version: /usr/bin/php -v rs: compile: /usr/bin/rustc -O -o /w/foo /w/foo.rs highlight: rust display: Rust + version: /usr/bin/rustc --version hs: compile: /usr/bin/ghc -O -outputdir /tmp -o foo foo.hs highlight: haskell display: Haskell + version: /usr/bin/ghc --version js: execute: /usr/bin/node /w/foo.js highlight: javascript display: NodeJS + version: /usr/bin/node --version go: compile: /usr/bin/go build -o foo foo.go display: Golang + version: /usr/bin/go version rb: execute: /usr/bin/ruby foo.rb highlight: ruby display: Ruby + version: /usr/bin/ruby --version cs: compile: /usr/bin/mcs -optimize+ -out:/w/foo /w/foo.cs execute: /usr/bin/mono foo highlight: csharp monaco: csharp display: 'C#' + version: /usr/bin/mcs --version r: display: R execute: /bin/sh -c "cat /dev/stdin >stdin && /usr/bin/Rscript --vanilla foo.r" diff --git a/packages/hydrooj/src/entry/setup.ts b/packages/hydrooj/src/entry/setup.ts index 2814c29f1..9f6e20fea 100644 --- a/packages/hydrooj/src/entry/setup.ts +++ b/packages/hydrooj/src/entry/setup.ts @@ -1,102 +1,39 @@ +/* eslint-disable no-await-in-loop */ import os from 'os'; import path from 'path'; -import cac from 'cac'; import fs from 'fs-extra'; -import type { Context } from 'koa'; import { MongoClient, WriteConcern } from 'mongodb'; import mongoUri from 'mongodb-uri'; -import { Context as PluginContext } from '../context'; +import { readline } from '../ui'; -const listenPort = cac().parse().options.port || 8888; -let resolve; - -async function get(ctx: Context) { - ctx.body = ` - - - - - - - Setup - - -
-
-
-
-

Setup

-
-
- -
-
-
- -
-
-
-
-
-
-
- - `; - ctx.response.type = 'text/html'; -} - -async function post(ctx: Context) { - try { - if (!ctx.request.body?.url) throw new Error('no url found'); - const url = mongoUri.parse(ctx.request.body.url); - const Database = await MongoClient.connect(ctx.request.body.url, { - readPreference: 'nearest', - writeConcern: new WriteConcern('majority'), - }); - const db = Database.db(url.database); - const coll = db.collection('system'); - await Promise.all([ - coll.updateOne( - { _id: 'server.url' }, - { $set: { value: ctx.request.href } }, - { upsert: true }, - ), - coll.updateOne( - { _id: 'server.port' }, - { $set: { value: parseInt(listenPort as string, 10) } }, - { upsert: true }, - ), - ]); - fs.ensureDirSync(path.resolve(os.homedir(), '.hydro')); - fs.writeFileSync(path.resolve(os.homedir(), '.hydro', 'config.json'), JSON.stringify({ url: ctx.request.body.url })); - ctx.body = '

Done! Hydro is now starting.

'; - resolve?.(); - } catch (e) { - ctx.body = `Error connecting to database: ${e.message}\n${e.stack}`; +export async function load() { + console.error('Config file not found.'); + if (!process.stdin.isTTY) { + console.error('Please fill in ~/.hydro/config.json with MongoDB connection string in the following format:'); + console.error('{"url":"mongodb://localhost:27017/hydro"}'); + process.exit(1); + } + // eslint-disable-next-line no-constant-condition + while (true) { + console.error('Please type your MongoDB connection string below:'); + console.error('Example: mongodb://localhost:27017/hydro'); + try { + const uri = await readline(); + if (!uri) throw new Error('no url found'); + const url = mongoUri.parse(uri); + const Database = await MongoClient.connect(uri, { + readPreference: 'nearest', + writeConcern: new WriteConcern('majority'), + }); + const db = Database.db(url.database); + const coll = db.collection('system'); + await coll.updateOne({ _id: '_test' }, { $set: { value: 1 } }, { upsert: true }); + await coll.deleteOne({ _id: '_test' }); + fs.ensureDirSync(path.resolve(os.homedir(), '.hydro')); + fs.writeFileSync(path.resolve(os.homedir(), '.hydro', 'config.json'), JSON.stringify({ url: uri })); + break; + } catch (e) { + console.error(e); + } } } - -export function load(ctx: PluginContext) { - let dispose; - ctx.plugin(require('@hydrooj/framework'), { - keys: [Math.random()], - proxy: false, - upload: false, - port: listenPort, - }); - ctx.inject(['server'], ({ server }) => { - dispose = server.addServerLayer('setup', (c) => { - if (c.request.method.toLowerCase() === 'post') return post(c); - return get(c); - }); - }); - return new Promise((r) => { - resolve = () => { - dispose?.(); - r(null); - }; - }); -} diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index eed061989..51be0b8ab 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -26,7 +26,7 @@ export async function apply(ctx: Context) { const config = load(); if (!process.env.CI && !config) { logger.info('Starting setup'); - await require('./setup').load(); + await require('./setup').load(ctx); } const pending = global.addons; const fail = []; @@ -65,10 +65,10 @@ export async function apply(ctx: Context) { for (const h of handlers.filter((i) => i.endsWith('.ts'))) { ctx.loader.reloadPlugin(ctx, path.resolve(handlerDir, h), {}, `hydrooj/handler/${h.split('.')[0]}`); } + ctx.plugin(require('../service/migration').default); await handler(pending, fail, ctx); await addon(pending, fail, ctx); await ctx.lifecycle.flush(); - for (const i in global.Hydro.handler) await global.Hydro.handler[i](); const scriptDir = path.resolve(__dirname, '..', 'script'); for (const h of await fs.readdir(scriptDir)) { ctx.loader.reloadPlugin(ctx, path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`); @@ -78,27 +78,18 @@ export async function apply(ctx: Context) { await ctx.lifecycle.flush(); await ctx.parallel('app/started'); if (process.env.NODE_APP_INSTANCE === '0') { - const { default: scripts, init } = require('../upgrade'); - let dbVer = (await modelSystem.get('db.ver')) ?? 0; - const isFresh = !dbVer; - const expected = scripts.length; - if (isFresh) { - await init(); - await modelSystem.set('db.ver', expected); - } else { - while (dbVer < expected) { - const func = scripts[dbVer]; - if (typeof func !== 'function') { - dbVer++; - continue; + await new Promise((resolve, reject) => { + ctx.inject(['migration'], async (c) => { + c.migration.registerChannel('hydrooj', require('../upgrade').coreScripts); + try { + await c.migration.doUpgrade(); + resolve(null); + } catch (e) { + logger.error('Upgrade failed: %O', e); + reject(e); } - logger.info('Upgrading database: from %d to %d', dbVer, expected); - const result = await func(); - if (!result) break; - dbVer++; - await modelSystem.set('db.ver', dbVer); - } - } + }); + }); } for (const f of global.addons) { const dir = path.join(f, 'public'); diff --git a/packages/hydrooj/src/handler/domain.ts b/packages/hydrooj/src/handler/domain.ts index 6bc1adcb5..438ff8629 100644 --- a/packages/hydrooj/src/handler/domain.ts +++ b/packages/hydrooj/src/handler/domain.ts @@ -8,7 +8,6 @@ import { } from '../error'; import type { DomainDoc } from '../interface'; import avatar from '../lib/avatar'; -import paginate from '../lib/paginate'; import { PERM, PERMS_BY_FAMILY, PRIV } from '../model/builtin'; import * as discussion from '../model/discussion'; import domain from '../model/domain'; @@ -24,10 +23,10 @@ import { log2 } from '../utils'; class DomainRankHandler extends Handler { @query('page', Types.PositiveInt, true) async get(domainId: string, page = 1) { - const [dudocs, upcount, ucount] = await paginate( + const [dudocs, upcount, ucount] = await this.paginate( domain.getMultiUserInDomain(domainId, { uid: { $gt: 1 }, rp: { $gt: 0 } }).sort({ rp: -1 }), page, - 100, + 'ranking', ); const udict = await user.getList(domainId, dudocs.map((dudoc) => dudoc.uid)); const udocs = dudocs.map((i) => udict[i.uid]); diff --git a/packages/hydrooj/src/handler/status.ts b/packages/hydrooj/src/handler/status.ts index 5d0137aef..f19c23a75 100644 --- a/packages/hydrooj/src/handler/status.ts +++ b/packages/hydrooj/src/handler/status.ts @@ -3,6 +3,7 @@ import { PRIV } from '../model/builtin'; import * as DocumentModel from '../model/document'; import DomainModel from '../model/domain'; import RecordModel from '../model/record'; +import * as SettingModel from '../model/setting'; import UserModel from '../model/user'; import db from '../service/db'; import { Handler } from '../service/server'; @@ -25,11 +26,47 @@ async function getStatus() { class StatusHandler extends Handler { async get() { const stats = await getStatus(); + const compilers = {}; + const warn = {}; + const result: Array<{ key: string[], message: string }> = []; + // For each language, select the most common compiler message version, + // then merge languages with the same message. for (const stat of stats) { - if (!stat.battery.hasBattery) stat.battery = 'No battery'; + if (!stat.battery?.hasBattery) stat.battery = 'No battery'; else stat.battery = `${stat.battery.type} ${stat.battery.model} ${stat.battery.percent}%${stat.battery.isCharging ? ' Charging' : ''}`; + if (stat.compilers) { + for (const key in stat.compilers) { + if (!compilers[key]) compilers[key] ||= []; + const related = compilers[key].find((i) => i.message === stat.compilers[key]); + if (related) related.related.push(stat._id); + else { + compilers[key].push({ + related: [stat._id], + message: stat.compilers[key], + }); + } + } + } } - this.response.body = { stats }; + for (const key in compilers) { + compilers[key].sort((a, b) => b.related.length - a.related.length); + const message = compilers[key][0].message; + for (let i = 1; i < compilers[key].length; i++) { + for (const id of compilers[key][i].related) { + warn[id] = true; + } + } + const t = result.find((i) => i.message === message); + if (t) t.key.push(key); + else result.push({ key: [key], message }); + } + const LANGS = SettingModel.langs; + const languages = {}; + for (const key in LANGS) { + if (LANGS[key].hidden) continue; + languages[`${LANGS[key].display}(${key})`] = LANGS[key].compile || LANGS[key].execute; + } + this.response.body = { stats, languages, compilers: result }; this.response.template = 'status.html'; } } diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 45e0d3bfc..7f6e20211 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -1,5 +1,5 @@ -import { AttestationFormat } from '@simplewebauthn/server/dist/helpers/decodeAttestationObject'; -import { AuthenticationExtensionsAuthenticatorOutputs } from '@simplewebauthn/server/dist/helpers/decodeAuthenticatorExtensions'; +import type { AttestationFormat } from '@simplewebauthn/server/dist/helpers/decodeAttestationObject'; +import type { AuthenticationExtensionsAuthenticatorOutputs } from '@simplewebauthn/server/dist/helpers/decodeAuthenticatorExtensions'; import { CredentialDeviceType } from '@simplewebauthn/typescript-types'; import type fs from 'fs'; import type { Dictionary, NumericDictionary } from 'lodash'; @@ -644,6 +644,8 @@ export interface FileNode { size?: number; /** AutoDelete */ autoDelete?: Date; + /** fileId if linked to an existing file */ + link?: string; owner?: number; operator?: number[]; meta?: Record; @@ -777,11 +779,10 @@ export interface ProblemSearchOptions { export type ProblemSearch = (domainId: string, q: string, options?: ProblemSearchOptions) => Promise; export interface Lib extends Record { - difficulty: typeof import('./lib/difficulty'); + difficulty: typeof import('./lib/difficulty').default; buildContent: typeof import('./lib/content').buildContent; mail: typeof import('./lib/mail'); - rank: typeof import('./lib/rank'); - rating: typeof import('./lib/rating'); + rating: typeof import('./lib/rating').default; testdataConfig: typeof import('./lib/testdataConfig'); problemSearch: ProblemSearch; } @@ -807,8 +808,6 @@ export interface ModuleInterfaces { export interface HydroGlobal { version: Record; model: Model; - /** @deprecated */ - handler: Record; script: Record; service: HydroService; lib: Lib; diff --git a/packages/hydrooj/src/lib/difficulty.ts b/packages/hydrooj/src/lib/difficulty.ts index 87d924d34..231376806 100644 --- a/packages/hydrooj/src/lib/difficulty.ts +++ b/packages/hydrooj/src/lib/difficulty.ts @@ -42,5 +42,5 @@ function difficultyAlgorithm(nSubmit: number, nAccept: number) { return Math.max(ans, 1); } -export = difficultyAlgorithm; +export default difficultyAlgorithm; global.Hydro.lib.difficulty = difficultyAlgorithm; diff --git a/packages/hydrooj/src/lib/hash.hydro.ts b/packages/hydrooj/src/lib/hash.hydro.ts index 8d246413a..be7bf7a8c 100644 --- a/packages/hydrooj/src/lib/hash.hydro.ts +++ b/packages/hydrooj/src/lib/hash.hydro.ts @@ -9,5 +9,5 @@ function hash(password: string, salt: string): Promise { }); } -export = hash; +export default hash; global.Hydro.module.hash.hydro = hash; diff --git a/packages/hydrooj/src/lib/index.ts b/packages/hydrooj/src/lib/index.ts index 0b836d544..9ba5eff22 100644 --- a/packages/hydrooj/src/lib/index.ts +++ b/packages/hydrooj/src/lib/index.ts @@ -1,7 +1,6 @@ import './i18n'; import './mail'; import './hash.hydro'; -import './rank'; import './ui'; import './testdataConfig'; import './difficulty'; diff --git a/packages/hydrooj/src/lib/paginate.ts b/packages/hydrooj/src/lib/paginate.ts deleted file mode 100644 index 59b3b7f79..000000000 --- a/packages/hydrooj/src/lib/paginate.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { FindCursor } from 'mongodb'; -import { ValidationError } from '../error'; -import db from '../service/db'; - -async function paginate( - cursor: FindCursor, page: number, pageSize: number, -): Promise<[docs: T[], numPages: number, count: number]> { - if (page <= 0) throw new ValidationError('page'); - let filter = {}; - for (const key of Object.getOwnPropertySymbols(cursor)) { - if (key.toString() !== 'Symbol(filter)') continue; - filter = cursor[key]; - break; - } - const coll = db.collection(cursor.namespace.collection as any); - const [count, pageDocs] = await Promise.all([ - Object.keys(filter).length ? coll.count(filter) : coll.countDocuments(filter), - cursor.skip((page - 1) * pageSize).limit(pageSize).toArray(), - ]); - const numPages = Math.floor((count + pageSize - 1) / pageSize); - return [pageDocs, numPages, count]; -} - -export = paginate; diff --git a/packages/hydrooj/src/lib/rank.ts b/packages/hydrooj/src/lib/rank.ts deleted file mode 100644 index 2f6fc17a9..000000000 --- a/packages/hydrooj/src/lib/rank.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { FindCursor } from 'mongodb'; - -type ranked = >(cursor: FindCursor, equ: ((a: T, b: T) => boolean)) => Promise<[number, T][]>; - -const ranked: ranked = async (cursor, equ) => { - let last = null; - let r = 0; - let count = 0; - const results = []; - const docs = await cursor.toArray(); - for (const doc of docs) { - if (doc.unrank) { - results.push([0, doc]); - continue; - } - count++; - if (!last || !equ(last, doc)) r = count; - last = doc; - results.push([r, doc]); - } - return results; -}; - -global.Hydro.lib.rank = ranked; -export = ranked; diff --git a/packages/hydrooj/src/lib/rating.ts b/packages/hydrooj/src/lib/rating.ts index 029cf8030..5c89c6678 100644 --- a/packages/hydrooj/src/lib/rating.ts +++ b/packages/hydrooj/src/lib/rating.ts @@ -118,6 +118,6 @@ function calculate(users: RatingInputUser[]): RatingOutputUser[] { return calculator.calculate() as RatingOutputUser[]; } -export = calculate; +export default calculate; global.Hydro.lib.rating = calculate; diff --git a/packages/hydrooj/src/libs.ts b/packages/hydrooj/src/libs.ts index b207256e6..ffd6b34eb 100644 --- a/packages/hydrooj/src/libs.ts +++ b/packages/hydrooj/src/libs.ts @@ -5,7 +5,7 @@ import Schema from 'schemastery'; import superagent from 'superagent'; import { Context } from './context'; export { ObjectId, Filter } from 'mongodb'; -export { WebSocket, WebSocketServer } from 'ws'; +export { WebSocket, WebSocketServer } from '@hydrooj/framework'; export * from './utils'; export * from './interface'; export * from './typeutils'; diff --git a/packages/hydrooj/src/model/contest.ts b/packages/hydrooj/src/model/contest.ts index 1e208bff3..03def05d9 100644 --- a/packages/hydrooj/src/model/contest.ts +++ b/packages/hydrooj/src/model/contest.ts @@ -9,7 +9,6 @@ import { BaseUserDict, ContestRule, ContestRules, ProblemDict, RecordDoc, ScoreboardConfig, ScoreboardNode, ScoreboardRow, SubtaskResult, Tdoc, } from '../interface'; -import ranked from '../lib/rank'; import * as bus from '../service/bus'; import db from '../service/db'; import type { Handler } from '../service/server'; @@ -224,7 +223,7 @@ const acm = buildContestRule({ return row; }, async scoreboard(config, _, tdoc, pdict, cursor) { - const rankedTsdocs = await ranked(cursor, (a, b) => a.score === b.score && a.time === b.time); + const rankedTsdocs = await db.ranked(cursor, (a, b) => a.score === b.score && a.time === b.time); const uids = rankedTsdocs.map(([, tsdoc]) => tsdoc.uid); const udict = await user.getListForRender(tdoc.domainId, uids, config.showDisplayName); // Find first accept @@ -257,7 +256,7 @@ const acm = buildContestRule({ return [rows, udict]; }, async ranked(tdoc, cursor) { - return await ranked(cursor, (a, b) => a.accept === b.accept && a.time === b.time); + return await db.ranked(cursor, (a, b) => a.accept === b.accept && a.time === b.time); }, applyProjection(tdoc, rdoc) { if (isDone(tdoc)) return rdoc; @@ -394,7 +393,7 @@ const oi = buildContestRule({ return row; }, async scoreboard(config, _, tdoc, pdict, cursor) { - const rankedTsdocs = await ranked(cursor, (a, b) => a.score === b.score); + const rankedTsdocs = await db.ranked(cursor, (a, b) => a.score === b.score); const uids = rankedTsdocs.map(([, tsdoc]) => tsdoc.uid); const udict = await user.getListForRender(tdoc.domainId, uids, config.showDisplayName); const psdict = {}; @@ -431,7 +430,7 @@ const oi = buildContestRule({ return [rows, udict]; }, async ranked(tdoc, cursor) { - return await ranked(cursor, (a, b) => a.score === b.score); + return await db.ranked(cursor, (a, b) => a.score === b.score); }, applyProjection(tdoc, rdoc) { if (isDone(tdoc)) return rdoc; @@ -744,7 +743,7 @@ const homework = buildContestRule({ return row; }, async scoreboard(config, _, tdoc, pdict, cursor) { - const rankedTsdocs = await ranked(cursor, (a, b) => a.score === b.score); + const rankedTsdocs = await db.ranked(cursor, (a, b) => a.score === b.score); const uids = rankedTsdocs.map(([, tsdoc]) => tsdoc.uid); const udict = await user.getListForRender(tdoc.domainId, uids, config.showDisplayName); const columns = await this.scoreboardHeader(config, _, tdoc, pdict); @@ -757,7 +756,7 @@ const homework = buildContestRule({ return [rows, udict]; }, async ranked(tdoc, cursor) { - return await ranked(cursor, (a, b) => a.score === b.score); + return await db.ranked(cursor, (a, b) => a.score === b.score); }, }); diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index dee3c7b47..ef1bde802 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -69,13 +69,13 @@ export class ProblemModel { static PROJECTION_CONTEST_DETAIL: Field[] = [ ...ProblemModel.PROJECTION_CONTEST_LIST, 'content', 'html', 'data', 'config', 'additional_file', - 'reference', + 'reference', 'maintainer', ]; static PROJECTION_PUBLIC: Field[] = [ ...ProblemModel.PROJECTION_LIST, 'content', 'html', 'data', 'config', 'additional_file', - 'reference', + 'reference', 'maintainer', ]; static default = { @@ -262,7 +262,8 @@ export class ProblemModel { const res = await Promise.all([ document.deleteOne(domainId, document.TYPE_PROBLEM, docId), document.deleteMultiStatus(domainId, document.TYPE_PROBLEM, { docId }), - storage.list(`problem/${domainId}/${docId}/`).then((items) => storage.del(items.map((item) => item.prefix + item.name))), + storage.list(`problem/${domainId}/${docId}/`) + .then((items) => storage.del(items.map((item) => `problem/${domainId}/${docId}/${item.name}`))), bus.parallel('problem/delete', domainId, docId), ]); await bus.emit('problem/del', domainId, docId); @@ -363,7 +364,7 @@ export class ProblemModel { domainId: string, pids: number[], canViewHidden: number | boolean = false, doThrow = true, projection = ProblemModel.PROJECTION_PUBLIC, indexByDocIdOnly = false, ): Promise { - if (!pids?.length) return []; + if (!pids?.length) return {}; const r: Record = {}; const l: Record = {}; const q: any = { docId: { $in: pids } }; diff --git a/packages/hydrooj/src/model/record.ts b/packages/hydrooj/src/model/record.ts index 093cf0664..087b1bccc 100644 --- a/packages/hydrooj/src/model/record.ts +++ b/packages/hydrooj/src/model/record.ts @@ -280,22 +280,23 @@ export function apply(ctx: Context) { }, { upsert: true }); } }); - ctx.on('ready', async () => { - await db.ensureIndexes( + ctx.on('ready', () => Promise.all([ + db.ensureIndexes( RecordModel.coll, + { key: { domainId: 1, pid: 1 }, name: 'delete' }, { key: { domainId: 1, contest: 1, _id: -1 }, name: 'basic' }, { key: { domainId: 1, contest: 1, uid: 1, _id: -1 }, name: 'withUser' }, { key: { domainId: 1, contest: 1, pid: 1, _id: -1 }, name: 'withProblem' }, { key: { domainId: 1, contest: 1, pid: 1, uid: 1, _id: -1 }, name: 'withUserAndProblem' }, { key: { domainId: 1, contest: 1, status: 1, _id: -1 }, name: 'withStatus' }, - ); - await db.ensureIndexes( + ), + db.ensureIndexes( RecordModel.collStat, { key: { domainId: 1, pid: 1, uid: 1, _id: -1 }, name: 'basic' }, { key: { domainId: 1, pid: 1, uid: 1, time: 1 }, name: 'time' }, { key: { domainId: 1, pid: 1, uid: 1, memory: 1 }, name: 'memory' }, { key: { domainId: 1, pid: 1, uid: 1, length: 1 }, name: 'length' }, - ); - }); + ), + ]) as any); } global.Hydro.model.record = RecordModel; diff --git a/packages/hydrooj/src/model/setting.ts b/packages/hydrooj/src/model/setting.ts index 8b7d7f1d2..feb7358b8 100644 --- a/packages/hydrooj/src/model/setting.ts +++ b/packages/hydrooj/src/model/setting.ts @@ -274,6 +274,7 @@ SystemSetting( Setting('setting_basic', 'pagination.contest', 20, 'number', 'pagination.contest', 'Contests per page'), Setting('setting_basic', 'pagination.discussion', 50, 'number', 'pagination.discussion', 'Discussions per page'), Setting('setting_basic', 'pagination.record', 100, 'number', 'pagination.record', 'Records per page'), + Setting('setting_basic', 'pagination.ranking', 100, 'number', 'pagination.ranking', 'Users per page'), Setting('setting_basic', 'pagination.solution', 20, 'number', 'pagination.solution', 'Solutions per page'), Setting('setting_basic', 'pagination.training', 10, 'number', 'pagination.training', 'Trainings per page'), Setting('setting_basic', 'pagination.reply', 50, 'number', 'pagination.reply', 'Replies per page'), diff --git a/packages/hydrooj/src/model/storage.ts b/packages/hydrooj/src/model/storage.ts index cd9c6631d..c0f2972ed 100644 --- a/packages/hydrooj/src/model/storage.ts +++ b/packages/hydrooj/src/model/storage.ts @@ -1,5 +1,5 @@ import { extname } from 'path'; -import { escapeRegExp } from 'lodash'; +import { escapeRegExp, omit } from 'lodash'; import moment from 'moment-timezone'; import { nanoid } from 'nanoid'; import type { Readable } from 'stream'; @@ -39,7 +39,7 @@ export class StorageModel { { $set: { lastUsage: new Date() } }, { returnDocument: 'after' }, ); - return await storage.get(value?._id || path, savePath); + return await storage.get(value?.link || value?._id || path, savePath); } static async rename(path: string, newPath: string, operator = 1) { @@ -51,6 +51,20 @@ export class StorageModel { static async del(path: string[], operator = 1) { if (!path.length) return; + const affected = await StorageModel.coll.find({ path: { $in: path } }).toArray(); + if (!affected.length) return; + const linked = await StorageModel.coll.find({ link: { $in: affected.map((i) => i._id) }, path: { $nin: path } }).toArray(); + const processedIds = []; + for (const i of linked || []) { + if (processedIds.includes(i.link)) continue; + const current = affected.find((t) => t._id === t.link); // to be deleted + // eslint-disable-next-line no-await-in-loop + await Promise.all([ + StorageModel.coll.updateOne({ _id: current._id }, { $set: omit(i, ['_id']) }), + StorageModel.coll.updateOne({ _id: i._id }, { $set: omit(current, ['_id']), $unset: { link: '' } }), + ]); + processedIds.push(i.link); + } const autoDelete = moment().add(7, 'day').toDate(); await StorageModel.coll.updateMany( { path: { $in: path }, autoDelete: null }, @@ -66,7 +80,7 @@ export class StorageModel { autoDelete: null, }).toArray(); return results.map((i) => ({ - ...i, name: i.path.split(target)[1], prefix: target, + ...i, name: i.path.split(target)[1], })); } @@ -106,12 +120,10 @@ export class StorageModel { // Make sure id is not used // eslint-disable-next-line no-await-in-loop while (await StorageModel.coll.findOne({ _id })) _id = StorageModel.generateId(extname(dst)); - const result = await storage.copy(value._id, _id); - const { metaData, size, etag } = await storage.getMeta(_id); await StorageModel.coll.insertOne({ - _id, meta: metaData, path: dst, size, etag, lastModified: new Date(), owner: value.owner || 1, + ...value, _id, path: dst, link: value._id, lastModified: new Date(), owner: value.owner || 1, }); - return result; + return _id; } } diff --git a/packages/hydrooj/src/model/user.ts b/packages/hydrooj/src/model/user.ts index eb2bda954..8a4389bd1 100644 --- a/packages/hydrooj/src/model/user.ts +++ b/packages/hydrooj/src/model/user.ts @@ -179,7 +179,7 @@ declare module '@hydrooj/framework' { interface UserModel extends User { } } -function handleMailLower(mail: string) { +export function handleMailLower(mail: string) { const [n, d] = mail.trim().toLowerCase().split('@'); const [name] = n.split('+'); return `${name.replace(/\./g, '')}@${d === 'googlemail.com' ? 'gmail.com' : d}`; diff --git a/packages/hydrooj/src/plugin-api.ts b/packages/hydrooj/src/plugin-api.ts index c58de3abf..fafe3f107 100644 --- a/packages/hydrooj/src/plugin-api.ts +++ b/packages/hydrooj/src/plugin-api.ts @@ -43,8 +43,10 @@ export { buildContent } from './lib/content'; export { default as mime } from './lib/mime'; export { default as rating } from './lib/rating'; export { default as avatar } from './lib/avatar'; -export { default as rank } from './lib/rank'; -export { default as paginate } from './lib/paginate'; +/** @deprecated use Handler.paginate instead */ +export const paginate = db.paginate.bind(db); +/** @deprecated use db.ranked instead */ +export const rank = db.ranked.bind(db); export { UiContextBase } from './service/layers/base'; export * from '@hydrooj/framework/decorators'; export * from '@hydrooj/framework/validator'; diff --git a/packages/hydrooj/src/service/db.ts b/packages/hydrooj/src/service/db.ts index 6e9768708..9430b9ea6 100644 --- a/packages/hydrooj/src/service/db.ts +++ b/packages/hydrooj/src/service/db.ts @@ -1,9 +1,10 @@ /* eslint-disable no-await-in-loop */ import { - Collection, Db, IndexDescription, MongoClient, + Collection, Db, FindCursor, IndexDescription, MongoClient, } from 'mongodb'; import mongoUri from 'mongodb-uri'; import { Time } from '@hydrooj/utils'; +import { ValidationError } from '../error'; import { Logger } from '../logger'; import { load } from '../options'; import * as bus from './bus'; @@ -102,6 +103,44 @@ class MongoService { } } + async paginate( + cursor: FindCursor, page: number, pageSize: number, + ): Promise<[docs: T[], numPages: number, count: number]> { + if (page <= 0) throw new ValidationError('page'); + let filter = {}; + for (const key of Object.getOwnPropertySymbols(cursor)) { + if (key.toString() !== 'Symbol(filter)') continue; + filter = cursor[key]; + break; + } + const coll = this.db.collection(cursor.namespace.collection as any); + const [count, pageDocs] = await Promise.all([ + Object.keys(filter).length ? coll.count(filter) : coll.countDocuments(filter), + cursor.skip((page - 1) * pageSize).limit(pageSize).toArray(), + ]); + const numPages = Math.floor((count + pageSize - 1) / pageSize); + return [pageDocs, numPages, count]; + } + + async ranked>(cursor: FindCursor, equ: (a: T, b: T) => boolean): Promise<[number, T][]> { + let last = null; + let r = 0; + let count = 0; + const results = []; + const docs = await cursor.toArray(); + for (const doc of docs) { + if ((doc as any).unrank) { + results.push([0, doc]); + continue; + } + count++; + if (!last || !equ(last, doc)) r = count; + last = doc; + results.push([r, doc]); + } + return results; + } + public async apply(ctx) { await this.start(); ctx.on('dispose', () => this.client.close()); diff --git a/packages/hydrooj/src/service/layers/domain.ts b/packages/hydrooj/src/service/layers/domain.ts index 93a43bb14..a9f1c92ae 100644 --- a/packages/hydrooj/src/service/layers/domain.ts +++ b/packages/hydrooj/src/service/layers/domain.ts @@ -1,4 +1,4 @@ -import { KoaContext } from '@hydrooj/framework'; +import { KoaContext, NotFoundError } from '@hydrooj/framework'; import BlackListModel from '../../model/blacklist'; import DomainModel from '../../model/domain'; import * as system from '../../model/system'; @@ -24,5 +24,12 @@ export default async (ctx: KoaContext, next) => { ctx.domainId = inferDomain?._id || domainId; ctx.domainInfo = inferDomain || absoluteDomain; if (ctx.domainInfo && ctx.domainId !== ctx.domainInfo._id) ctx.redirect(ctx.originalPath.replace(/^\/d\/[^/]+\//, `/d/${ctx.domainInfo._id}/`)); - else await next(); + else { + if (!ctx.domainInfo) { + ctx.pendingError = new NotFoundError(ctx.domainId); + ctx.domainId = 'system'; + ctx.domainInfo = await DomainModel.get('system'); + } + await next(); + } }; diff --git a/packages/hydrooj/src/service/layers/user.ts b/packages/hydrooj/src/service/layers/user.ts index 60575f45e..c9df9cb76 100644 --- a/packages/hydrooj/src/service/layers/user.ts +++ b/packages/hydrooj/src/service/layers/user.ts @@ -1,5 +1,4 @@ import type { KoaContext } from '@hydrooj/framework'; -import { NotFoundError } from '../../error'; import { PERM } from '../../model/builtin'; import UserModel from '../../model/user'; @@ -15,9 +14,5 @@ export default async (ctx: KoaContext, next) => { } if (user._id === 0) delete user.viewLang; ctx.HydroContext.user = await user.private(); - if (!domain) { - ctx.pendingError = new NotFoundError(args.domainId); - args.domainId = 'system'; - } await next(); }; diff --git a/packages/hydrooj/src/service/migration.ts b/packages/hydrooj/src/service/migration.ts new file mode 100644 index 000000000..176968ab5 --- /dev/null +++ b/packages/hydrooj/src/service/migration.ts @@ -0,0 +1,52 @@ +/* eslint-disable no-await-in-loop */ +import { Logger } from '@hydrooj/utils'; +import { Context, Service } from '../context'; +import * as system from '../model/system'; + +export type MigrationScript = null | (() => Promise); +const logger = new Logger('migration'); + +declare module '../context' { + interface Context { + migration: MigrationService; + } +} + +export default class MigrationService extends Service { + private channels: Record = {}; + + constructor(ctx: Context) { + super(ctx, 'migration', true); + } + + async registerChannel(name: string, s: MigrationScript[]) { + this.channels[name] = s; + } + + async doUpgrade() { + for (const channel in this.channels) { + const name = `db.ver${channel === 'hydrooj' ? '' : `-${channel}`}`; + let dbVer = system.get(name) ?? 0; + const scripts = this.channels[channel]; + const isFresh = !dbVer; + const expected = scripts.length; + if (isFresh) { + await scripts[0]?.(); + await system.set(name, expected); + continue; + } + while (dbVer < expected) { + const func = scripts[dbVer]; + if (typeof func !== 'function') { + dbVer++; + continue; + } + logger.info('Upgrading database [%s]: from %d to %d', channel, dbVer, expected); + const result = await func(); + if (!result) break; + dbVer++; + await system.set(name, dbVer); + } + } + } +} diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index 7657b8445..e86a69b1a 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -11,13 +11,13 @@ import { errorMessage, Time } from '@hydrooj/utils'; import { Context } from '../context'; import { PermissionError, PrivilegeError } from '../error'; import type { DomainDoc } from '../interface'; -import paginate from '../lib/paginate'; import { Logger } from '../logger'; import { PERM, PRIV } from '../model/builtin'; import * as opcount from '../model/opcount'; import * as OplogModel from '../model/oplog'; import * as system from '../model/system'; import { builtinConfig } from '../settings'; +import db from './db'; import baseLayer from './layers/base'; import domainLayer from './layers/domain'; import userLayer from './layers/user'; @@ -144,8 +144,18 @@ export async function apply(ctx: Context) { }); if (process.env.HYDRO_CLI) return; ctx.inject(['server'], ({ server, on }) => { + let endpoint = builtinConfig.file.endPoint; + if (builtinConfig.file.type === 's3' && !builtinConfig.file.pathStyle) { + try { + const parsed = new URL(builtinConfig.file.endPoint); + parsed.hostname = `${builtinConfig.file.bucket}.${parsed.hostname}`; + endpoint = parsed.toString(); + } catch (e) { + logger.warn('Failed to parse file endpoint'); + } + } const proxyMiddleware = proxy('/fs', { - target: builtinConfig.file.endPoint, + target: endpoint, changeOrigin: true, rewrite: (p) => p.replace('/fs', ''), }); @@ -209,7 +219,7 @@ export async function apply(ctx: Context) { ? (!host.includes(this.request.host)) : this.request.host !== host) )) withDomainId ||= domainId; - res = this.context.router.url.call(this.context.router, name, args, { query }).toString(); + res = ctx.server.router.url.call(ctx.server.router, name, args, { query }).toString(); if (anchor) res = `${res}#${anchor}`; if (withDomainId) res = `/d/${withDomainId}${res}`; } catch (e) { @@ -228,7 +238,7 @@ export async function apply(ctx: Context) { return res; }, paginate(cursor: FindCursor, page: number, key: string) { - return paginate(cursor, page, this.ctx.setting.get(`pagination.${key}`)); + return db.paginate(cursor, page, this.ctx.setting.get(`pagination.${key}`)); }, checkPerm(...args: bigint[]) { if (!this.user.hasPerm(...args)) { @@ -255,7 +265,7 @@ export async function apply(ctx: Context) { }, renderTitle(str: string) { const name = this.ctx.setting.get('server.name'); - if (this.UiContext.extraTitleContent) return `${this.translate(str)} - ${this.UiContext.extraTitleContent} - ${name}`; + if (this.UiContext.extraTitleContent) return `${this.UiContext.extraTitleContent} - ${this.translate(str)} - ${name}`; return `${this.translate(str)} - ${name}`; }, }); @@ -276,6 +286,7 @@ export async function apply(ctx: Context) { text: global.Hydro.module.oauth[key].text, })); if (!h.noCheckPermView && !h.user.hasPriv(PRIV.PRIV_VIEW_ALL_DOMAIN)) h.checkPerm(PERM.PERM_VIEW); + if (h.context.pendingError) throw h.context.pendingError; }); on('app/listen', () => { diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index fdb15a668..06b7c54df 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -2,8 +2,8 @@ import { dirname, resolve } from 'path'; import { PassThrough, Readable } from 'stream'; import { URL } from 'url'; import { - CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, - GetObjectCommand, HeadObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client, + DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, + HeadObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client, } from '@aws-sdk/client-s3'; import { Upload } from '@aws-sdk/lib-storage'; import { createPresignedPost } from '@aws-sdk/s3-presigned-post'; @@ -243,16 +243,6 @@ class RemoteStorageService { return { url, fields }; } - async copy(src: string, target: string) { - src = convertPath(src); - target = convertPath(target); - return await this.client.send(new CopyObjectCommand({ - Bucket: this.bucket, - Key: target, - CopySource: src, - })); - } - async status() { return { type: 'S3', @@ -332,14 +322,6 @@ class LocalStorageService { throw new Error('Not implemented'); } - async copy(src: string, target: string) { - src = resolve(this.dir, convertPath(src)); - target = resolve(this.dir, convertPath(target)); - await ensureDir(dirname(target)); - await copyFile(src, target); - return { etag: target, lastModified: new Date() }; - } - async status() { return { type: 'Local', diff --git a/packages/hydrooj/src/ui.ts b/packages/hydrooj/src/ui.ts index 9372c9002..57747aca6 100644 --- a/packages/hydrooj/src/ui.ts +++ b/packages/hydrooj/src/ui.ts @@ -22,7 +22,6 @@ process.on('SIGTERM', terminate); const shell = new Logger('shell'); async function executeCommand(input: string) { - input = input.trim(); // Clear the stack setImmediate(async () => { if (input === 'exit' || input === 'quit' || input === 'shutdown') { @@ -39,9 +38,20 @@ async function executeCommand(input: string) { }); } +let readlineCallback; + process.stdin.setEncoding('utf-8'); process.stdin.setRawMode?.(false); process.stdin.on('data', (buf) => { - const input = buf.toString(); - executeCommand(input); + const input = buf.toString().trim(); + if (readlineCallback) { + readlineCallback(input); + readlineCallback = null; + } else executeCommand(input); }); + +export const useReadline = (callback: (string) => any) => { + if (readlineCallback) throw new Error('Already waiting for input.'); + readlineCallback = callback; +}; +export const readline = () => new Promise((resolve) => { useReadline(resolve); }); diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 7ed111307..303dfef41 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -21,37 +21,35 @@ import StorageModel from './model/storage'; import * as system from './model/system'; import TaskModel from './model/task'; import * as training from './model/training'; -import user from './model/user'; +import user, { handleMailLower } from './model/user'; import { iterateAllContest, iterateAllDomain, iterateAllProblem, iterateAllUser, } from './pipelineUtils'; import db from './service/db'; +import { MigrationScript } from './service/migration'; import { setBuiltinConfig } from './settings'; import welcome from './welcome'; const logger = new Logger('upgrade'); -type UpgradeScript = void | (() => Promise); const unsupportedUpgrade = async function _26_27() { throw new Error('This upgrade was no longer supported in hydrooj@4. \ Please use hydrooj@3 to perform these upgrades before upgrading to v4'); }; -export async function init() { - if (!await user.getById('system', 0)) { - await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER); - } - if (!await user.getById('system', 1)) { - await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE); - } - const ddoc = await domain.get('system'); - if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!'); - await welcome(); - return true; -} - -const scripts: UpgradeScript[] = [ +export const coreScripts: MigrationScript[] = [ // Mark as used - null, + async function init() { + if (!await user.getById('system', 0)) { + await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER); + } + if (!await user.getById('system', 1)) { + await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE); + } + const ddoc = await domain.get('system'); + if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!'); + await welcome(); + return true; + }, // Init ...new Array(26).fill(unsupportedUpgrade), null, @@ -675,6 +673,16 @@ const scripts: UpgradeScript[] = [ if (bulk.batches.length) await bulk.execute(); return true; }, + async function _89_90() { + return await iterateAllUser(async (udoc) => { + const wanted = handleMailLower(udoc.mail); + if (wanted !== udoc.mailLower) { + if (await user.getByEmail('system', wanted)) { + console.warn('Email conflict when trying to rename %s to %s', udoc.mailLower, wanted); + return; + } + await user.coll.updateOne({ _id: udoc._id }, { $set: { mailLower: wanted } }); + } + }); + }, ]; - -export default scripts; diff --git a/packages/prom-client/package.json b/packages/prom-client/package.json index 53cac434d..e82739b4d 100644 --- a/packages/prom-client/package.json +++ b/packages/prom-client/package.json @@ -3,6 +3,6 @@ "version": "0.1.9", "main": "index.ts", "dependencies": { - "prom-client": "^15.1.2" + "prom-client": "^15.1.3" } } diff --git a/packages/ui-default/backendlib/builder.ts b/packages/ui-default/backendlib/builder.ts index 15d328852..f79137b0d 100644 --- a/packages/ui-default/backendlib/builder.ts +++ b/packages/ui-default/backendlib/builder.ts @@ -1,7 +1,7 @@ import esbuild from 'esbuild'; import { Context, fs, Handler, Logger, NotFoundError, param, SettingModel, sha1, - size, Types, UiContextBase, + size, SystemModel, Types, UiContextBase, } from 'hydrooj'; import { debounce } from 'lodash'; import { tmpdir } from 'os'; @@ -46,6 +46,7 @@ const build = async (contents: string) => { format: 'iife' as 'iife', bundle: true, outdir: tmp, + sourcemap: SystemModel.get('ui-default.nosourcemap') ? false : 'external', splitting: false, write: false, target: ['chrome65'], @@ -103,11 +104,11 @@ export async function buildUI() { } const entry = await build([ `window.lazyloadMetadata = ${JSON.stringify(hashes)};`, + `window.LANGS=${JSON.stringify(SettingModel.langs)};`, ...entryPoints.map((i) => `import '${relative(tmp, i).replace(/\\/g, '\\\\')}';`), ].join('\n')); - const pages = entry.outputFiles.map((i) => i.text); - const str = `window.LANGS=${JSON.stringify(SettingModel.langs)};${pages.join('\n')}`; - addFile('entry.js', str); + const pages = entry.outputFiles.filter((i) => i.path.endsWith('.js')).map((i) => i.text); + addFile('entry.js', `window._hydroLoad=()=>{ ${pages.join('\n')} };`); UiContextBase.constantVersion = hashes['entry.js']; for (const key in vfs) { if (newFiles.includes(key)) continue; diff --git a/packages/ui-default/backendlib/markdown-it-imsize.ts b/packages/ui-default/backendlib/markdown-it-imsize.ts new file mode 100644 index 000000000..5a43ab588 --- /dev/null +++ b/packages/ui-default/backendlib/markdown-it-imsize.ts @@ -0,0 +1,235 @@ +// Process ![test]( x =100x200) +// ^^^^^^^^ this size specification +function parseNextNumber(str, pos, max) { + const start = pos; + const result = { + ok: false, + pos, + value: '', + }; + let code = str.charCodeAt(pos); + while (pos < max && (code >= 0x30 /* 0 */ && code <= 0x39 /* 9 */) || code === 0x25 /* % */) { + code = str.charCodeAt(++pos); + } + result.ok = true; + result.pos = pos; + result.value = str.slice(start, pos); + return result; +} + +function parseImageSize(str, pos, max) { + const result = { + ok: false, + pos: 0, + width: '', + height: '', + }; + if (pos >= max) { return result; } + let code = str.charCodeAt(pos); + if (code !== 0x3d /* = */) { return result; } + pos++; + code = str.charCodeAt(pos); + if (code !== 0x78 /* x */ && (code < 0x30 || code > 0x39) /* [0-9] */) { + return result; + } + // parse width + const resultW = parseNextNumber(str, pos, max); + pos = resultW.pos; + // next charactor must be 'x' + code = str.charCodeAt(pos); + if (code !== 0x78 /* x */) { return result; } + pos++; + // parse height + const resultH = parseNextNumber(str, pos, max); + pos = resultH.pos; + result.width = resultW.value; + result.height = resultH.value; + result.pos = pos; + result.ok = true; + return result; +} + +export default function plugin(md) { + md.inline.ruler.before('emphasis', 'image', (state, silent) => { + let attrs; + let code; + let label; + let pos; + let ref; + let res; + let title; + let width = ''; + let height = ''; + let token; + let tokens; + let start; + let href = ''; + const oldPos = state.pos; + const max = state.posMax; + + if (state.src.charCodeAt(state.pos) !== 0x21/* ! */) { return false; } + if (state.src.charCodeAt(state.pos + 1) !== 0x5B/* [ */) { return false; } + + const labelStart = state.pos + 2; + const labelEnd = md.helpers.parseLinkLabel(state, state.pos + 1, false); + + // parser failed to find ']', so it's not a valid link + if (labelEnd < 0) { return false; } + + pos = labelEnd + 1; + if (pos < max && state.src.charCodeAt(pos) === 0x28/* ( */) { + // + // Inline link + // + + // [link]( "title" ) + // ^^ skipping these spaces + pos++; + for (; pos < max; pos++) { + code = state.src.charCodeAt(pos); + if (code !== 0x20 && code !== 0x0A) { break; } + } + if (pos >= max) { return false; } + + // [link]( "title" ) + // ^^^^^^ parsing link destination + start = pos; + res = md.helpers.parseLinkDestination(state.src, pos, state.posMax); + if (res.ok) { + href = state.md.normalizeLink(res.str); + if (state.md.validateLink(href)) { + pos = res.pos; + } else { + href = ''; + } + } + + // [link]( "title" ) + // ^^ skipping these spaces + start = pos; + for (; pos < max; pos++) { + code = state.src.charCodeAt(pos); + if (code !== 0x20 && code !== 0x0A) { break; } + } + + // [link]( "title" ) + // ^^^^^^^ parsing link title + res = md.helpers.parseLinkTitle(state.src, pos, state.posMax); + if (pos < max && start !== pos && res.ok) { + title = res.str; + pos = res.pos; + + // [link]( "title" ) + // ^^ skipping these spaces + for (; pos < max; pos++) { + code = state.src.charCodeAt(pos); + if (code !== 0x20 && code !== 0x0A) { break; } + } + } else { + title = ''; + } + + // [link]( "title" =WxH ) + // ^^^^ parsing image size + if (pos - 1 >= 0) { + code = state.src.charCodeAt(pos - 1); + + // there must be at least one white spaces + // between previous field and the size + if (code === 0x20) { + res = parseImageSize(state.src, pos, state.posMax); + if (res.ok) { + width = res.width; + height = res.height; + pos = res.pos; + + // [link]( "title" =WxH ) + // ^^ skipping these spaces + for (; pos < max; pos++) { + code = state.src.charCodeAt(pos); + if (code !== 0x20 && code !== 0x0A) { break; } + } + } + } + } + + if (pos >= max || state.src.charCodeAt(pos) !== 0x29/* ) */) { + state.pos = oldPos; + return false; + } + pos++; + } else { + // + // Link reference + // + if (typeof state.env.references === 'undefined') { return false; } + + // [foo] [bar] + // ^^ optional whitespace (can include newlines) + for (; pos < max; pos++) { + code = state.src.charCodeAt(pos); + if (code !== 0x20 && code !== 0x0A) { break; } + } + + if (pos < max && state.src.charCodeAt(pos) === 0x5B/* [ */) { + start = pos + 1; + pos = md.helpers.parseLinkLabel(state, pos); + if (pos >= 0) { + label = state.src.slice(start, pos++); + } else { + pos = labelEnd + 1; + } + } else { + pos = labelEnd + 1; + } + + // covers label === '' and label === undefined + // (collapsed reference link and shortcut reference link respectively) + label ||= state.src.slice(labelStart, labelEnd); + + ref = state.env.references[md.utils.normalizeReference(label)]; + if (!ref) { + state.pos = oldPos; + return false; + } + href = ref.href; + title = ref.title; + } + + // + // We found the end of the link, and know for a fact it's a valid link; + // so all that's left to do is to call tokenizer. + // + if (!silent) { + state.pos = labelStart; + state.posMax = labelEnd; + + const newState = new state.md.inline.State( + state.src.slice(labelStart, labelEnd), + state.md, + state.env, + tokens = [], + ); + newState.md.inline.tokenize(newState); + token = state.push('image', 'img', 0); + token.attrs = attrs = [['src', href], + ['alt', '']]; + token.children = tokens; + if (title) { + attrs.push(['title', title]); + } + + if (width !== '') { + attrs.push(['width', width]); + } + + if (height !== '') { + attrs.push(['height', height]); + } + } + + state.pos = pos; + state.posMax = max; + return true; + }); +}; diff --git a/packages/ui-default/backendlib/markdown-it-katex.js b/packages/ui-default/backendlib/markdown-it-katex.ts similarity index 95% rename from packages/ui-default/backendlib/markdown-it-katex.js rename to packages/ui-default/backendlib/markdown-it-katex.ts index 20a2cc8fc..cd49f5e9a 100644 --- a/packages/ui-default/backendlib/markdown-it-katex.js +++ b/packages/ui-default/backendlib/markdown-it-katex.ts @@ -1,7 +1,8 @@ // https://github.com/waylonflinn/markdown-it-katex/blob/master/index.js -const katex = require('katex'); +import { Logger } from 'hydrooj'; +import katex from 'katex'; -const logger = new global.Hydro.Logger('katex'); +const logger = new Logger('katex'); function isValidDelim(state, pos) { const max = state.posMax; @@ -109,8 +110,8 @@ function block(state, start, end, silent) { return true; } -module.exports = function plugin(md) { - const options = { throwOnError: false, strict: 'ignore' }; +export default function plugin(md) { + const options = { throwOnError: false, strict: 'ignore', displayMode: false }; const render = (latex: string, displayMode = false) => { options.displayMode = displayMode; try { @@ -136,4 +137,4 @@ module.exports = function plugin(md) { }); md.renderer.rules.math_inline = inlineRenderer; md.renderer.rules.math_block = blockRenderer; -}; +} diff --git a/packages/ui-default/backendlib/markdown-it-media.ts b/packages/ui-default/backendlib/markdown-it-media.ts index ce990fef7..fcd8be461 100644 --- a/packages/ui-default/backendlib/markdown-it-media.ts +++ b/packages/ui-default/backendlib/markdown-it-media.ts @@ -1,8 +1,8 @@ /* eslint-disable max-len */ /* eslint-disable prefer-destructuring */ -import { randomUUID } from 'crypto'; -import MarkdownIt from 'markdown-it'; +import type MarkdownIt from 'markdown-it'; +import { v4 as uuid } from 'uuid'; const allowFullScreen = ' webkitallowfullscreen mozallowfullscreen allowfullscreen'; @@ -110,7 +110,7 @@ export function Media(md: MarkdownIt) { if (service === 'pdf') { if (src.startsWith('file://') || src.startsWith('./')) src += src.includes('?') ? '&noDisposition=1' : '?noDisposition=1'; return `\ - + diff --git a/packages/ui-default/backendlib/markdown.js b/packages/ui-default/backendlib/markdown.js index 8d03c4b0d..ad675fa9a 100644 --- a/packages/ui-default/backendlib/markdown.js +++ b/packages/ui-default/backendlib/markdown.js @@ -23,7 +23,7 @@ const MergeCells = require('markdown-it-merge-cells'); // @[youtube](https://youtube.com/watch?v=xxx) // @[pdf](https://foo.com/bar.pdf) const { Media } = require('./markdown-it-media'); -const Katex = require('./markdown-it-katex'); +const { default: Katex } = require('./markdown-it-katex'); const { xssProtector } = require('./markdown-it-xss'); class Markdown extends MarkdownIt { diff --git a/packages/ui-default/build/config/webpack.ts b/packages/ui-default/build/config/webpack.ts index 0e7d38fe3..b74a2e248 100644 --- a/packages/ui-default/build/config/webpack.ts +++ b/packages/ui-default/build/config/webpack.ts @@ -2,17 +2,42 @@ import { sentryWebpackPlugin } from '@sentry/webpack-plugin'; import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import CopyWebpackPlugin from 'copy-webpack-plugin'; +import { version as coreJsVersion } from 'core-js/package.json'; +import compat from 'core-js-compat'; import { EsbuildPlugin } from 'esbuild-loader'; +import fs from 'fs'; import { DuplicatesPlugin } from 'inspectpack/plugin'; import ExtractCssPlugin from 'mini-css-extract-plugin'; +import packageJson from 'package-json'; import { dirname } from 'path'; +import { gt } from 'semver'; import webpack from 'webpack'; import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer'; import { WebpackManifestPlugin } from 'webpack-manifest-plugin'; import WebpackBar from 'webpackbar'; +import { version } from '../../package.json'; import root from '../utils/root'; -export default function (env: { watch?: boolean, production?: boolean, measure?: boolean } = {}) { +const { + list, + targets, +} = compat({ + targets: '> 1%, chrome 70, firefox 60, safari 16, ios_saf 16, not ie 11, not op_mini all', + modules: [ + 'core-js/stable', + ], + exclude: [], + version: coreJsVersion, + inverse: false, +}); +fs.writeFileSync(root('__core-js.js'), `${list.map((i) => `import 'core-js/modules/${i}';`).join('\n')}\n`); + +export default async function (env: { watch?: boolean, production?: boolean, measure?: boolean } = {}) { + if (env.production) console.log(targets); + let isNew = false; + const { version: latest } = await packageJson('@hydrooj/ui-default', { version: 'latest' }); + if (typeof version === 'string' && gt(version, latest)) isNew = true; + function cssLoader() { return { loader: 'css-loader', @@ -61,11 +86,9 @@ export default function (env: { watch?: boolean, production?: boolean, measure?: // sentry requires source-map while keep it simple in dev mode devtool: env.production ? 'source-map' : false, entry: { - hydro: './entry.js', - polyfill: './polyfill.ts', + [`hydro-${version}`]: './entry.js', 'default.theme': './theme/default.js', 'service-worker': './service-worker.ts', - 'messages-shared-worker': './components/message/worker.ts', }, cache: { type: 'filesystem', @@ -78,6 +101,7 @@ export default function (env: { watch?: boolean, production?: boolean, measure?: output: { path: root('public'), publicPath: '/', // overwrite in entry.js + workerPublicPath: '/', hashFunction: 'sha1', hashDigest: 'hex', hashDigestLength: 10, @@ -233,18 +257,30 @@ export default function (env: { watch?: boolean, production?: boolean, measure?: filename: '[name].css?[fullhash:6]', }), new WebpackManifestPlugin({}), - new webpack.IgnorePlugin({ resourceRegExp: /(^\.\/locale$|mathjax|abcjs|vditor.+\.d\.ts)/ }), + new webpack.IgnorePlugin({ resourceRegExp: /(^\.\/locale$)/ }), new CopyWebpackPlugin({ patterns: [ { from: root('static') }, { from: root('components/navigation/nav-logo-small_dark.png'), to: 'components/navigation/nav-logo-small_dark.png' }, { from: root(`${dirname(require.resolve('streamsaver/package.json'))}/mitm.html`), to: 'streamsaver/mitm.html' }, { from: root(`${dirname(require.resolve('streamsaver/package.json'))}/sw.js`), to: 'streamsaver/sw.js' }, - { from: root(`${dirname(require.resolve('vditor/package.json'))}/dist`), to: 'vditor/dist' }, { from: root(`${dirname(require.resolve('graphiql/package.json'))}/graphiql.min.css`), to: 'graphiql.min.css' }, { from: `${dirname(require.resolve('monaco-themes/package.json'))}/themes`, to: 'monaco/themes/' }, ], }), + sentryWebpackPlugin({ + authToken: process.env.SENTRY_AUTH_TOKEN, + org: 'hydro-dev', + project: 'hydro-web', + url: 'https://sentry.hydro.ac', + sourcemaps: { + rewriteSources: (source) => source.replace('@hydrooj/ui-default/../../node_modules/', ''), + }, + release: (process.env.CI && isNew) ? { + name: `hydro-web@${version}`, + uploadLegacySourcemaps: root('public'), + } : {}, + }), new webpack.DefinePlugin({ 'process.env.VERSION': JSON.stringify(require('@hydrooj/ui-default/package.json').version), }), @@ -265,6 +301,7 @@ export default function (env: { watch?: boolean, production?: boolean, measure?: url: 'https://sentry.hydro.ac', }), ] : []), + new webpack.NormalModuleReplacementPlugin(/core-js\/stable/, root('__core-js.js')), ...env.measure ? [ new BundleAnalyzerPlugin({ analyzerPort: 'auto' }), new DuplicatesPlugin(), diff --git a/packages/ui-default/build/main.ts b/packages/ui-default/build/main.ts index 0c9fb5ffa..d0c251daa 100644 --- a/packages/ui-default/build/main.ts +++ b/packages/ui-default/build/main.ts @@ -19,7 +19,7 @@ const argv = cac().parse(); async function runWebpack({ watch, production, measure, dev, https, }) { - const compiler = webpack(webpackConfig({ watch, production, measure })); + const compiler = webpack(await webpackConfig({ watch, production, measure })); if (dev) { const server = new WebpackDevServer({ port: https ? 8001 : 8000, @@ -49,7 +49,10 @@ async function runWebpack({ return; } if (argv.options.detail) console.log(stats.toString()); - if (!watch && (!stats || stats.hasErrors())) process.exitCode = 1; + if (!watch && (!stats || stats.hasErrors())) { + if (!argv.options.detail) console.log(stats.toString()); + process.exitCode = 1; + } resolve(stats); } if (watch) compiler.watch({}, compilerCallback); @@ -121,22 +124,9 @@ async function main() { if (argv.options.gulp) await runGulp(); else { await runWebpack(argv.options as any); - if (fs.existsSync('public/hydro.js')) { - fs.copyFileSync('public/hydro.js', `public/hydro-${pkg.version}.js`); - } - if (fs.existsSync('public/polyfill.js')) { - fs.copyFileSync('public/polyfill.js', `public/polyfill-${pkg.version}.js`); - } if (fs.existsSync('public/theme.css')) { fs.copyFileSync('public/theme.css', `public/theme-${pkg.version}.css`); } - if (argv.options.production) { - for (const f of ['echarts', 'graphviz', 'mermaid', 'mathjax']) { - fs.removeSync(`public/vditor/dist/js/${f}`); - } - const files = fs.readdirSync('public'); - files.filter((i) => /(^[in]\..+|worker)\.js\.map$/.test(i)).forEach((i) => fs.removeSync(`public/${i}`)); - } await Promise.all(globbySync('public/**/*.map').map((i) => fs.remove(i))); } process.chdir(dir); diff --git a/packages/ui-default/components/autocomplete/CustomSelectAutoComplete.tsx b/packages/ui-default/components/autocomplete/CustomSelectAutoComplete.tsx index c8f3c82a3..760850fa9 100644 --- a/packages/ui-default/components/autocomplete/CustomSelectAutoComplete.tsx +++ b/packages/ui-default/components/autocomplete/CustomSelectAutoComplete.tsx @@ -24,9 +24,8 @@ const Component = React.forwardRef((props, ref) => { ); }); -export default class CustomSelectAutoComplete extends AutoComplete { +export default class CustomSelectAutoComplete extends AutoComplete { static DOMAttachKey = 'ucwCustomSelectAutoCompleteInstance'; - options: CustomSelectOptions & AutoCompleteOptions; constructor($dom, options: CustomSelectOptions & AutoCompleteOptions) { super($dom, { diff --git a/packages/ui-default/components/autocomplete/FileSelectAutoComplete.tsx b/packages/ui-default/components/autocomplete/FileSelectAutoComplete.tsx index e128c9e6b..a30207072 100644 --- a/packages/ui-default/components/autocomplete/FileSelectAutoComplete.tsx +++ b/packages/ui-default/components/autocomplete/FileSelectAutoComplete.tsx @@ -27,9 +27,8 @@ const Component = React.forwardRef((props, ref) => { ); }); -export default class FileSelectAutoComplete extends AutoComplete { +export default class FileSelectAutoComplete extends AutoComplete { static DOMAttachKey = 'ucwFileSelectAutoCompleteInstance'; - options: FileSelectOptions & AutoCompleteOptions; constructor($dom, options: FileSelectOptions & AutoCompleteOptions) { super($dom, { diff --git a/packages/ui-default/components/autocomplete/components/CustomSelectAutoComplete.tsx b/packages/ui-default/components/autocomplete/components/CustomSelectAutoComplete.tsx index 9afc9b171..c9322d3af 100644 --- a/packages/ui-default/components/autocomplete/components/CustomSelectAutoComplete.tsx +++ b/packages/ui-default/components/autocomplete/components/CustomSelectAutoComplete.tsx @@ -11,10 +11,7 @@ const CustomSelectAutoComplete = React.forwardRef((props, ref) => ( ref={ref as any} fetchItems={(keys) => props.data.filter((i) => (i._id ? keys.includes(i._id) : keys.includes(i)))} - queryItems={(query) => props.data.filter((i) => (i.name - ? i.name.toLowerCase().match(query.toLowerCase()) - : i.toString().toLowerCase().match(query.toLowerCase()) - ))} + queryItems={(query) => props.data.filter((i) => (i.name || i).toString().toLowerCase().includes(query.toLowerCase()))} itemText={(item) => `${item.name || item}`} itemKey={(item) => `${item._id?.toString() || item.name || item}`} renderItem={(item) => `${item.name || item}`} diff --git a/packages/ui-default/components/autocomplete/index.tsx b/packages/ui-default/components/autocomplete/index.tsx index 2c603c9f0..52f0a6a01 100644 --- a/packages/ui-default/components/autocomplete/index.tsx +++ b/packages/ui-default/components/autocomplete/index.tsx @@ -8,6 +8,7 @@ export interface AutoCompleteOptions { defaultItems?: string; width?: string; height?: string; + classes?: string; listStyle?: any; allowEmptyQuery?: boolean; freeSolo?: boolean; @@ -18,17 +19,17 @@ export interface AutoCompleteOptions { text?: () => string; } -export default class AutoComplete extends DOMAttachedObject { +export default class AutoComplete = {}> extends DOMAttachedObject { static DOMAttachKey = 'ucwAutoCompleteInstance'; ref = null; container = document.createElement('div'); - options: AutoCompleteOptions; + options: AutoCompleteOptions & Options; component = ReactDOM.createRoot(this.container); changeListener = [ (val) => this.$dom.val(val), ]; - constructor($dom, options = {}) { + constructor($dom, options = {} as Options) { super($dom); this.options = { items: async () => [], diff --git a/packages/ui-default/components/dropdown/dropdown.page.styl b/packages/ui-default/components/dropdown/dropdown.page.styl index 03ecb8ad3..2af5705b9 100644 --- a/packages/ui-default/components/dropdown/dropdown.page.styl +++ b/packages/ui-default/components/dropdown/dropdown.page.styl @@ -6,9 +6,13 @@ .drop.nav__dropdown max-height: calc(100% - 60px) + overflow-x: hidden overflow-y: auto box-shadow: 0rem 0.125rem 0.4375rem rgba(0 0 0 , 30%); +.drop.nav__dropdown.drop-open + overflow-x: auto + .dropdown .menu min-width: rem(150px) diff --git a/packages/ui-default/components/editor/cmeditor.styl b/packages/ui-default/components/editor/cmeditor.styl index c5237498b..f104bc1dc 100644 --- a/packages/ui-default/components/editor/cmeditor.styl +++ b/packages/ui-default/components/editor/cmeditor.styl @@ -1,12 +1,3 @@ -.vditor - margin-bottom: rem(10px) - -.vditor-preview__action - display: none - -.vditor ol - list-style: decimal - .decoration-hide color: grey !important background-color: grey !important diff --git a/packages/ui-default/components/editor/index.ts b/packages/ui-default/components/editor/index.tsx similarity index 59% rename from packages/ui-default/components/editor/index.ts rename to packages/ui-default/components/editor/index.tsx index 67263759e..e4b2d8ce7 100644 --- a/packages/ui-default/components/editor/index.ts +++ b/packages/ui-default/components/editor/index.tsx @@ -1,36 +1,11 @@ import $ from 'jquery'; import type { editor } from 'monaco-editor'; import { nanoid } from 'nanoid'; +import React from 'react'; +import ReactDOM from 'react-dom/client'; import DOMAttachedObject from 'vj/components/DOMAttachedObject'; -import Notification from 'vj/components/notification'; -import { i18n, request } from 'vj/utils'; - -export const config = { - toolbar: [ - 'emoji', 'headings', 'bold', 'italic', 'strike', 'link', '|', - 'list', 'ordered-list', 'check', 'outdent', 'indent', '|', - 'quote', 'line', 'code', 'inline-code', 'table', '|', - 'upload', 'edit-mode', 'fullscreen', 'export', - ], - mode: UserContext.preferredEditorType || 'ir', - toolbarConfig: { - pin: true, - }, - cdn: `${UiContext.cdn_prefix}vditor`, - counter: { - enable: true, - max: 65536, - }, - preview: { - math: { - inlineDigit: true, - }, - theme: { - path: `${UiContext.cdn_prefix}vditor/dist/css/content-theme`, - current: 'light', - }, - }, -}; +import { i18n } from 'vj/utils'; +import uploadFiles from '../upload'; interface MonacoOptions { language?: string; @@ -43,23 +18,23 @@ interface MonacoOptions { hide?: string[]; lineNumbers?: 'on' | 'off' | 'relative' | 'interval'; } -interface VditorOptions { - theme?: 'classic' | 'dark' -} -type Options = MonacoOptions & VditorOptions; +type Options = MonacoOptions; export default class Editor extends DOMAttachedObject { static DOMAttachKey = 'vjEditorInstance'; model: editor.IModel; editor: editor.IStandaloneCodeEditor; - vditor: import('vditor').default; + markdownEditor: any; + valueCache?: string; + setMarkdownEditorValue?: (v: string) => void; + reactRoot?: ReactDOM.Root; isValid: boolean; constructor($dom, public options: Options = {}) { super($dom); if (UserContext.preferredEditorType === 'monaco') this.initMonaco(); else if (options.language && options.language !== 'markdown') this.initMonaco(); - else this.initVditor(); + else this.initMarkdownEditor(); } async initMonaco() { @@ -169,87 +144,102 @@ export default class Editor extends DOMAttachedObject { window.editor = this.editor; } - async initVditor() { + async initMarkdownEditor() { const pagename = document.documentElement.getAttribute('data-page'); const isProblemPage = ['problem_create', 'problem_edit'].includes(pagename); const isProblemEdit = pagename === 'problem_edit'; - const { default: Vditor } = await import('vditor'); + const that = this; const { $dom } = this; const hasFocus = $dom.is(':focus') || $dom.hasClass('autofocus'); const origin = $dom.get(0); const ele = document.createElement('div'); const value = $dom.val(); const { onChange } = this.options; - await new Promise((resolve) => { - this.vditor = new Vditor(ele, { - ...config, - upload: { - multiple: false, - handler: (files) => { - let wrapper = ['', '']; - let ext: string; - const matches = files[0].type.match(/^image\/(png|jpg|jpeg|gif)$/i); - if (matches) { - wrapper = ['![image](', ')']; - [, ext] = matches; - } else if (files[0].type === 'application/x-zip-compressed') { - wrapper = ['[file](', ')']; - ext = 'zip'; - } - if (!ext) return i18n('No Supported file type.'); - const filename = `${nanoid()}.${ext}`; - const data = new FormData(); - data.append('filename', filename); - data.append('file', files[0]); - data.append('operation', 'upload_file'); - if (isProblemEdit) data.append('type', 'additional_file'); - let progress = 0; - request.postFile(isProblemEdit ? './files' : '/file', data, { - xhr: () => { - const xhr = new XMLHttpRequest(); - xhr.upload.addEventListener('loadstart', () => this.vditor.vditor.tip.show(i18n('Uploading...'), 0)); - xhr.upload.addEventListener('progress', (e) => { - if (!e.lengthComputable) return; - const percentComplete = Math.round((e.loaded / e.total) * 100); - if (percentComplete === progress) return; - progress = percentComplete; - this.vditor.vditor.tip.show(`${i18n('Uploading...')} ${percentComplete}%`, 0); - }, false); - return xhr; - }, - }) - .then(() => { - this.vditor.insertValue(`${wrapper.join(`${isProblemPage ? 'file://' : `/file/${UserContext._id}/`}${filename}`)} `); - this.vditor.vditor.tip.hide(); - }) - .catch((e: { message: string; }) => { - console.error(e); - return `${i18n('Upload Failed')}: ${e.message}`; - }); - return null; - }, - }, - ...this.options, - after: () => { - const pos = $(ele).find('button[data-mode="sv"]').get(); - const button = $(''); - button.on('click', (e) => { - Notification.info('You can select this in PreferenceSettings'); - e.preventDefault(); - }); - button.insertAfter(pos); - resolve(null); - }, - input(v) { + const { MdEditor } = await import('./mdeditor'); + + const renderCallback = (ref) => { + this.markdownEditor = ref; + }; + + function EditorComponent() { + const [val, setVal] = React.useState(value); + that.setMarkdownEditorValue = setVal; + return { + that.valueCache = v; + setVal(v); $dom.val(v); $dom.text(v); - if (onChange) onChange(v); - }, - value, - cache: { id: Math.random().toString() }, - }); - }); - $(ele).addClass('textbox'); + onChange?.(v); + }} + onUploadImg={async (files, callback) => { + let ext: string; + const matches = files[0].type.match(/^image\/(png|jpg|jpeg|gif)$/i); + if (matches) { + [, ext] = matches; + } else if (files[0].type === 'application/x-zip-compressed') { + ext = 'zip'; + } + if (!ext) return i18n('No Supported file type.'); + const filename = `${nanoid()}.${ext}`; + await uploadFiles(isProblemEdit ? './files' : '/file', [files[0]], { + type: isProblemEdit ? 'additional_file' : undefined, + filenameCallback: () => filename, + }).then(() => { + callback([`${isProblemPage ? 'file://' : `/file/${UserContext._id}/`}${filename}`]); + }).catch(() => { + callback([]); + }); + return null; + }} + />; + } + + this.reactRoot = ReactDOM.createRoot(ele); + this.reactRoot.render(); $dom.hide(); origin.parentElement.appendChild(ele); this.isValid = true; @@ -258,7 +248,7 @@ export default class Editor extends DOMAttachedObject { destroy() { this.detach(); - if (this.vditor?.destroy) this.vditor.destroy(); + if (this.reactRoot) this.reactRoot.unmount(); else if (this.editor?.dispose) this.editor.dispose(); } @@ -268,13 +258,17 @@ export default class Editor extends DOMAttachedObject { value(val?: string) { this.ensureValid(); - if (typeof val === 'string') return (this.editor || this.vditor).setValue(val); - return (this.editor || this.vditor).getValue({ lineEnding: '\n', preserveBOM: false }); + if (typeof val === 'string') { + if (this.editor) return this.editor.setValue(val); + this.setMarkdownEditorValue?.(val); + this.markdownEditor?.resetHistory?.(); + } + if (this.editor) return this.editor.getValue({ lineEnding: '\n', preserveBOM: false }); + return this.valueCache; } focus() { this.ensureValid(); - this.vditor?.focus(); if (!this.editor || !this.model) return; this.editor.focus(); const range = this.model.getFullModelRange(); diff --git a/packages/ui-default/components/editor/mdeditor.ts b/packages/ui-default/components/editor/mdeditor.ts new file mode 100644 index 000000000..62e6ef4ac --- /dev/null +++ b/packages/ui-default/components/editor/mdeditor.ts @@ -0,0 +1,39 @@ +import 'md-editor-rt/lib/style.css'; + +import Anchor from 'markdown-it-anchor'; +import Footnote from 'markdown-it-footnote'; +import Mark from 'markdown-it-mark'; +import MergeCells from 'markdown-it-merge-cells'; +import TOC from 'markdown-it-table-of-contents'; +import { config, MdEditor } from 'md-editor-rt'; +import Imsize from '../../backendlib/markdown-it-imsize'; +import { Media } from '../../backendlib/markdown-it-media'; +import { xssProtector } from '../../backendlib/markdown-it-xss'; + +const pagename = document.documentElement.getAttribute('data-page'); +const isProblemPage = ['problem_create', 'problem_edit'].includes(pagename); + +config({ + markdownItConfig(mdit) { + mdit.options.html = true; + mdit.options.linkify = true; + mdit.linkify.tlds('.py', false); + mdit.linkify.tlds('.zip', false); + mdit.linkify.tlds('.mov', false); + mdit.use(Media); + mdit.use(Footnote); + mdit.use(Mark); + mdit.use(Imsize); + mdit.use(Anchor); + mdit.use(TOC); + mdit.use(MergeCells); + mdit.use(xssProtector); + if (isProblemPage) { + mdit.core.ruler.before('normalize', 'xss', (state) => { + state.src = state.src.replace(/file:\/\//g, pagename === 'problem_create' ? `/file/${UserContext._id}/` : './file/'); + }); + } + }, +}); + +export { MdEditor }; diff --git a/packages/ui-default/components/message/index.page.ts b/packages/ui-default/components/message/index.page.ts index 2707b3737..2c45b18a9 100644 --- a/packages/ui-default/components/message/index.page.ts +++ b/packages/ui-default/components/message/index.page.ts @@ -1,5 +1,4 @@ import { nanoid } from 'nanoid'; -import ReconnectingWebsocket from 'reconnecting-websocket'; import { InfoDialog } from 'vj/components/dialog'; import VjNotification from 'vj/components/notification/index'; import { @@ -7,6 +6,7 @@ import { } from 'vj/constant/message'; import { AutoloadPage } from 'vj/misc/Page'; import { i18n, tpl } from 'vj/utils'; +import Sock from '../socket'; let previous: VjNotification; const onmessage = (msg) => { @@ -61,7 +61,7 @@ const onmessage = (msg) => { const initWorkerMode = (endpoint) => { console.log('Messages: using SharedWorker'); - const worker = new SharedWorker('/messages-shared-worker.js', { name: 'HydroMessagesWorker' }); + const worker = new SharedWorker(new URL('./worker.ts', import.meta.url), { name: 'HydroMessagesWorker' }); worker.port.start(); window.addEventListener('beforeunload', () => { worker.port.postMessage({ type: 'unload' }); @@ -128,12 +128,8 @@ const messagePage = new AutoloadPage('messagePage', (pagename) => { isMaster = true; localStorage.setItem('page.master', selfId); const masterChannel = new BroadcastChannel('hydro-messages'); - const sock = new ReconnectingWebsocket(endpoint); - sock.onopen = () => console.log('Connected'); - sock.onerror = console.error; - sock.onclose = (...args) => console.log('Closed', ...args); + const sock = new Sock(endpoint); sock.onmessage = async (message) => { - if (process.env.NODE_ENV !== 'production') console.log('onmessage: ', message); const payload = JSON.parse(message.data); masterChannel.postMessage({ type: 'message', payload }); }; diff --git a/packages/ui-default/components/message/worker.ts b/packages/ui-default/components/message/worker.ts index 4e186061b..28d722b75 100644 --- a/packages/ui-default/components/message/worker.ts +++ b/packages/ui-default/components/message/worker.ts @@ -31,12 +31,17 @@ function broadcastMsg(message: any) { function initConn(path: string, port: MessagePort, cookie: any) { if (cookie !== lcookie) conn?.close(); else if (conn && conn.readyState === conn.OPEN) return; + const url = new URL(path, location.origin); + if (cookie) url.searchParams.set('sid', cookie.split('sid=')[1].split(';')[0]); lcookie = cookie; console.log('Init connection for', path); conn?.close(); - conn = new ReconnectingWebsocket(path); + conn = new ReconnectingWebsocket(url.toString()); ports.push(port); - conn.onopen = () => conn.send(cookie); + conn.onopen = () => { + console.log('Connected to', path); + broadcastMsg({ type: 'open' }); + }; conn.onerror = () => broadcastMsg({ type: 'error' }); conn.onclose = (ev) => broadcastMsg({ type: 'close', error: ev.reason }); conn.onmessage = (message) => { @@ -46,14 +51,9 @@ function initConn(path: string, port: MessagePort, cookie: any) { return; } const payload = JSON.parse(message.data); - if (payload.event === 'auth') { - if (['PermissionError', 'PrivilegeError'].includes(payload.error)) { - broadcastMsg({ type: 'close', error: payload.error }); - conn.close(); - } else { - console.log('Connected to', path); - broadcastMsg({ type: 'open' }); - } + if (['PermissionError', 'PrivilegeError'].includes(payload.error)) { + broadcastMsg({ type: 'close', error: payload.error }); + conn.close(); } else { broadcastMsg({ type: 'message', payload }); let acked = false; diff --git a/packages/ui-default/components/problemconfig/ProblemConfigEditor.tsx b/packages/ui-default/components/problemconfig/ProblemConfigEditor.tsx index d381d605b..e08e18c2d 100644 --- a/packages/ui-default/components/problemconfig/ProblemConfigEditor.tsx +++ b/packages/ui-default/components/problemconfig/ProblemConfigEditor.tsx @@ -87,8 +87,8 @@ export function configYamlFormat(config: ProblemConfigFile) { export default connect(mapStateToProps, mapDispatchToProps)(class MonacoEditor extends React.PureComponent { disposable = []; containerElement: HTMLElement; - private __preventUpdate = false; - private __preventFormat = false; + __preventUpdate = false; + __preventFormat = false; editor: editor.IStandaloneCodeEditor; model: editor.ITextModel; diff --git a/packages/ui-default/components/scratchpad/reducers/editor.ts b/packages/ui-default/components/scratchpad/reducers/editor.ts index 41736b675..22ce18946 100644 --- a/packages/ui-default/components/scratchpad/reducers/editor.ts +++ b/packages/ui-default/components/scratchpad/reducers/editor.ts @@ -1,6 +1,7 @@ let cacheKey = `${UserContext._id}/${UiContext.pdoc.domainId}/${UiContext.pdoc.docId}`; if (UiContext.tdoc?._id && UiContext.tdoc.rule !== 'homework') cacheKey += `@${UiContext.tdoc._id}`; +// TODO switch to indexeddb export default function reducer(state = { lang: localStorage.getItem(`${cacheKey}#lang`) || UiContext.codeLang, code: localStorage.getItem(cacheKey) || UiContext.codeTemplate, diff --git a/packages/ui-default/components/socket/index.ts b/packages/ui-default/components/socket/index.ts index b741602e8..981e78aa8 100644 --- a/packages/ui-default/components/socket/index.ts +++ b/packages/ui-default/components/socket/index.ts @@ -54,6 +54,10 @@ export default class Sock { onclose: (code: number, reason: string) => void; onopen: (sock: ReconnectingWebSocket) => void; + on(event: 'message' | 'close' | 'open', callback: (...args: any[]) => void) { + this[`on${event}`] = callback; + } + send(data) { this.sock.send(data); } diff --git a/packages/ui-default/components/tab/Tab.js b/packages/ui-default/components/tab/Tab.js index 64ab314cc..b2ba79de5 100644 --- a/packages/ui-default/components/tab/Tab.js +++ b/packages/ui-default/components/tab/Tab.js @@ -1,5 +1,4 @@ import $ from 'jquery'; -import _ from 'lodash'; import DOMAttachedObject from 'vj/components/DOMAttachedObject'; const TAB_TRANSITION_DURATION = 300; @@ -144,5 +143,3 @@ export default class Tab extends DOMAttachedObject { return true; } } - -_.assign(Tab, DOMAttachedObject); diff --git a/packages/ui-default/components/tab/tab.page.styl b/packages/ui-default/components/tab/tab.page.styl index 23d4de1ca..906e91e19 100644 --- a/packages/ui-default/components/tab/tab.page.styl +++ b/packages/ui-default/components/tab/tab.page.styl @@ -73,6 +73,26 @@ background: $secondary-color content: '' +.section__tab-container:not(.immersive) + .section__tab-header + padding-top: 0 !important + + .section__tab-header-item + border-top: 3px solid transparent; + transition: border-color 0.2s, background 0.2s, color 0.2s; + transition-timing-function: cubic-bezier(0.215, 0.61, 0.355, 1); + margin-right: 0 + + &:hover + border-color: #5f9fd6; + background: #f8f8f8; + + &.tab--active + border-color: #ed5f82; + + &.selected + border-color: #ed5f82; + .section__tab-content position: relative diff --git a/packages/ui-default/components/upload.ts b/packages/ui-default/components/upload.ts index 3be3ec434..d70855b13 100644 --- a/packages/ui-default/components/upload.ts +++ b/packages/ui-default/components/upload.ts @@ -13,6 +13,7 @@ interface UploadOptions { pjax?: boolean; sidebar?: boolean; singleFileUploadCallback?: (file: File) => any; + filenameCallback?: (file: File) => string; } export default async function uploadFiles(endpoint = '', files: File[] | FileList = [], options: UploadOptions = {}) { const dialog = new Dialog({ @@ -38,7 +39,7 @@ export default async function uploadFiles(endpoint = '', files: File[] | FileLis if (Number.isNaN(+i)) continue; const file = files[i]; const data = new FormData(); - data.append('filename', file.name); + data.append('filename', options.filenameCallback?.(file) || file.name); data.append('file', file); if (options.type) data.append('type', options.type); data.append('operation', 'upload_file'); diff --git a/packages/ui-default/entry.js b/packages/ui-default/entry.js index a9f838e58..dbd83224b 100644 --- a/packages/ui-default/entry.js +++ b/packages/ui-default/entry.js @@ -1,3 +1,5 @@ +import './polyfill'; + import * as Sentry from '@sentry/browser'; import $ from 'jquery'; @@ -29,17 +31,23 @@ console.log( window.UiContext = JSON.parse(window.UiContext); window.UserContext = JSON.parse(window.UserContext); if (process.env.NODE_ENV === 'production' && !UiContext.sentry_disable) { - window.captureException = (e) => Sentry.captureException(e); + window.captureException = (e) => { + if (!e.isUserFacingError) Sentry.captureException(e); + }; Sentry.init({ - dsn: UiContext.sentry_dsn || 'https://04d1d77b095231d55e1202bdc912ff16@sentry.hydro.ac/3', - release: process.env.VERSION, + dsn: UiContext.sentry_dsn || 'https://2f95d53751e08c74c1af1c4b93ccaff7@sentry.hydro.ac/2', + release: `hydro-web@${process.env.VERSION}`, integrations: [ Sentry.browserTracingIntegration(), - Sentry.replayIntegration(), + Sentry.browserApiErrorsIntegration(), + Sentry.replayIntegration({ + networkRequestHeaders: ['Content-Type'], + networkResponseHeaders: ['Content-Type', 'Location'], + }), ], tracesSampleRate: 0.1, tracePropagationTargets: ['localhost', /^\//, window.location.host], - replaysSessionSampleRate: 0.1, + replaysSessionSampleRate: 0.03, replaysOnErrorSampleRate: 1.0, }); } @@ -57,16 +65,10 @@ const PageLoader = '