From d7412fcc1e8426cddca6b67250b06c677a55d9bb Mon Sep 17 00:00:00 2001 From: undefined Date: Wed, 15 Sep 2021 17:28:57 +0800 Subject: [PATCH] core&ui: remove static manifest --- install/ubuntu-1604.sh | 1 - install/ubuntu-1804.sh | 1 - install/ubuntu-2004.sh | 1 - packages/hydrooj/package.json | 2 +- packages/hydrooj/src/entry/common.ts | 13 ----- packages/hydrooj/src/entry/worker.ts | 3 +- packages/hydrooj/src/loader.ts | 12 +++- packages/hydrooj/src/model/domain.ts | 4 +- packages/hydrooj/src/script/rating.ts | 55 ++++++++----------- packages/hydrooj/src/service/server.ts | 2 +- packages/hydrooj/src/upgrade.ts | 1 - packages/migrate-vijos/script.ts | 1 - packages/ui-default/backendlib/template.js | 3 - packages/ui-default/build/config/webpack.js | 8 --- packages/ui-default/build/main.js | 3 + .../build/plugins/manifest.webpack.js | 49 ----------------- packages/ui-default/handler.ts | 13 ++--- packages/ui-default/package.json | 2 +- 18 files changed, 50 insertions(+), 124 deletions(-) delete mode 100644 packages/ui-default/build/plugins/manifest.webpack.js diff --git a/install/ubuntu-1604.sh b/install/ubuntu-1604.sh index dd8049ca1..f72dc985c 100644 --- a/install/ubuntu-1604.sh +++ b/install/ubuntu-1604.sh @@ -64,7 +64,6 @@ echo 'Installing g++' apt-get install -y g++ >/dev/null # Install HydroOJ -# TODO: install basic addons? echo "Installing Hydro" yarn global add hydrooj @hydrooj/ui-default @hydrooj/hydrojudge wget https://s3.undefined.moe/file/executor-amd64 -O /usr/bin/sandbox diff --git a/install/ubuntu-1804.sh b/install/ubuntu-1804.sh index a24f9cdde..1d8bd39cd 100644 --- a/install/ubuntu-1804.sh +++ b/install/ubuntu-1804.sh @@ -64,7 +64,6 @@ echo 'Installing g++' apt-get install -y g++ >/dev/null # Install HydroOJ -# TODO: install basic addons? echo "Installing Hydro" yarn global add hydrooj @hydrooj/ui-default @hydrooj/hydrojudge wget https://s3.undefined.moe/file/executor-amd64 -O /usr/bin/sandbox diff --git a/install/ubuntu-2004.sh b/install/ubuntu-2004.sh index 8c574181d..3583be4ba 100644 --- a/install/ubuntu-2004.sh +++ b/install/ubuntu-2004.sh @@ -64,7 +64,6 @@ echo 'Installing g++' apt-get install -y g++ >/dev/null # Install HydroOJ -# TODO: install basic addons? echo "Installing Hydro" yarn global add hydrooj @hydrooj/ui-default @hydrooj/hydrojudge wget https://s3.undefined.moe/file/executor-amd64 -O /usr/bin/sandbox diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 5203860dc..abe9f5b24 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.34.9", + "version": "2.35.0", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index 26eeeb83e..caf416932 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -148,19 +148,6 @@ export async function template(pending: string[], fail: string[]) { await bus.serial('app/load/template'); } -export async function uistatic(pending: string[], fail: string[]) { - for (const i of pending) { - const p = path.resolve(i, 'public', 'static-manifest.json'); - if (fs.existsSync(p) && fs.statSync(p).isFile() && !fail.includes(i)) { - try { - Object.assign(global.Hydro.ui.manifest, eval('require')(p)); - } catch (e) { - fail.push(i); - } - } - } -} - export async function model(pending: string[], fail: string[]) { for (const i of pending) { let p = path.resolve(i, 'model.ts'); diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index c4cf7d542..03da48465 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -10,7 +10,7 @@ import * as bus from '../service/bus'; import db from '../service/db'; import { builtinHandler, builtinLib, builtinModel, - builtinScript, handler, lib, locale, model, script, service, setting, template, uistatic, + builtinScript, handler, lib, locale, model, script, service, setting, template, } from './common'; const logger = new Logger('worker'); @@ -34,7 +34,6 @@ export async function load() { await Promise.all([ locale(pending, fail), template(pending, fail), - uistatic(pending, fail), ]); if (detail) logger.info('finish: locale/template/static'); const opts = options(); diff --git a/packages/hydrooj/src/loader.ts b/packages/hydrooj/src/loader.ts index 2f679aae1..fd91ba164 100644 --- a/packages/hydrooj/src/loader.ts +++ b/packages/hydrooj/src/loader.ts @@ -30,7 +30,17 @@ export function addon(addonPath: string, prepend = false) { const packagejson = require.resolve(`${addonPath}/package.json`); const modulePath = path.dirname(packagejson); const publicPath = path.resolve(modulePath, 'public'); - if (fs.existsSync(publicPath)) global.publicDirs[prepend ? 'push' : 'unshift'](publicPath); + if (fs.existsSync(publicPath)) { + global.publicDirs[prepend ? 'push' : 'unshift'](publicPath); + const targets = fs.readdirSync(publicPath); + for (const target of targets) { + if (global.ui.manifest[target] && !prepend) { + global.ui.manifest[target] = publicPath; + } else if (!global.staticFiles[target]) { + global.ui.manifest[target] = publicPath; + } + } + } global.addons[prepend ? 'unshift' : 'push'](modulePath); } catch (e) { logger.error(`Addon not found: ${addonPath}`); diff --git a/packages/hydrooj/src/model/domain.ts b/packages/hydrooj/src/model/domain.ts index 7a0b77926..75cad97dc 100644 --- a/packages/hydrooj/src/model/domain.ts +++ b/packages/hydrooj/src/model/domain.ts @@ -103,7 +103,9 @@ class DomainModel { static async getList(domainIds: string[]) { const r: Record = {}; // eslint-disable-next-line no-await-in-loop - for (const domainId of domainIds) r[domainId] = await DomainModel.get(domainId); + const tasks = []; + for (const domainId of domainIds) tasks.push(DomainModel.get(domainId).then((ddoc) => { r[domainId] = ddoc; })); + await Promise.all(tasks); return r; } diff --git a/packages/hydrooj/src/script/rating.ts b/packages/hydrooj/src/script/rating.ts index 1f11ae8c4..67da1a134 100644 --- a/packages/hydrooj/src/script/rating.ts +++ b/packages/hydrooj/src/script/rating.ts @@ -23,25 +23,16 @@ async function runProblem(...arg: any[]) { ? await problem.get(arg[0], arg[1]) : arg[0]; const udict: ND = (typeof arg[0] === 'string') ? arg[2] : arg[1]; - const [, nPages] = await paginate( - problem.getMultiStatus( + const nPages = Math.floor( + (await problem.getMultiStatus( pdoc.domainId, { docId: pdoc.docId, rid: { $ne: null } }, - ), - 1, - 100, + ).count() + 99) / 100, ); - const nAccept = await problem.getMultiStatus( - pdoc.domainId, { docId: pdoc.docId, status: STATUS.STATUS_ACCEPTED }, - ).count(); - const p = (pdoc.difficulty || 5) / (Math.sqrt(Math.sqrt(nAccept)) + 1); + const p = (pdoc.difficulty || 5) / (Math.sqrt(Math.sqrt(pdoc.nAccept)) + 1); for (let page = 1; page <= nPages; page++) { - const [psdocs] = await paginate( - problem.getMultiStatus( - pdoc.domainId, { docId: pdoc.docId, rid: { $ne: null } }, - ), - page, - 100, - ) as any[][]; + const psdocs = await problem.getMultiStatus( + pdoc.domainId, { docId: pdoc.docId, rid: { $ne: null } }, + ).limit(100).skip((page - 1) * 100).project({ rid: 1 }).toArray(); const rdict = await record.getList(pdoc.domainId, psdocs.map((psdoc) => psdoc.rid), true); for (const psdoc of psdocs) { if (rdict[psdoc.rid.toHexString()]) { @@ -95,8 +86,10 @@ export async function calcLevel(domainId: string, report: Function) { const coll = db.collection('domain.user'); const ducur = domain.getMultiUserInDomain(domainId, filter).project({ rp: 1 }).sort({ rp: -1 }); let bulk = coll.initializeUnorderedBulkOp(); - while (await ducur.hasNext()) { + // eslint-disable-next-line no-constant-condition + while (true) { const dudoc = await ducur.next(); + if (!dudoc) break; if ([0, 1].includes(dudoc.uid)) continue; count++; if (!dudoc.rp) dudoc.rp = null; @@ -130,18 +123,6 @@ async function runInDomain(id: string, isSub: boolean, report: Function) { ...await domain.getMultiUserInDomain('', { domainId }).toArray(), ], (a, b) => a.uid === b.uid); for (const dudoc of dudocs) deltaudict[dudoc.uid] = dudoc.rpdelta; - const contests: Tdoc<30 | 60>[] = await contest.getMulti('', { domainId, rated: true }) - .toArray() as any; - await report({ message: `Found ${contests.length} contests in ${id}` }); - for (const i in contests) { - const tdoc = contests[i]; - await runContest(tdoc, udict, report); - if (!isSub) { - await report({ - progress: Math.floor(((parseInt(i, 10) + 1) / contests.length) * 100), - }); - } - } // TODO pagination const problems = await problem.getMulti('', { domainId, hidden: false }).toArray(); await report({ message: `Found ${problems.length} problems in ${id}` }); @@ -150,7 +131,19 @@ async function runInDomain(id: string, isSub: boolean, report: Function) { await runProblem(pdoc, udict); if (!isSub) { await report({ - progress: Math.floor(((parseInt(i, 10) + 1) / problems.length) * 100), + progress: Math.floor(((+i + 1) / problems.length) * 100), + }); + } + } + const contests: Tdoc<30 | 60>[] = await contest.getMulti('', { domainId, rated: true }) + .toArray() as any; + await report({ message: `Found ${contests.length} contests in ${id}` }); + for (const i in contests) { + const tdoc = contests[i]; + await runContest(tdoc, udict, report); + if (!isSub) { + await report({ + progress: Math.floor(((+i + 1) / contests.length) * 100), }); } } @@ -182,7 +175,7 @@ export async function run({ domainId }, report: Function) { memory: 0, score: 0, }, - progress: Math.floor(((parseInt(i, 10) + 1) / domains.length) * 100), + progress: Math.floor(((+i + 1) / domains.length) * 100), }); } } else await runInDomain(domainId, false, report); diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index b7429054c..ead01b042 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -309,7 +309,7 @@ export class HandlerCommon { } renderTitle(str: string) { - const name = this.domain.ui?.name || system.get('server.name'); + const name = this.domain?.ui?.name || system.get('server.name'); if (this.extraTitleContent) return `${this.translate(str)} - ${this.extraTitleContent} - ${name}`; return `${this.translate(str)} - ${name}`; } diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index beebe6df1..613300f43 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -54,7 +54,6 @@ const scripts: UpgradeScript[] = [ await document.getMulti(d._id, document.TYPE_DISCUSSION_REPLY).forEach((drdoc) => { bulk.find({ _id: drdoc._id }).updateOne({ $set: { history: [] } }); }); - // TODO tail reply await document.getMulti(d._id, document.TYPE_PROBLEM_SOLUTION).forEach((psdoc) => { bulk.find({ _id: psdoc._id }).updateOne({ $set: { history: [] } }); }); diff --git a/packages/migrate-vijos/script.ts b/packages/migrate-vijos/script.ts index f8fc4bf9c..981d8fe53 100644 --- a/packages/migrate-vijos/script.ts +++ b/packages/migrate-vijos/script.ts @@ -6,7 +6,6 @@ import { const dst = global.Hydro.service.db; const { discussion, document } = global.Hydro.model; -// TODO output enhancement const map = {}; const pid = (id) => { diff --git a/packages/ui-default/backendlib/template.js b/packages/ui-default/backendlib/template.js index 1d2f1718a..1bcc17bd0 100644 --- a/packages/ui-default/backendlib/template.js +++ b/packages/ui-default/backendlib/template.js @@ -115,9 +115,6 @@ nunjucks.runtime.memberLookup = function memberLookup(obj, val) { const env = new Nunjucks(); env.addGlobal('static_url', (assetName) => { const cdnPrefix = process.env.DEV ? '/' : global.Hydro.model.system.get('server.cdn'); - if (global.Hydro.ui.manifest[assetName]) { - return `${cdnPrefix}${global.Hydro.ui.manifest[assetName]}`; - } return `${cdnPrefix}${assetName}`; }); // eslint-disable-next-line no-eval diff --git a/packages/ui-default/build/config/webpack.js b/packages/ui-default/build/config/webpack.js index 6114ed85d..e460ba842 100644 --- a/packages/ui-default/build/config/webpack.js +++ b/packages/ui-default/build/config/webpack.js @@ -12,7 +12,6 @@ import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer'; import SpeedMeasurePlugin from 'speed-measure-webpack-plugin'; import WebpackBar from 'webpackbar'; import mapWebpackUrlPrefix from '../utils/mapWebpackUrlPrefix'; -import StaticManifestPlugin from '../plugins/manifest.webpack'; import root from '../utils/root'; const beautifyOutputUrl = mapWebpackUrlPrefix([ @@ -237,13 +236,6 @@ export default function (env = {}) { }, }], }), - new StaticManifestPlugin({ - fileName: 'static-manifest.json', - ignore: [ - 'img/emoji/', - 'katex/', - ], - }), ...env.measure ? [new BundleAnalyzerPlugin({ analyzerPort: 'auto' })] : [], ], }; diff --git a/packages/ui-default/build/main.js b/packages/ui-default/build/main.js index c96350b85..83e9be9a6 100644 --- a/packages/ui-default/build/main.js +++ b/packages/ui-default/build/main.js @@ -76,6 +76,9 @@ async function main() { await runWebpack(argv.options); if (argv.options.production) { fs.removeSync('public/vditor/dist/js/mathjax'); + fs.removeSync('public/vditor/dist/js/echarts'); + fs.removeSync('public/vditor/dist/js/graphviz'); + fs.removeSync('public/vditor/dist/js/mermaid'); fs.removeSync('public/vditor/dist/js/abcjs'); } process.chdir(dir); diff --git a/packages/ui-default/build/plugins/manifest.webpack.js b/packages/ui-default/build/plugins/manifest.webpack.js deleted file mode 100644 index 90252ef6d..000000000 --- a/packages/ui-default/build/plugins/manifest.webpack.js +++ /dev/null @@ -1,49 +0,0 @@ -/* eslint-disable import/no-extraneous-dependencies */ -const _ = require('lodash'); -const crypto = require('crypto'); - -export default class StaticManifestPlugin { - constructor({ fileName, ignore }) { - this.fileName = fileName; - this.ignore = ignore; - } - - apply(compiler) { - compiler.hooks.emit.tap('sync', (compilation) => { - const stats = compilation.getStats().toJson(); - const manifest = _(stats.assets) - .map((asset) => { - const { name } = asset; - // Skip files listed in ignore - if (_.some(this.ignore, (ignorePattern) => name.indexOf(ignorePattern) > -1)) { - return null; - } - // Skip calculating hash for names like ?xxxx - if (name.indexOf('?') > -1) { - return [ - name.replace(/\?[\s\S]*/, ''), // origin name - name, // name with hash - ]; - } - // We need to calculate hash for the remaining files. - const source = compilation.assets[asset.name].source(); - const shasum = crypto.createHash('sha1'); - shasum.update(source); - const hash = shasum.digest('hex').substr(0, 10); - return [ - name, // origin name - `${name}?${hash}`, // name with hash - ]; - }) - .filter() - .orderBy(['0']) - .fromPairs() - .value(); - const json = JSON.stringify(manifest, null, 2); - compilation.assets[this.fileName] = { - source: () => json, - size: () => json.length, - }; - }); - } -} diff --git a/packages/ui-default/handler.ts b/packages/ui-default/handler.ts index eecd94a0d..fc6d78d2f 100644 --- a/packages/ui-default/handler.ts +++ b/packages/ui-default/handler.ts @@ -1,9 +1,8 @@ /* eslint-disable no-return-await */ /* eslint-disable camelcase */ import crypto from 'crypto'; -import { readdir, readFile } from 'fs-extra'; +import { readFileSync } from 'fs-extra'; import { join } from 'path'; -import { tmpdir } from 'os'; import { ObjectID } from 'mongodb'; import * as bus from 'hydrooj/src/service/bus'; import { Route, Handler } from 'hydrooj/src/service/server'; @@ -31,8 +30,10 @@ declare module 'hydrooj/src/interface' { const cache = {}; const coll = db.collection('cache'); +const pages = Object.keys(global.ui.manifest) + .filter((file) => file.endsWith('.page.js')) + .map((i) => readFileSync(join(global.ui.manifest[i], i), 'utf-8')); -const basedir = join(tmpdir(), 'hydro', 'public'); async function constant(args: ConstantArgs) { // CompileLangs const payload = [`window.LANGS=${JSON.stringify(setting.langs)};`]; @@ -64,11 +65,7 @@ async function constant(args: ConstantArgs) { }` : ''}\`; document.body.appendChild(e);`; - const files = await readdir(basedir); - const pages = files - .filter((file) => file.endsWith('.page.js')) - .map((i) => readFile(join(basedir, i), 'utf-8')); - payload.push(...await Promise.all(pages)); + payload.push(...pages); const c = crypto.createHash('sha1'); c.update(JSON.stringify(payload)); diff --git a/packages/ui-default/package.json b/packages/ui-default/package.json index 95f83e0bb..69c57093b 100644 --- a/packages/ui-default/package.json +++ b/packages/ui-default/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/ui-default", - "version": "4.20.8", + "version": "4.21.0", "author": "undefined ", "license": "AGPL-3.0", "main": "hydro.js",