From 160b4482ccceee3cff276cd7c0ef5686177ead49 Mon Sep 17 00:00:00 2001 From: masnn Date: Wed, 10 Jun 2020 12:20:07 +0800 Subject: [PATCH] =?UTF-8?q?=E9=BB=91=E5=90=8D=E5=8D=95=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hydro/handler/domain.js | 7 +- hydro/handler/training.js | 1 - hydro/handler/user.js | 7 +- hydro/lib/hpm.js | 18 ++-- hydro/loader.js | 192 ++++++++++++--------------------- hydro/model/discussion.js | 14 ++- hydro/model/user.js | 10 ++ hydro/script/blacklist.js | 64 +++++++++++ hydro/service/server.js | 12 ++- hydro/unzip.js | 103 ++++++++++++++++++ module/migrate-vijos/script.js | 12 ++- 11 files changed, 282 insertions(+), 158 deletions(-) create mode 100644 hydro/script/blacklist.js create mode 100644 hydro/unzip.js diff --git a/hydro/handler/domain.js b/hydro/handler/domain.js index cb6c99be4..19e1b54b2 100644 --- a/hydro/handler/domain.js +++ b/hydro/handler/domain.js @@ -106,11 +106,12 @@ class DomainUserHandler extends ManageHandler { for (const role of roles) rudocs[role._id] = []; for (const udoc of udocs) { uids.push(udoc.uid); - // TODO Improve here - // eslint-disable-next-line no-await-in-loop - const ud = await user.getById(domainId, udoc.uid); + const ud = user.getById(domainId, udoc.uid); rudocs[ud.role].push(ud); } + const tasks = []; + for (const key in rudocs) tasks.push(Promise.all(rudocs[key])); + await Promise.all(tasks); const rolesSelect = roles.map((role) => [role._id, role._id]); const udict = await user.getList(domainId, uids); const path = [ diff --git a/hydro/handler/training.js b/hydro/handler/training.js index 5bb5e3e07..f9786f7cb 100644 --- a/hydro/handler/training.js +++ b/hydro/handler/training.js @@ -97,7 +97,6 @@ class TrainingDetailHandler extends TrainingHandler { async get({ domainId, tid }) { const tdoc = await training.get(domainId, tid); const pids = training.getPids(tdoc); - // TODO(twd2): check status, eg. test, hidden problem, ... const f = this.user.hasPerm(PERM_VIEW_PROBLEM_HIDDEN) ? {} : { hidden: false }; const [udoc, pdict] = await Promise.all([ user.getById(domainId, tdoc.owner), diff --git a/hydro/handler/user.js b/hydro/handler/user.js index bcad89c3e..52bc6575e 100644 --- a/hydro/handler/user.js +++ b/hydro/handler/user.js @@ -8,7 +8,7 @@ const { PERM_LOGGEDIN } = require('../permission'); const { UserAlreadyExistError, InvalidTokenError, VerifyPasswordError, UserNotFoundError, LoginError, SystemError, - PermissionError, + PermissionError, BlacklistedError, } = require('../error'); class UserLoginHandler extends Handler { @@ -23,8 +23,7 @@ class UserLoginHandler extends Handler { if (!udoc) throw new LoginError(uname); if (udoc) udoc.checkPassword(password); await user.setById(udoc._id, { loginat: new Date(), loginip: this.request.ip }); - udoc.salt = ''; - udoc.password = ''; + if (udoc.ban) throw new BlacklistedError(uname); this.session.uid = udoc._id; this.session.rememberme = rememberme; const referer = this.request.headers.referer || '/'; @@ -63,7 +62,7 @@ class UserRegisterHandler extends Handler { ); if (await system.get('smtp.user')) { const m = await this.renderHTML('user_register_mail.html', { - url: `${await system.get('baseurl')}/register/${t[0]}`, + url: `${await system.get('server.url')}/register/${t[0]}`, }); await sendMail(mail, 'Sign Up', 'user_register_mail', m); this.response.template = 'user_register_mail_sent.html'; diff --git a/hydro/lib/hpm.js b/hydro/lib/hpm.js index 2d2dbe828..f3ab414f6 100644 --- a/hydro/lib/hpm.js +++ b/hydro/lib/hpm.js @@ -3,9 +3,7 @@ /* eslint-disable no-eval */ const fs = require('fs'); const os = require('os'); -const zlib = require('zlib'); const path = require('path'); -const yaml = require('js-yaml'); const download = require('./download'); function root(name) { @@ -28,18 +26,14 @@ for (const i of moduleRoots) { async function getInstalled() { const modules = []; - const files = fs.readdirSync(moduleRoot); + const files = fs.readdirSync(`${os.tmpdir()}/hydro/tmp`); for (const file of files) { - if (file.endsWith('.hydro')) { + const info = `${os.tmpdir()}/hydro/tmp/${file}/hydro.json`; + if (fs.existsSync(info)) { try { - const f = fs.readFileSync(root(`${moduleRoot}/${file}`)); - const s = fs.statSync(root(`${moduleRoot}/${file}`)); - const m = { - ...yaml.safeLoad(zlib.gunzipSync(f)), - filename: file.split('.')[0], - size: s.size, - }; - modules.push(m); + const m = file; + const t = JSON.parse(fs.readFileSync(info).toString()); + modules.push(`${m}|${t.id}`); } catch (e) { if (e.code === 'Z_DATA_ERROR') { console.error(`Module Load Fail: ${file} (File Corrupted)`); diff --git a/hydro/loader.js b/hydro/loader.js index 5dec8560e..b9f51fcdb 100644 --- a/hydro/loader.js +++ b/hydro/loader.js @@ -3,11 +3,9 @@ /* eslint-disable no-eval */ const fs = require('fs'); const os = require('os'); -const zlib = require('zlib'); const path = require('path'); const cluster = require('cluster'); const numCPUs = require('os').cpus().length; -const yaml = require('js-yaml'); function ensureDir(dir) { if (!fs.existsSync(dir)) { @@ -18,73 +16,30 @@ function ensureDir(dir) { } } +function superRequire(p) { + // eslint-disable-next-line camelcase + if (typeof __non_webpack_require__ === 'function') { + // eslint-disable-next-line no-undef + return __non_webpack_require__(p); + } return require(p); +} + let pending = []; const active = []; const fail = []; -if (global._hydroModule) { - for (const filename in global._hydroModule) { - const m = { - ...yaml.safeLoad(zlib.gunzipSync(Buffer.from(global._hydroModule[filename], 'base64'))), - filename, - isBuiltin: true, - }; - pending.push(m); - } -} - -async function unzip() { - for (const i of pending) { - try { - if (i.file) { - i.files = {}; - ensureDir(path.resolve(os.tmpdir(), 'hydro', i.id)); - for (const n in i.file) { - if (i.file[n] === null) { - ensureDir(path.resolve(os.tmpdir(), 'hydro', i.id, n)); - } else { - const e = path.resolve(os.tmpdir(), 'hydro', i.id, n); - fs.writeFileSync(e, Buffer.from(i.file[n], 'base64'), { mode: 755 }); - i.files[n] = e; - } - } - } - } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Module Load Fail: ${i.id}`); - console.error(e); - } - } -} - -async function preload() { - for (const i of pending) { - try { - if (i.os) { - if (!i.os.includes(os.platform().toLowerCase())) throw new Error('Unsupported OS'); - } - } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Module Load Fail: ${i.id}`); - console.error(e); - } - } -} - async function handler() { for (const i of pending) { - if (i.handler && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/handler.js`; + if (fs.existsSync(p) && i.fail) { try { - console.log(`Handler init: ${i.id}`); - console.time(`Handler init: ${i.id}`); - eval(i.handler); - console.timeEnd(`Handler init: ${i.id}`); + console.log(`Handler init: ${i}`); + console.time(`Handler init: ${i}`); + superRequire(p); + console.timeEnd(`Handler init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Handler Load Fail: ${i.id}`); + fail.push(i); + console.error(`Handler Load Fail: ${i}`); } } } @@ -92,14 +47,14 @@ async function handler() { async function locale() { for (const i of pending) { - if (i.locale && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/locale.json`; + if (fs.existsSync(p) && i.fail) { try { - global.Hydro.lib.i18n(i.locale); - console.log(`Locale init: ${i.id}`); + global.Hydro.lib.i18n(superRequire(p)); + console.log(`Locale init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Locale Load Fail: ${i.id}`); + fail.push(i); + console.error(`Locale Load Fail: ${i}`); } } } @@ -107,14 +62,14 @@ async function locale() { async function template() { for (const i of pending) { - if (i.template && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/template.json`; + if (fs.existsSync(p) && !i.fail) { try { - Object.assign(global.Hydro.template, i.template); - console.log(`Template init: ${i.id}`); + Object.assign(global.Hydro.template, superRequire(p)); + console.log(`Template init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Template Load Fail: ${i.id}`); + fail.push(i); + console.error(`Template Load Fail: ${i}`); } } } @@ -122,17 +77,16 @@ async function template() { async function model() { for (const i of pending) { - if (i.model && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/model.js`; + if (fs.existsSync(p) && !i.fail) { try { - console.log(`Model init: ${i.id}`); - console.time(`Model init: ${i.id}`); - const m = eval(i.model); - if ((m || {}).index) await m.index(); - console.timeEnd(`Model init: ${i.id}`); + console.log(`Model init: ${i}`); + console.time(`Model init: ${i}`); + superRequire(p); + console.timeEnd(`Model init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Model Load Fail: ${i.id}`); + fail.push(i); + console.error(`Model Load Fail: ${i}`); } } } @@ -140,16 +94,16 @@ async function model() { async function lib() { for (const i of pending) { - if (i.lib && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/lib.js`; + if (fs.existsSync(p) && !i.fail) { try { - console.log(`Lib init: ${i.id}`); - console.time(`Lib init: ${i.id}`); - eval(i.lib); - console.timeEnd(`Lib init: ${i.id}`); + console.log(`Lib init: ${i}`); + console.time(`Lib init: ${i}`); + superRequire(p); + console.timeEnd(`Lib init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Lib Load Fail: ${i.id}`); + fail.push(i); + console.error(`Lib Load Fail: ${i}`); } } } @@ -157,16 +111,16 @@ async function lib() { async function service() { for (const i of pending) { - if (i.service && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/service.js`; + if (fs.existsSync(p) && !i.fail) { try { - console.time(`Service init: ${i.id}`); - const m = eval(i.service); - if ((m || {}).init) await m.init(); - console.timeEnd(`Service init: ${i.id}`); + console.log(`Service init: ${i}`); + console.time(`Service init: ${i}`); + superRequire(p); + console.timeEnd(`Service init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Service Load Fail: ${i.id}`); + fail.push(i); + console.error(`Service Load Fail: ${i}`); console.error(e); } } @@ -175,19 +129,18 @@ async function service() { async function script() { for (const i of pending) { - if (i.script && !i.fail) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/script.js`; + if (fs.existsSync(p) && !i.fail) { try { - console.time(`Script init: ${i.id}`); - eval(i.script); - console.timeEnd(`Script init: ${i.id}`); + console.time(`Script init: ${i}`); + superRequire(p); + console.timeEnd(`Script init: ${i}`); } catch (e) { - i.fail = true; - fail.push(i.id); - console.error(`Script Load Fail: ${i.id}`); + fail.push(i); + console.error(`Script Load Fail: ${i}`); console.error(e); } } - if (!i.fail) active.push(i.name); } } @@ -217,13 +170,14 @@ const builtinHandler = [ const builtinScript = [ 'install', 'uninstall', 'rating', 'recalcRating', 'register', + 'blacklist', ]; async function loadAsMaster() { ensureDir(path.resolve(os.tmpdir(), 'hydro')); - pending.push(...await require('./lib/hpm').getInstalled()); - await unzip(); - await preload(); + // TODO better run in another process as this needs lots of memory + require('./unzip')(); + pending = await require('./lib/hpm').getInstalled(); require('./lib/i18n'); require('./utils'); require('./error'); @@ -247,10 +201,7 @@ async function loadAsMaster() { }); for (const i of builtinLib) require(`./lib/${i}`); await lib(); - require('./service/gridfs'); require('./service/monitor'); - const server = require('./service/server'); - await server.prepare(); await service(); for (const i of builtinModel) { const m = require(`./model/${i}`); @@ -262,14 +213,7 @@ async function loadAsMaster() { const ins = require('./script/install'); await ins.run({ username: 'Root', password: 'rootroot' }); } - for (const i of builtinHandler) require(`./handler/${i}`); await model(); - await handler(); - for (const i in global.Hydro.handler) { - await global.Hydro.handler[i](); - } - const notfound = require('./handler/notfound'); - await notfound(); for (const i in global.Hydro.service) { if (global.Hydro.service[i].postInit) { try { @@ -279,15 +223,11 @@ async function loadAsMaster() { } } } - for (const i of builtinScript) require(`./script/${i}`); - await script(); pending = []; - await server.start(await system.get('server.port')); } async function loadAsWorker() { - pending.push(...await require('./lib/hpm').getInstalled()); - await preload(); + pending = await require('./lib/hpm').getInstalled(); require('./lib/i18n'); require('./utils'); require('./error'); @@ -385,7 +325,8 @@ async function load() { cluster.on('online', (worker) => { console.log(`Worker ${worker.process.pid} is online`); }); - for (let i = 1; i < numCPUs; i++) { + // FIXME this requires lots of memory + for (let i = 0; i < numCPUs; i++) { cluster.fork(); } } else { @@ -393,6 +334,7 @@ async function load() { await loadAsWorker(); console.log(`Worker ${process.pid} Started`); } + if (global.gc) global.gc(); } module.exports = { diff --git a/hydro/model/discussion.js b/hydro/model/discussion.js index c0a4115e0..45315e434 100644 --- a/hydro/model/discussion.js +++ b/hydro/model/discussion.js @@ -37,12 +37,12 @@ function edit(domainId, did, title, content, highlight) { } function del(domainId, did) { - // TODO(masnn) delete status return Promise.all([ document.deleteOne(domainId, document.TYPE_DISCUSSION, did), document.deleteMulti(domainId, document.TYPE_DISCUSSION_REPLY, { parentType: document.TYPE_DISCUSSION, parentId: did, }), + document.deleteMultiStatus(domainId, document.TYPE_DISCUSSION, { docId: did }), ]); } @@ -167,12 +167,16 @@ async function getVnode(domainId, ddoc, handler) { } async function getListVnodes(domainId, ddocs, handler) { + const tasks = []; const res = {}; - for (const ddoc of ddocs) { - if (!res[ddoc.parentType]) res[ddoc.parentType] = {}; - // FIXME no-await-in-loop - res[ddoc.parentType][ddoc.parentId] = await getVnode(domainId, ddoc, handler); // eslint-disable-line no-await-in-loop + function task(ddoc) { + return getVnode(domainId, ddoc, handler).then((vnode) => { + if (!res[ddoc.parentType]) res[ddoc.parentType] = {}; + res[ddoc.parentType][ddoc.parentId] = vnode; + }); } + for (const ddoc of ddocs) tasks.push(task(ddoc)); + await Promise.all(tasks); return res; } diff --git a/hydro/model/user.js b/hydro/model/user.js index ce901d95b..53152cd68 100644 --- a/hydro/model/user.js +++ b/hydro/model/user.js @@ -1,6 +1,7 @@ const builtin = require('./builtin'); const document = require('./document'); const system = require('./system'); +const token = require('./token'); const { UserNotFoundError, UserAlreadyExistError } = require('../error'); const perm = require('../permission'); const pwhash = require('../lib/hash.hydro'); @@ -25,6 +26,7 @@ class USER { this.loginat = udoc.loginat; this.bio = udoc.bio || ''; this.gravatar = udoc.gravatar || ''; + this.ban = udoc.ban || false; this.nAccept = dudoc.nAccept || 0; this.nSubmit = dudoc.nSubmit || 0; this.rating = dudoc.rating || 1500; @@ -215,6 +217,13 @@ function deleteRoles(domainId, roles) { ]); } +function ban(uid) { + return Promise.all([ + coll.updateOne({ _id: uid }, { $set: { ban: true } }), + token.deleteByUid(uid), + ]); +} + function ensureIndexes() { return Promise.all([ coll.createIndex('unameLower', { unique: true }), @@ -246,5 +255,6 @@ global.Hydro.model.user = module.exports = { getInDomain, addRole, deleteRoles, + ban, ensureIndexes, }; diff --git a/hydro/script/blacklist.js b/hydro/script/blacklist.js new file mode 100644 index 000000000..5b26d5713 --- /dev/null +++ b/hydro/script/blacklist.js @@ -0,0 +1,64 @@ +// This code format is just for fun. DO NOT DO THIS IN YOUR CODE ! +/* eslint-disable */ +const { ObjectID } = require('bson'); +const document = require('../model/document'); +const discussion = require('../model/discussion'); +const user = require('../model/user'); +const blacklist = require('../model/blacklist'); +const db = require('../service/db.js'); + +async function _address(ip, bset, uset, dset, dryrun) { + if (bset.has(ip)) + return + bset.add(ip) + console.log('ip %s', ip) + const users = await db.collection('user').find({ loginip: ip }).toArray() + const tasks = [] + for (const udoc of users) { + tasks.push(_user(udoc._id, bset, uset, dset, dryrun)) } + await Promise.all(tasks) + if (!dryrun) + await blacklist.add(ip) } + +async function _discussion(domainId, did, bset, uset, dset, dryrun) { + if (dset.has(did)) + return + dset.add(did) + const ddoc = await discussion.get(domainId, did) + if (!ddoc) + return + console.log('discussion %s', ddoc['title']) + await _user(ddoc['owner'], bset, uset, dset, dryrun) + if (ddoc.ip) + await _address(ddoc['ip'], bset, uset, dset, dryrun) + if (!dryrun) + await discussion.delete(domainId, ddoc['docId']) } + +async function _user(uid, bset, uset, dset, dryrun) { + if (uset.has(uid)) + return + uset.add(uid) + const udoc = await user.getById(uid) + if (!udoc) + return + console.log('user %s %s', udoc['_id'], udoc['uname']) + await _address(udoc['loginip'], bset, uset, dset, dryrun) + const ddocs = await db.collection('document') + .find({ docType: document.TYPE_DISCUSSION }).sort({ domainId: 1, docId: 1 }).toArray() + const tasks = [] + for (const ddoc of ddocs) + tasks.push(_discussion(ddoc.domainId, ddoc.docId, bset, uset, dset, dryrun)) + await Promise.all(tasks) + if (!dryrun) + await user.ban(uid) } + +async function run({address = null, discuss = null, user = null, dryrun = true}) { + if (address) + await _address(address, new Set(), new Set(), new Set(), dryrun) + if (discuss) + await _discussion(discuss.domainId, new ObjectID(discuss.did), + new Set(), new Set(), new Set(), dryrun) + if (user) + await _user(user, new Set(), new Set(), new Set(), dryrun) } + +global.Hydro.script.blacklist = module.exports = { run }; diff --git a/hydro/service/server.js b/hydro/service/server.js index b0f205091..de67dd371 100644 --- a/hydro/service/server.js +++ b/hydro/service/server.js @@ -1,6 +1,7 @@ const assert = require('assert'); const path = require('path'); const os = require('os'); +const cluster = require('cluster'); const { ObjectID } = require('bson'); const Koa = require('koa'); const yaml = require('js-yaml'); @@ -438,12 +439,17 @@ function Validate(key, func) { else validate[key] = [func]; } -async function start(port) { +async function start() { app.use(morgan(':method :url :status :res[content-length] - :response-time ms')); app.use(router.routes()).use(router.allowedMethods()); Route('*', Handler); - server.listen(port); - if (port) console.log('Server listening at: %s', port); + if (cluster.worker.id === 1) { + const port = await system.get('server.port'); + server.listen(port); + console.log('Server listening at: %s', port); + } else { + server.listen(); + } } global.Hydro.service.server = module.exports = { diff --git a/hydro/unzip.js b/hydro/unzip.js new file mode 100644 index 000000000..757d2e1fd --- /dev/null +++ b/hydro/unzip.js @@ -0,0 +1,103 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const zlib = require('zlib'); +const yaml = require('js-yaml'); + +function ensureDir(dir) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir); + } else if (!fs.statSync(dir).isDirectory()) { + fs.unlinkSync(dir); + fs.mkdirSync(dir); + } +} + +function root(name) { + return path.resolve(process.cwd(), name); +} + +const moduleRoots = [ + root('.build/module'), + root('module'), + root(path.resolve(os.homedir(), '.hydro', 'module')), + root('.'), +]; +let moduleRoot; +for (const i of moduleRoots) { + if (fs.existsSync(i) && fs.statSync(i).isDirectory()) { + moduleRoot = i; + break; + } +} +function exec() { + const files = fs.readdirSync(moduleRoot); + const t = ['service', 'lib', 'model', 'handler', 'script']; + ensureDir(`${os.tmpdir()}/hydro`); + ensureDir(`${os.tmpdir()}/hydro/tmp`); + for (const file of files) { + if (file.endsWith('.hydro')) { + try { + const f = fs.readFileSync(root(`${moduleRoot}/${file}`)); + const s = fs.statSync(root(`${moduleRoot}/${file}`)); + const m = yaml.safeLoad(zlib.gunzipSync(f)); + const { id } = m; + if (m.os) { + if (!m.os.includes(os.platform().toLowerCase())) { + console.error(`Module load fail: ${id} ${file.split('.')[0]} Unsupported OS`); + // eslint-disable-next-line no-continue + continue; + } + } + ensureDir(`${os.tmpdir()}/hydro/tmp/${id}`); + if (m.locale) { + fs.writeFileSync( + `${os.tmpdir()}/hydro/tmp/${id}/locale.json`, + JSON.stringify(m.locale), + ); + } + if (m.template) { + fs.writeFileSync( + `${os.tmpdir()}/hydro/tmp/${id}/template.json`, + JSON.stringify(m.template), + ); + } + for (const i of t) { + if (m[i]) { + fs.writeFileSync( + `${os.tmpdir()}/hydro/tmp/${id}/${i}.js`, + f[i], + ); + } + } + if (m.file) { + ensureDir(path.resolve(os.tmpdir(), 'hydro', m.id)); + for (const n in m.file) { + if (m.file[n] === null) { + ensureDir(path.resolve(os.tmpdir(), 'hydro', m.id, n)); + } else { + const e = path.resolve(os.tmpdir(), 'hydro', m.id, n); + fs.writeFileSync(e, Buffer.from(m.file[n], 'base64'), { mode: 755 }); + } + } + } + fs.writeFileSync( + root(`${os.tmpdir()}/hydro/tmp/${id}/hydro.json`), + JSON.stringify({ + size: s.size, + version: m.version, + id: m.id, + name: m.name, + description: m.description, + }), + ); + } catch (e) { + if (e.code === 'Z_DATA_ERROR') { + console.error(`Module Load Fail: ${file} (File Corrupted)`); + } else console.error(`Module Load Fail: ${file} ${e}`); + } + } + } +} + +module.exports = exec; diff --git a/module/migrate-vijos/script.js b/module/migrate-vijos/script.js index 3cffdf8c4..c9fe00001 100644 --- a/module/migrate-vijos/script.js +++ b/module/migrate-vijos/script.js @@ -246,6 +246,7 @@ async function domainUser(src, report) { const docs = await src.collection('domain.user') .find().skip(i * 50).limit(50) .toArray(); + const t = []; for (const doc of docs) { const mapper = { _id: '_id', @@ -277,7 +278,7 @@ async function domainUser(src, report) { docWithoutId[key] = d[key]; } } - d.push(dst.collection('document').updateOne( + t.push(dst.collection('document.status').updateOne( { _id: d._id }, { $set: { @@ -287,6 +288,7 @@ async function domainUser(src, report) { { upsert: true }, )); } + await Promise.all(t); await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) }); } } @@ -341,9 +343,9 @@ async function task(name, src, report) { const mapper = tasks[name]; for (const key in doc) { if (typeof mapper[key] === 'string') { - res[mapper[key]] = doc[key]; + d[mapper[key]] = doc[key]; } else if (typeof mapper[key] === 'object') { - res[mapper[key].field] = mapper[key].processer(doc[key]); + d[mapper[key].field] = mapper[key].processer(doc[key]); } else if (mapper[key] === null) { // Ignore this key } else { @@ -369,7 +371,7 @@ async function task(name, src, report) { res.push((async () => { const data = await dst.collection(name).findOne(query); if (data) { - await dst.collection(name).updateOne(query, { $set: docWithoutId }); + await dst.collection(name).updateOne(query, { $set: docWithoutDid }); } else if (d._id) { const dat = await dst.collection(name).findOne({ _id: d._id }); if (dat) { @@ -390,7 +392,7 @@ async function task(name, src, report) { } else res.push(dst.collection(name).insertOne(d)); } } - await Promise.all(res); + await Promise.all(res).catch((e) => console.log(e)); const progress = Math.round(100 * ((i + 1) / (total + 1))); if (progress > lastProgress) { await report({ progress });