黑名单功能

2026-10-17 09:36:08 +08:00 · 2020-06-10 12:20:07 +08:00
1 parent 55912b087d
commit 160b4482cc
@@ -106,11 +106,12 @@ class DomainUserHandler extends ManageHandler {
        for (const role of roles) rudocs[role._id] = [];
        for (const udoc of udocs) {
            uids.push(udoc.uid);
-            // TODO Improve here
-            // eslint-disable-next-line no-await-in-loop
-            const ud = await user.getById(domainId, udoc.uid);
+            const ud = user.getById(domainId, udoc.uid);
            rudocs[ud.role].push(ud);
        }
+        const tasks = [];
+        for (const key in rudocs) tasks.push(Promise.all(rudocs[key]));
+        await Promise.all(tasks);
        const rolesSelect = roles.map((role) => [role._id, role._id]);
        const udict = await user.getList(domainId, uids);
        const path = [
@@ -97,7 +97,6 @@ class TrainingDetailHandler extends TrainingHandler {
    async get({ domainId, tid }) {
        const tdoc = await training.get(domainId, tid);
        const pids = training.getPids(tdoc);
-        // TODO(twd2): check status, eg. test, hidden problem, ...
        const f = this.user.hasPerm(PERM_VIEW_PROBLEM_HIDDEN) ? {} : { hidden: false };
        const [udoc, pdict] = await Promise.all([
            user.getById(domainId, tdoc.owner),
@@ -8,7 +8,7 @@ const { PERM_LOGGEDIN } = require('../permission');
 const {
    UserAlreadyExistError, InvalidTokenError, VerifyPasswordError,
    UserNotFoundError, LoginError, SystemError,
-    PermissionError,
+    PermissionError, BlacklistedError,
 } = require('../error');

 class UserLoginHandler extends Handler {
@@ -23,8 +23,7 @@ class UserLoginHandler extends Handler {
        if (!udoc) throw new LoginError(uname);
        if (udoc) udoc.checkPassword(password);
        await user.setById(udoc._id, { loginat: new Date(), loginip: this.request.ip });
-        udoc.salt = '';
-        udoc.password = '';
+        if (udoc.ban) throw new BlacklistedError(uname);
        this.session.uid = udoc._id;
        this.session.rememberme = rememberme;
        const referer = this.request.headers.referer || '/';
@@ -63,7 +62,7 @@ class UserRegisterHandler extends Handler {
        );
        if (await system.get('smtp.user')) {
            const m = await this.renderHTML('user_register_mail.html', {
-                url: `${await system.get('baseurl')}/register/${t[0]}`,
+                url: `${await system.get('server.url')}/register/${t[0]}`,
            });
            await sendMail(mail, 'Sign Up', 'user_register_mail', m);
            this.response.template = 'user_register_mail_sent.html';
@@ -3,9 +3,7 @@
 /* eslint-disable no-eval */
 const fs = require('fs');
 const os = require('os');
-const zlib = require('zlib');
 const path = require('path');
-const yaml = require('js-yaml');
 const download = require('./download');

 function root(name) {
@@ -28,18 +26,14 @@ for (const i of moduleRoots) {

 async function getInstalled() {
    const modules = [];
-    const files = fs.readdirSync(moduleRoot);
+    const files = fs.readdirSync(`${os.tmpdir()}/hydro/tmp`);
    for (const file of files) {
-        if (file.endsWith('.hydro')) {
+        const info = `${os.tmpdir()}/hydro/tmp/${file}/hydro.json`;
+        if (fs.existsSync(info)) {
            try {
-                const f = fs.readFileSync(root(`${moduleRoot}/${file}`));
-                const s = fs.statSync(root(`${moduleRoot}/${file}`));
-                const m = {
-                    ...yaml.safeLoad(zlib.gunzipSync(f)),
-                    filename: file.split('.')[0],
-                    size: s.size,
-                };
-                modules.push(m);
+                const m = file;
+                const t = JSON.parse(fs.readFileSync(info).toString());
+                modules.push(`${m}|${t.id}`);
            } catch (e) {
                if (e.code === 'Z_DATA_ERROR') {
                    console.error(`Module Load Fail: ${file} (File Corrupted)`);
@@ -3,11 +3,9 @@
 /* eslint-disable no-eval */
 const fs = require('fs');
 const os = require('os');
-const zlib = require('zlib');
 const path = require('path');
 const cluster = require('cluster');
 const numCPUs = require('os').cpus().length;
-const yaml = require('js-yaml');

 function ensureDir(dir) {
    if (!fs.existsSync(dir)) {
@@ -18,73 +16,30 @@ function ensureDir(dir) {
    }
 }

+function superRequire(p) {
+    // eslint-disable-next-line camelcase
+    if (typeof __non_webpack_require__ === 'function') {
+        // eslint-disable-next-line no-undef
+        return __non_webpack_require__(p);
+    } return require(p);
+}
+
 let pending = [];
 const active = [];
 const fail = [];

-if (global._hydroModule) {
-    for (const filename in global._hydroModule) {
-        const m = {
-            ...yaml.safeLoad(zlib.gunzipSync(Buffer.from(global._hydroModule[filename], 'base64'))),
-            filename,
-            isBuiltin: true,
-        };
-        pending.push(m);
-    }
-}
-
-async function unzip() {
-    for (const i of pending) {
-        try {
-            if (i.file) {
-                i.files = {};
-                ensureDir(path.resolve(os.tmpdir(), 'hydro', i.id));
-                for (const n in i.file) {
-                    if (i.file[n] === null) {
-                        ensureDir(path.resolve(os.tmpdir(), 'hydro', i.id, n));
-                    } else {
-                        const e = path.resolve(os.tmpdir(), 'hydro', i.id, n);
-                        fs.writeFileSync(e, Buffer.from(i.file[n], 'base64'), { mode: 755 });
-                        i.files[n] = e;
-                    }
-                }
-            }
-        } catch (e) {
-            i.fail = true;
-            fail.push(i.id);
-            console.error(`Module Load Fail: ${i.id}`);
-            console.error(e);
-        }
-    }
-}
-
-async function preload() {
-    for (const i of pending) {
-        try {
-            if (i.os) {
-                if (!i.os.includes(os.platform().toLowerCase())) throw new Error('Unsupported OS');
-            }
-        } catch (e) {
-            i.fail = true;
-            fail.push(i.id);
-            console.error(`Module Load Fail: ${i.id}`);
-            console.error(e);
-        }
-    }
-}
-
 async function handler() {
    for (const i of pending) {
-        if (i.handler && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/handler.js`;
+        if (fs.existsSync(p) && i.fail) {
            try {
-                console.log(`Handler init: ${i.id}`);
-                console.time(`Handler init: ${i.id}`);
-                eval(i.handler);
-                console.timeEnd(`Handler init: ${i.id}`);
+                console.log(`Handler init: ${i}`);
+                console.time(`Handler init: ${i}`);
+                superRequire(p);
+                console.timeEnd(`Handler init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Handler Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Handler Load Fail: ${i}`);
            }
        }
    }
@@ -92,14 +47,14 @@ async function handler() {

 async function locale() {
    for (const i of pending) {
-        if (i.locale && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/locale.json`;
+        if (fs.existsSync(p) && i.fail) {
            try {
-                global.Hydro.lib.i18n(i.locale);
-                console.log(`Locale init: ${i.id}`);
+                global.Hydro.lib.i18n(superRequire(p));
+                console.log(`Locale init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Locale Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Locale Load Fail: ${i}`);
            }
        }
    }
@@ -107,14 +62,14 @@ async function locale() {

 async function template() {
    for (const i of pending) {
-        if (i.template && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/template.json`;
+        if (fs.existsSync(p) && !i.fail) {
            try {
-                Object.assign(global.Hydro.template, i.template);
-                console.log(`Template init: ${i.id}`);
+                Object.assign(global.Hydro.template, superRequire(p));
+                console.log(`Template init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Template Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Template Load Fail: ${i}`);
            }
        }
    }
@@ -122,17 +77,16 @@ async function template() {

 async function model() {
    for (const i of pending) {
-        if (i.model && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/model.js`;
+        if (fs.existsSync(p) && !i.fail) {
            try {
-                console.log(`Model init: ${i.id}`);
-                console.time(`Model init: ${i.id}`);
-                const m = eval(i.model);
-                if ((m || {}).index) await m.index();
-                console.timeEnd(`Model init: ${i.id}`);
+                console.log(`Model init: ${i}`);
+                console.time(`Model init: ${i}`);
+                superRequire(p);
+                console.timeEnd(`Model init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Model Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Model Load Fail: ${i}`);
            }
        }
    }
@@ -140,16 +94,16 @@ async function model() {

 async function lib() {
    for (const i of pending) {
-        if (i.lib && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/lib.js`;
+        if (fs.existsSync(p) && !i.fail) {
            try {
-                console.log(`Lib init: ${i.id}`);
-                console.time(`Lib init: ${i.id}`);
-                eval(i.lib);
-                console.timeEnd(`Lib init: ${i.id}`);
+                console.log(`Lib init: ${i}`);
+                console.time(`Lib init: ${i}`);
+                superRequire(p);
+                console.timeEnd(`Lib init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Lib Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Lib Load Fail: ${i}`);
            }
        }
    }
@@ -157,16 +111,16 @@ async function lib() {

 async function service() {
    for (const i of pending) {
-        if (i.service && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/service.js`;
+        if (fs.existsSync(p) && !i.fail) {
            try {
-                console.time(`Service init: ${i.id}`);
-                const m = eval(i.service);
-                if ((m || {}).init) await m.init();
-                console.timeEnd(`Service init: ${i.id}`);
+                console.log(`Service init: ${i}`);
+                console.time(`Service init: ${i}`);
+                superRequire(p);
+                console.timeEnd(`Service init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Service Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Service Load Fail: ${i}`);
                console.error(e);
            }
        }
@@ -175,19 +129,18 @@ async function service() {

 async function script() {
    for (const i of pending) {
-        if (i.script && !i.fail) {
+        const p = `${os.tmpdir()}/hydro/tmp/${i}/script.js`;
+        if (fs.existsSync(p) && !i.fail) {
            try {
-                console.time(`Script init: ${i.id}`);
-                eval(i.script);
-                console.timeEnd(`Script init: ${i.id}`);
+                console.time(`Script init: ${i}`);
+                superRequire(p);
+                console.timeEnd(`Script init: ${i}`);
            } catch (e) {
-                i.fail = true;
-                fail.push(i.id);
-                console.error(`Script Load Fail: ${i.id}`);
+                fail.push(i);
+                console.error(`Script Load Fail: ${i}`);
                console.error(e);
            }
        }
-        if (!i.fail) active.push(i.name);
    }
 }

@@ -217,13 +170,14 @@ const builtinHandler = [

 const builtinScript = [
    'install', 'uninstall', 'rating', 'recalcRating', 'register',
+    'blacklist',
 ];

 async function loadAsMaster() {
    ensureDir(path.resolve(os.tmpdir(), 'hydro'));
-    pending.push(...await require('./lib/hpm').getInstalled());
-    await unzip();
-    await preload();
+    // TODO better run in another process as this needs lots of memory
+    require('./unzip')();
+    pending = await require('./lib/hpm').getInstalled();
    require('./lib/i18n');
    require('./utils');
    require('./error');
@@ -247,10 +201,7 @@ async function loadAsMaster() {
    });
    for (const i of builtinLib) require(`./lib/${i}`);
    await lib();
-    require('./service/gridfs');
    require('./service/monitor');
-    const server = require('./service/server');
-    await server.prepare();
    await service();
    for (const i of builtinModel) {
        const m = require(`./model/${i}`);
@@ -262,14 +213,7 @@ async function loadAsMaster() {
        const ins = require('./script/install');
        await ins.run({ username: 'Root', password: 'rootroot' });
    }
-    for (const i of builtinHandler) require(`./handler/${i}`);
    await model();
-    await handler();
-    for (const i in global.Hydro.handler) {
-        await global.Hydro.handler[i]();
-    }
-    const notfound = require('./handler/notfound');
-    await notfound();
    for (const i in global.Hydro.service) {
        if (global.Hydro.service[i].postInit) {
            try {
@@ -279,15 +223,11 @@ async function loadAsMaster() {
            }
        }
    }
-    for (const i of builtinScript) require(`./script/${i}`);
-    await script();
    pending = [];
-    await server.start(await system.get('server.port'));
 }

 async function loadAsWorker() {
-    pending.push(...await require('./lib/hpm').getInstalled());
-    await preload();
+    pending = await require('./lib/hpm').getInstalled();
    require('./lib/i18n');
    require('./utils');
    require('./error');
@@ -385,7 +325,8 @@ async function load() {
        cluster.on('online', (worker) => {
            console.log(`Worker ${worker.process.pid} is online`);
        });
-        for (let i = 1; i < numCPUs; i++) {
+        // FIXME this requires lots of memory
+        for (let i = 0; i < numCPUs; i++) {
            cluster.fork();
        }
    } else {
@@ -393,6 +334,7 @@ async function load() {
        await loadAsWorker();
        console.log(`Worker ${process.pid} Started`);
    }
+    if (global.gc) global.gc();
 }

 module.exports = {
@@ -37,12 +37,12 @@ function edit(domainId, did, title, content, highlight) {
 }

 function del(domainId, did) {
-    // TODO(masnn) delete status
    return Promise.all([
        document.deleteOne(domainId, document.TYPE_DISCUSSION, did),
        document.deleteMulti(domainId, document.TYPE_DISCUSSION_REPLY, {
            parentType: document.TYPE_DISCUSSION, parentId: did,
        }),
+        document.deleteMultiStatus(domainId, document.TYPE_DISCUSSION, { docId: did }),
    ]);
 }

@@ -167,12 +167,16 @@ async function getVnode(domainId, ddoc, handler) {
 }

 async function getListVnodes(domainId, ddocs, handler) {
+    const tasks = [];
    const res = {};
-    for (const ddoc of ddocs) {
-        if (!res[ddoc.parentType]) res[ddoc.parentType] = {};
-        // FIXME no-await-in-loop
-        res[ddoc.parentType][ddoc.parentId] = await getVnode(domainId, ddoc, handler); // eslint-disable-line no-await-in-loop
+    function task(ddoc) {
+        return getVnode(domainId, ddoc, handler).then((vnode) => {
+            if (!res[ddoc.parentType]) res[ddoc.parentType] = {};
+            res[ddoc.parentType][ddoc.parentId] = vnode;
+        });
    }
+    for (const ddoc of ddocs) tasks.push(task(ddoc));
+    await Promise.all(tasks);
    return res;
 }

@@ -1,6 +1,7 @@
 const builtin = require('./builtin');
 const document = require('./document');
 const system = require('./system');
+const token = require('./token');
 const { UserNotFoundError, UserAlreadyExistError } = require('../error');
 const perm = require('../permission');
 const pwhash = require('../lib/hash.hydro');
@@ -25,6 +26,7 @@ class USER {
        this.loginat = udoc.loginat;
        this.bio = udoc.bio || '';
        this.gravatar = udoc.gravatar || '';
+        this.ban = udoc.ban || false;
        this.nAccept = dudoc.nAccept || 0;
        this.nSubmit = dudoc.nSubmit || 0;
        this.rating = dudoc.rating || 1500;
@@ -215,6 +217,13 @@ function deleteRoles(domainId, roles) {
    ]);
 }

+function ban(uid) {
+    return Promise.all([
+        coll.updateOne({ _id: uid }, { $set: { ban: true } }),
+        token.deleteByUid(uid),
+    ]);
+}
+
 function ensureIndexes() {
    return Promise.all([
        coll.createIndex('unameLower', { unique: true }),
@@ -246,5 +255,6 @@ global.Hydro.model.user = module.exports = {
    getInDomain,
    addRole,
    deleteRoles,
+    ban,
    ensureIndexes,
 };
@@ -0,0 +1,64 @@
+// This code format is just for fun. DO NOT DO THIS IN YOUR CODE !
+/* eslint-disable */
+const { ObjectID } = require('bson');
+const document = require('../model/document');
+const discussion = require('../model/discussion');
+const user = require('../model/user');
+const blacklist = require('../model/blacklist');
+const db = require('../service/db.js');
+
+async function _address(ip, bset, uset, dset, dryrun)                                              {
+    if (bset.has(ip))
+        return
+    bset.add(ip)
+    console.log('ip %s', ip)
+    const users = await db.collection('user').find({ loginip: ip }).toArray()
+    const tasks = []
+    for (const udoc of users)                                                                      {
+        tasks.push(_user(udoc._id, bset, uset, dset, dryrun))                                      }
+    await Promise.all(tasks)
+    if (!dryrun)
+        await blacklist.add(ip)                                                                    }
+
+async function _discussion(domainId, did, bset, uset, dset, dryrun)                                {
+    if (dset.has(did))
+        return
+    dset.add(did)
+    const ddoc = await discussion.get(domainId, did)
+    if (!ddoc)
+        return
+    console.log('discussion %s', ddoc['title'])
+    await _user(ddoc['owner'], bset, uset, dset, dryrun)
+    if (ddoc.ip)
+        await _address(ddoc['ip'], bset, uset, dset, dryrun)
+    if (!dryrun)
+        await discussion.delete(domainId, ddoc['docId'])                                           }
+
+async function _user(uid, bset, uset, dset, dryrun)                                                {
+    if (uset.has(uid))
+        return
+    uset.add(uid)
+    const udoc = await user.getById(uid)
+    if (!udoc)
+        return
+    console.log('user %s %s', udoc['_id'], udoc['uname'])
+    await _address(udoc['loginip'], bset, uset, dset, dryrun)
+    const ddocs = await db.collection('document')
+        .find({ docType: document.TYPE_DISCUSSION }).sort({ domainId: 1, docId: 1 }).toArray()
+    const tasks = []
+    for (const ddoc of ddocs)
+        tasks.push(_discussion(ddoc.domainId, ddoc.docId, bset, uset, dset, dryrun))
+    await Promise.all(tasks)
+    if (!dryrun)
+        await user.ban(uid)                                                                        }
+
+async function run({address = null, discuss = null, user = null, dryrun = true})                   {
+    if (address)
+        await _address(address, new Set(), new Set(), new Set(), dryrun)
+    if (discuss)
+        await _discussion(discuss.domainId, new ObjectID(discuss.did),
+                          new Set(), new Set(), new Set(), dryrun)
+    if (user)
+        await _user(user, new Set(), new Set(), new Set(), dryrun)                                 }
+
+global.Hydro.script.blacklist = module.exports = { run };
@@ -1,6 +1,7 @@
 const assert = require('assert');
 const path = require('path');
 const os = require('os');
+const cluster = require('cluster');
 const { ObjectID } = require('bson');
 const Koa = require('koa');
 const yaml = require('js-yaml');
@@ -438,12 +439,17 @@ function Validate(key, func) {
    else validate[key] = [func];
 }

-async function start(port) {
+async function start() {
    app.use(morgan(':method :url :status :res[content-length] - :response-time ms'));
    app.use(router.routes()).use(router.allowedMethods());
    Route('*', Handler);
-    server.listen(port);
-    if (port) console.log('Server listening at: %s', port);
+    if (cluster.worker.id === 1) {
+        const port = await system.get('server.port');
+        server.listen(port);
+        console.log('Server listening at: %s', port);
+    } else {
+        server.listen();
+    }
 }

 global.Hydro.service.server = module.exports = {
@@ -0,0 +1,103 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const zlib = require('zlib');
+const yaml = require('js-yaml');
+
+function ensureDir(dir) {
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir);
+    } else if (!fs.statSync(dir).isDirectory()) {
+        fs.unlinkSync(dir);
+        fs.mkdirSync(dir);
+    }
+}
+
+function root(name) {
+    return path.resolve(process.cwd(), name);
+}
+
+const moduleRoots = [
+    root('.build/module'),
+    root('module'),
+    root(path.resolve(os.homedir(), '.hydro', 'module')),
+    root('.'),
+];
+let moduleRoot;
+for (const i of moduleRoots) {
+    if (fs.existsSync(i) && fs.statSync(i).isDirectory()) {
+        moduleRoot = i;
+        break;
+    }
+}
+function exec() {
+    const files = fs.readdirSync(moduleRoot);
+    const t = ['service', 'lib', 'model', 'handler', 'script'];
+    ensureDir(`${os.tmpdir()}/hydro`);
+    ensureDir(`${os.tmpdir()}/hydro/tmp`);
+    for (const file of files) {
+        if (file.endsWith('.hydro')) {
+            try {
+                const f = fs.readFileSync(root(`${moduleRoot}/${file}`));
+                const s = fs.statSync(root(`${moduleRoot}/${file}`));
+                const m = yaml.safeLoad(zlib.gunzipSync(f));
+                const { id } = m;
+                if (m.os) {
+                    if (!m.os.includes(os.platform().toLowerCase())) {
+                        console.error(`Module load fail: ${id} ${file.split('.')[0]} Unsupported OS`);
+                        // eslint-disable-next-line no-continue
+                        continue;
+                    }
+                }
+                ensureDir(`${os.tmpdir()}/hydro/tmp/${id}`);
+                if (m.locale) {
+                    fs.writeFileSync(
+                        `${os.tmpdir()}/hydro/tmp/${id}/locale.json`,
+                        JSON.stringify(m.locale),
+                    );
+                }
+                if (m.template) {
+                    fs.writeFileSync(
+                        `${os.tmpdir()}/hydro/tmp/${id}/template.json`,
+                        JSON.stringify(m.template),
+                    );
+                }
+                for (const i of t) {
+                    if (m[i]) {
+                        fs.writeFileSync(
+                            `${os.tmpdir()}/hydro/tmp/${id}/${i}.js`,
+                            f[i],
+                        );
+                    }
+                }
+                if (m.file) {
+                    ensureDir(path.resolve(os.tmpdir(), 'hydro', m.id));
+                    for (const n in m.file) {
+                        if (m.file[n] === null) {
+                            ensureDir(path.resolve(os.tmpdir(), 'hydro', m.id, n));
+                        } else {
+                            const e = path.resolve(os.tmpdir(), 'hydro', m.id, n);
+                            fs.writeFileSync(e, Buffer.from(m.file[n], 'base64'), { mode: 755 });
+                        }
+                    }
+                }
+                fs.writeFileSync(
+                    root(`${os.tmpdir()}/hydro/tmp/${id}/hydro.json`),
+                    JSON.stringify({
+                        size: s.size,
+                        version: m.version,
+                        id: m.id,
+                        name: m.name,
+                        description: m.description,
+                    }),
+                );
+            } catch (e) {
+                if (e.code === 'Z_DATA_ERROR') {
+                    console.error(`Module Load Fail: ${file} (File Corrupted)`);
+                } else console.error(`Module Load Fail: ${file} ${e}`);
+            }
+        }
+    }
+}
+
+module.exports = exec;
@@ -246,6 +246,7 @@ async function domainUser(src, report) {
        const docs = await src.collection('domain.user')
            .find().skip(i * 50).limit(50)
            .toArray();
+        const t = [];
        for (const doc of docs) {
            const mapper = {
                _id: '_id',
@@ -277,7 +278,7 @@ async function domainUser(src, report) {
                    docWithoutId[key] = d[key];
                }
            }
-            d.push(dst.collection('document').updateOne(
+            t.push(dst.collection('document.status').updateOne(
                { _id: d._id },
                {
                    $set: {
@@ -287,6 +288,7 @@ async function domainUser(src, report) {
                { upsert: true },
            ));
        }
+        await Promise.all(t);
        await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
    }
 }
@@ -341,9 +343,9 @@ async function task(name, src, report) {
                const mapper = tasks[name];
                for (const key in doc) {
                    if (typeof mapper[key] === 'string') {
-                        res[mapper[key]] = doc[key];
+                        d[mapper[key]] = doc[key];
                    } else if (typeof mapper[key] === 'object') {
-                        res[mapper[key].field] = mapper[key].processer(doc[key]);
+                        d[mapper[key].field] = mapper[key].processer(doc[key]);
                    } else if (mapper[key] === null) {
                        // Ignore this key
                    } else {
@@ -369,7 +371,7 @@ async function task(name, src, report) {
                    res.push((async () => {
                        const data = await dst.collection(name).findOne(query);
                        if (data) {
-                            await dst.collection(name).updateOne(query, { $set: docWithoutId });
+                            await dst.collection(name).updateOne(query, { $set: docWithoutDid });
                        } else if (d._id) {
                            const dat = await dst.collection(name).findOne({ _id: d._id });
                            if (dat) {
@@ -390,7 +392,7 @@ async function task(name, src, report) {
                } else res.push(dst.collection(name).insertOne(d));
            }
        }
-        await Promise.all(res);
+        await Promise.all(res).catch((e) => console.log(e));
        const progress = Math.round(100 * ((i + 1) / (total + 1)));
        if (progress > lastProgress) {
            await report({ progress });