mirror of
https://gitcode.com/gh_mirrors/hy/Hydro.git
synced 2026-10-20 17:46:28 +08:00
core: logger upd
This commit is contained in:
undefined committed
1 parent
d4088f429a
commit
4d031eb9ab
9 files changed
+141
-131
No files matched your search
@@ -1,7 +1,9 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import { ObjectID, Collection } from 'mongodb';
|
||||
// @ts-ignore
|
||||
import { terminal } from 'terminal-kit';
|
||||
import AdmZip from 'adm-zip';
|
||||
import Queue from 'p-queue';
|
||||
import { Ufdoc } from './interface';
|
||||
import { Logger } from './logger';
|
||||
import { streamToBuffer } from './utils';
|
||||
@@ -60,83 +62,82 @@ const scripts: UpgradeScript[] = [
|
||||
return false;
|
||||
}
|
||||
const coll: Collection<Ufdoc> = db.collection('file');
|
||||
const collFile = db.collection('fs.files');
|
||||
const collChunk = db.collection('fs.chunks');
|
||||
function del(_id: ObjectID) {
|
||||
return Promise.all([
|
||||
coll.deleteOne({ _id }),
|
||||
collFile.deleteOne({ _id }),
|
||||
collChunk.deleteMany({ files_id: _id }),
|
||||
]);
|
||||
}
|
||||
logger.info('Changing storage engine. This may take a long time.');
|
||||
// Problem file and User file
|
||||
try {
|
||||
let savedProgress = system.get('upgrade.file.progress.domain');
|
||||
if (savedProgress) savedProgress = JSON.parse(savedProgress);
|
||||
else savedProgress = { pdocs: [] };
|
||||
const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
|
||||
logger.info('Total found %d domains.', ddocs.length);
|
||||
for (let i = 0; i < ddocs.length; i++) {
|
||||
const ddoc = ddocs[i];
|
||||
logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
|
||||
const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
|
||||
const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
|
||||
for (let j = 0; j < pdocs.length; j++) {
|
||||
const pdoc = pdocs[j];
|
||||
domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
|
||||
if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
|
||||
try {
|
||||
const file = await streamToBuffer(gridfs.openDownloadStream(pdoc.data));
|
||||
const zip = new AdmZip(file);
|
||||
const entries = zip.getEntries();
|
||||
await Promise.all(entries.map(
|
||||
(entry) => storage.put(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.name}`, entry.getData()),
|
||||
));
|
||||
} catch (e) {
|
||||
if (e.toString().includes('FileNotFound')) {
|
||||
logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
|
||||
} else throw e;
|
||||
let savedProgress = system.get('upgrade.file.progress.domain');
|
||||
if (savedProgress) savedProgress = JSON.parse(savedProgress);
|
||||
else savedProgress = { pdocs: [] };
|
||||
const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
|
||||
logger.info('Total found %d domains.', ddocs.length);
|
||||
for (let i = 0; i < ddocs.length; i++) {
|
||||
const ddoc = ddocs[i];
|
||||
logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
|
||||
const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
|
||||
const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
|
||||
for (let j = 0; j < pdocs.length; j++) {
|
||||
const pdoc = pdocs[j];
|
||||
domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
|
||||
if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
|
||||
try {
|
||||
const [file, current] = await Promise.all([
|
||||
streamToBuffer(gridfs.openDownloadStream(pdoc.data)),
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true),
|
||||
]);
|
||||
const zip = new AdmZip(file);
|
||||
const entries = zip.getEntries();
|
||||
if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) {
|
||||
await storage.del(current.map((entry) => entry.prefix + entry.name));
|
||||
const queue = new Queue({ concurrency: 5 });
|
||||
const tasks = [];
|
||||
for (const entry of entries) {
|
||||
tasks.push(queue.add(() => storage.put(
|
||||
`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.entryName}`,
|
||||
entry.getData(),
|
||||
)));
|
||||
}
|
||||
await Promise.all(tasks);
|
||||
}
|
||||
savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
|
||||
} catch (e) {
|
||||
if (e.toString().includes('FileNotFound')) {
|
||||
logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
|
||||
} else throw e;
|
||||
}
|
||||
system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
|
||||
domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
|
||||
savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
|
||||
}
|
||||
domainProgress.stop();
|
||||
system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
|
||||
domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
|
||||
}
|
||||
const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
|
||||
const userfileProgress = terminal.progressBar({
|
||||
items: udocs.length, title: 'Users', y: 3,
|
||||
});
|
||||
for (const udoc of udocs) {
|
||||
userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
|
||||
const ufdocs = await coll.find({ owner: udoc._id }).toArray();
|
||||
const currentUser = terminal.progressBar({
|
||||
items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
|
||||
});
|
||||
const namelist = [];
|
||||
for (const ufdoc of ufdocs) {
|
||||
currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`);
|
||||
ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString();
|
||||
const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id));
|
||||
if (!namelist.includes(ufdoc.filename)) {
|
||||
await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file);
|
||||
} else {
|
||||
await storage.put(`user/${udoc._id}/${ufdoc._id}`, file);
|
||||
}
|
||||
namelist.push(ufdoc.filename);
|
||||
currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`);
|
||||
}
|
||||
currentUser.stop();
|
||||
userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`);
|
||||
}
|
||||
userfileProgress.stop();
|
||||
return true;
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
return false;
|
||||
domainProgress.stop();
|
||||
}
|
||||
const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
|
||||
const userfileProgress = terminal.progressBar({
|
||||
items: udocs.length, title: 'Users', y: 3,
|
||||
});
|
||||
for (const udoc of udocs) {
|
||||
userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
|
||||
const ufdocs = await coll.find({ owner: udoc._id }).toArray();
|
||||
const currentUser = terminal.progressBar({
|
||||
items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
|
||||
});
|
||||
const namelist = [];
|
||||
for (const ufdoc of ufdocs) {
|
||||
currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`);
|
||||
ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString();
|
||||
const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id));
|
||||
if (!namelist.includes(ufdoc.filename)) {
|
||||
await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file);
|
||||
} else {
|
||||
await storage.put(`user/${udoc._id}/${ufdoc._id}`, file);
|
||||
}
|
||||
namelist.push(ufdoc.filename);
|
||||
currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`);
|
||||
}
|
||||
currentUser.stop();
|
||||
userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`);
|
||||
}
|
||||
userfileProgress.stop();
|
||||
logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.');
|
||||
return true;
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
Reference in new issue
Block a user