core: use aws-sdk/s3 instead of minio

.eslintrc.yaml

@@ -26,6 +26,7 @@ rules:
   '@typescript-eslint/no-throw-literal': 0
   '@typescript-eslint/return-await': 0
 
+  no-multi-str: 0
   '@typescript-eslint/indent':
     - warn
     - 4

packages/hydrooj/package.json

@@ -12,11 +12,14 @@
     },
     "preferUnplugged": true,
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.178.0",
+        "@aws-sdk/middleware-endpoint": "^3.178.0",
+        "@aws-sdk/s3-presigned-post": "^3.178.0",
+        "@aws-sdk/s3-request-presigner": "^3.178.0",
         "@graphql-tools/schema": "^8.5.1",
         "@hydrooj/utils": "workspace:*",
         "adm-zip": "0.5.5",
         "cac": "^6.7.14",
-        "cookies": "^0.8.0",
         "cordis": "^2.2.0",
         "detect-browser": "^5.3.0",
         "emoji-regex": "^10.1.0",
@@ -35,7 +38,6 @@
         "lodash": "^4.17.21",
         "lru-cache": "7.12.0",
         "mime-types": "^2.1.35",
-        "minio": "7.0.32",
         "moment-timezone": "^0.5.37",
         "mongodb": "^3.7.3",
         "nanoid": "^4.0.0",

packages/hydrooj/src/interface.ts

@@ -1,6 +1,5 @@
 import type fs from 'fs';
 import type { Dictionary, NumericDictionary } from 'lodash';
-import type { ItemBucketMetadata } from 'minio';
 import type { Cursor, ObjectID } from 'mongodb';
 import { Context } from './context';
 import type { ProblemDoc } from './model/problem';
@@ -583,7 +582,7 @@ export interface FileNode {
     autoDelete?: Date;
     owner?: number;
     operator?: number[];
-    meta?: ItemBucketMetadata;
+    meta?: Record<string, string | number>;
 }
 
 export interface EventDoc {

packages/hydrooj/src/service/storage.ts

@@ -1,14 +1,17 @@
 import { dirname, resolve } from 'path';
-import { Readable } from 'stream';
+import { PassThrough, Readable } from 'stream';
 import { URL } from 'url';
 import {
-    copyFile, createReadStream, ensureDir,
+    CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand,
+    GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client,
+} from '@aws-sdk/client-s3';
+import { createPresignedPost } from '@aws-sdk/s3-presigned-post';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import {
+    copyFile, createReadStream, createWriteStream, ensureDir,
     remove, stat, writeFile,
 } from 'fs-extra';
 import { lookup } from 'mime-types';
-import {
-    BucketItem, Client, CopyConditions, ItemBucketMetadata,
-} from 'minio';
 import { Logger } from '../logger';
 import { builtinConfig } from '../settings';
 import { MaybeArray } from '../typeutils';
@@ -27,31 +30,6 @@ interface StorageOptions {
     endPointForJudge?: string;
 }
 
-interface EndpointConfig {
-    endPoint: string;
-    port: number;
-    useSSL: boolean;
-}
-
-function parseMainEndpointUrl(endpoint: string): EndpointConfig {
-    if (!endpoint) throw new Error('Empty endpoint');
-    const url = new URL(endpoint);
-    const result: Partial<EndpointConfig> = {};
-    if (url.pathname !== '/') throw new Error('Main endpoint URL of a sub-directory is not supported.');
-    if (url.username || url.password || url.hash || url.search) {
-        throw new Error('Authorization, search parameters and hash are not supported for main endpoint URL.');
-    }
-    if (url.protocol === 'http:') result.useSSL = false;
-    else if (url.protocol === 'https:') result.useSSL = true;
-    else {
-        throw new Error(
-            `Invalid protocol "${url.protocol}" for main endpoint URL. Only HTTP and HTTPS are supported.`,
-        );
-    }
-    result.endPoint = url.hostname;
-    result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80;
-    return result as EndpointConfig;
-}
 function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string {
     if (!endpoint) return (originalUrl) => originalUrl;
     const pathonly = endpoint.startsWith('/');
@@ -82,7 +60,7 @@ export function encodeRFC5987ValueChars(str: string) {
 }
 
 class RemoteStorageService {
-    public client: Client;
+    public client: S3Client;
     public error = '';
     public opts: StorageOptions;
     private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>;
@@ -110,19 +88,15 @@ class RemoteStorageService {
                 endPointForUser,
                 endPointForJudge,
             };
-            this.client = new Client({
+            this.client = new S3Client({
-                ...parseMainEndpointUrl(this.opts.endPoint),
+                endpoint: this.opts.endPoint,
-                pathStyle: this.opts.pathStyle,
+                region: this.opts.region,
-                accessKey: this.opts.accessKey,
+                forcePathStyle: this.opts.pathStyle,
-                secretKey: this.opts.secretKey,
+                credentials: {
+                    accessKeyId: this.opts.accessKey,
+                    secretAccessKey: this.opts.secretKey,
+                },
             });
-            try {
-                const exists = await this.client.bucketExists(this.opts.bucket);
-                if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region);
-            } catch (e) {
-                // Some platform doesn't support bucketExists & makeBucket API.
-                // Ignore this error.
-            }
             this.replaceWithAlternativeUrlFor = {
                 user: parseAlternativeEndpointUrl(this.opts.endPointForUser),
                 judge: parseAlternativeEndpointUrl(this.opts.endPointForJudge),
@@ -137,88 +111,90 @@ class RemoteStorageService {
         }
     }
 
-    async put(target: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) {
+    async put(target: string, file: string | Buffer | Readable, meta: Record<string, string> = {}) {
         if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
         if (typeof file === 'string') file = createReadStream(file);
-        try {
+        await this.client.send(new PutObjectCommand({
-            await this.client.putObject(this.opts.bucket, target, file, meta);
+            Bucket: this.opts.bucket,
-        } catch (e) {
+            Body: file,
-            e.stack = new Error().stack;
+            Key: target,
-            throw e;
+            Metadata: meta,
-        }
+        }));
     }
 
     async get(target: string, path?: string) {
         if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        try {
+        const res = await this.client.send(new GetObjectCommand({
-            if (path) return await this.client.fGetObject(this.opts.bucket, target, path);
+            Bucket: this.opts.bucket,
-            return await this.client.getObject(this.opts.bucket, target);
+            Key: target,
-        } catch (e) {
+        }));
-            e.stack = new Error().stack;
+        if (!res.Body) throw new Error();
-            throw e;
+        const stream = res.Body as Readable;
+        if (path) {
+            await new Promise((end, reject) => {
+                const file = createWriteStream(path);
+                stream.on('error', reject);
+                stream.on('end', () => {
+                    file.close();
+                    end(null);
+                });
+                stream.pipe(file);
+            });
+            return null;
         }
+        const p = new PassThrough();
+        stream.pipe(p);
+        return p;
     }
 
     async del(target: string | string[]) {
         if (typeof target === 'string') {
             if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
         } else if (target.find((t) => t.includes('..') || t.includes('//'))) throw new Error('Invalid path');
-        try {
+        if (typeof target === 'string') {
-            if (typeof target === 'string') return await this.client.removeObject(this.opts.bucket, target);
+            return await this.client.send(new DeleteObjectCommand({
-            return await this.client.removeObjects(this.opts.bucket, target);
+                Bucket: this.opts.bucket,
-        } catch (e) {
+                Key: target,
-            e.stack = new Error().stack;
+            }));
-            throw e;
         }
+        return await this.client.send(new DeleteObjectsCommand({
+            Bucket: this.opts.bucket,
+            Delete: {
+                Objects: target.map((i) => ({ Key: i })),
+            },
+        }));
     }
 
     /** @deprecated use StorageModel.list instead. */
-    async list(target: string, recursive = true) {
+    async list() {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        throw new Error('listObjectsAPI was no longer supported in hydrooj@4. Please use hydrooj@3 to migrate your files first.');
-        try {
-            const stream = this.client.listObjects(this.opts.bucket, target, recursive);
-            return await new Promise<BucketItem[]>((r, reject) => {
-                const results: BucketItem[] = [];
-                stream.on('data', (result) => {
-                    if (result.size) {
-                        results.push({
-                            ...result,
-                            prefix: target,
-                            name: result.name.split(target)[1],
-                        });
-                    }
-                });
-                stream.on('end', () => r(results));
-                stream.on('error', reject);
-            });
-        } catch (e) {
-            e.stack = new Error().stack;
-            throw e;
-        }
     }
 
     async getMeta(target: string) {
         if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        try {
+        const res = await this.client.send(new HeadObjectCommand({
-            const result = await this.client.statObject(this.opts.bucket, target);
+            Bucket: this.opts.bucket,
-            return { ...result.metaData, ...result };
+            Key: target,
-        } catch (e) {
+        }));
-            e.stack = new Error().stack;
+        return {
-            throw e;
+            size: res.ContentLength,
-        }
+            lastModified: res.LastModified,
+            etag: res.ETag,
+            metaData: res.Metadata,
+        };
     }
 
     async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise<string> {
         if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
         try {
-            const headers: Record<string, string> = {};
+            const url = await getSignedUrl(this.client, new GetObjectCommand({
-            if (filename) headers['response-content-disposition'] = `attachment; filename="${encodeRFC5987ValueChars(filename)}"`;
+                Bucket: this.opts.bucket,
-            const url = await this.client.presignedGetObject(
+                Key: target,
-                this.opts.bucket,
+                ResponseContentDisposition: filename ? `attachment; filename="${encodeRFC5987ValueChars(filename)}"` : '',
-                target,
+            }), {
-                noExpire ? 24 * 60 * 60 * 7 : 10 * 60,
+                expiresIn: noExpire ? 24 * 60 * 60 * 7 : 10 * 60,
-                headers,
+            });
-            );
+            console.log(url);
             if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url);
             return url;
         } catch (e) {
@@ -228,28 +204,34 @@ class RemoteStorageService {
     }
 
     async signUpload(target: string, size: number) {
-        const policy = this.client.newPostPolicy();
+        const { url, fields } = await createPresignedPost(this.client, {
-        policy.setBucket(this.opts.bucket);
+            Bucket: this.opts.bucket,
-        policy.setKey(target);
+            Key: target,
-        policy.setExpires(new Date(Date.now() + 30 * 60 * 1000));
+            Conditions: [
-        if (size) policy.setContentLengthRange(size - 50, size + 50);
+                { $key: target },
-        const policyResult = await this.client.presignedPostPolicy(policy);
+                { acl: 'public-read' },
+                { bucket: this.opts.bucket },
+                ['content-length-range', size - 50, size + 50],
+            ],
+            Fields: {
+                acl: 'public-read',
+            },
+            Expires: 600,
+        });
         return {
-            url: this.replaceWithAlternativeUrlFor.user(policyResult.postURL),
+            url: this.replaceWithAlternativeUrlFor.user(url),
-            extraFormData: policyResult.formData,
+            fields,
         };
     }
 
     async copy(src: string, target: string) {
         if (src.includes('..') || src.includes('//')) throw new Error('Invalid path');
         if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        try {
+        return await this.client.send(new CopyObjectCommand({
-            const result = await this.client.copyObject(this.opts.bucket, target, src, new CopyConditions());
+            Bucket: this.opts.bucket,
-            return result;
+            Key: target,
-        } catch (e) {
+            CopySource: src,
-            e.stack = new Error().stack;
+        }));
-            throw e;
-        }
     }
 }
 

packages/hydrooj/src/upgrade.ts

@@ -1,15 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable @typescript-eslint/naming-convention */
-import AdmZip from 'adm-zip';
 import yaml from 'js-yaml';
 import { pick } from 'lodash';
-import { BucketItem } from 'minio';
 import moment from 'moment';
-import { GridFSBucket, ObjectID } from 'mongodb';
+import { ObjectID } from 'mongodb';
-import Queue from 'p-queue';
-import { convertIniConfig } from '@hydrooj/utils/lib/cases';
-import { size } from '@hydrooj/utils/lib/utils';
 import { buildContent } from './lib/content';
 import { Logger } from './logger';
 import { PERM, PRIV, STATUS } from './model/builtin';
@@ -20,21 +15,23 @@ import domain from './model/domain';
 import MessageModel from './model/message';
 import problem from './model/problem';
 import RecordModel from './model/record';
-import StorageModel from './model/storage';
 import * as system from './model/system';
 import TaskModel from './model/task';
 import user from './model/user';
 import {
-    iterateAllDomain, iterateAllProblem, iterateAllPsdoc, iterateAllUser,
+    iterateAllDomain, iterateAllProblem, iterateAllUser,
 } from './pipelineUtils';
 import db from './service/db';
-import storage from './service/storage';
 import { setBuiltinConfig } from './settings';
-import { streamToBuffer } from './utils';
 import welcome from './welcome';
 
 const logger = new Logger('upgrade');
 type UpgradeScript = void | (() => Promise<boolean | void>);
+const unsupportedUpgrade = async function _26_27() {
+    const _FRESH_INSTALL_IGNORE = 1;
+    throw new Error('This upgrade was no longer supported in hydrooj@4. \
+Please use hydrooj@3 to perform these upgrades before upgrading to v4');
+};
 
 const scripts: UpgradeScript[] = [
     // Mark as used
@@ -47,288 +44,7 @@ const scripts: UpgradeScript[] = [
         // TODO discussion node?
         return true;
     },
-    // Add history column to ddoc,drdoc,psdoc
+    ...new Array(25).fill(unsupportedUpgrade),
-    async function _2_3() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllDomain(async (d) => {
-            const bulk = document.coll.initializeUnorderedBulkOp();
-            await document.getMulti(d._id, document.TYPE_DISCUSSION).forEach((ddoc) => {
-                bulk.find({ _id: ddoc._id }).updateOne({ $set: { history: [] } });
-            });
-            await document.getMulti(d._id, document.TYPE_DISCUSSION_REPLY).forEach((drdoc) => {
-                bulk.find({ _id: drdoc._id }).updateOne({ $set: { history: [] } });
-            });
-            await document.getMulti(d._id, document.TYPE_PROBLEM_SOLUTION).forEach((psdoc) => {
-                bulk.find({ _id: psdoc._id }).updateOne({ $set: { history: [] } });
-            });
-            if (bulk.length) await bulk.execute();
-        });
-        return true;
-    },
-    async function _3_4() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await db.collection('document').updateMany({ pid: /^\d+$/i }, { $unset: { pid: '' } });
-        return true;
-    },
-    async function _4_5() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        if (storage.error) {
-            logger.error('Cannot upgrade. Please change storage config.');
-            return false;
-        }
-        logger.info('Changing storage engine. This may take a long time.');
-        // Problem file and User file
-        let savedProgress = system.get('upgrade.file.progress.domain');
-        if (savedProgress) savedProgress = JSON.parse(savedProgress);
-        else savedProgress = { pdocs: [] };
-        const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
-        logger.info('Found %d domains.', ddocs.length);
-        const gridfs = new GridFSBucket(db.db);
-        for (let i = 0; i < ddocs.length; i++) {
-            const ddoc = ddocs[i];
-            logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
-            const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
-            for (let j = 0; j < pdocs.length; j++) {
-                const pdoc = pdocs[j];
-                console.log(`${pdoc.docId}: ${pdoc.title}`);
-                if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
-                    try {
-                        const [file, current] = await Promise.all([
-                            streamToBuffer(gridfs.openDownloadStream(pdoc.data)),
-                            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`) as any,
-                        ]);
-                        const zip = new AdmZip(file);
-                        const entries = zip.getEntries();
-                        if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) {
-                            await storage.del(current.map((entry) => entry.prefix + entry.name));
-                            const queue = new Queue({ concurrency: 5 });
-                            await Promise.all(entries.map(
-                                (entry) => queue.add(() => problem.addTestdata(pdoc.domainId, pdoc.docId, entry.entryName, entry.getData())),
-                            ));
-                        }
-                    } catch (e) {
-                        if (e.toString().includes('FileNotFound')) {
-                            logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
-                        } else throw e;
-                    }
-                    savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
-                }
-                system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
-                console.log(`${pdoc.docId}: ${pdoc.title} done`);
-            }
-        }
-        logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.');
-        return true;
-    },
-    async function _5_6() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllDomain(async (d) => {
-            const bulk = document.coll.initializeUnorderedBulkOp();
-            const pdocs = await document.getMulti(d._id, document.TYPE_PROBLEM).project({ domainId: 1, docId: 1 }).toArray();
-            for (const pdoc of pdocs) {
-                const data = await storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true);
-                bulk.find({ _id: pdoc._id }).updateOne({ $set: { data } });
-            }
-            if (bulk.length) await bulk.execute();
-        });
-        return true;
-    },
-    async function _6_7() {
-        // Issue #58
-        const _FRESH_INSTALL_IGNORE = 1;
-        await domain.edit('system', { owner: 1 });
-        return true;
-    },
-    async function _7_8() {
-        return true; // invalid
-    },
-    async function _8_9() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => {
-            logger.info('%s/%s', pdoc.domainId, pdoc.docId);
-            const [data, additional_file] = await Promise.all([
-                storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`) as any,
-                storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`) as any,
-            ]);
-            await problem.edit(
-                pdoc.domainId, pdoc.docId,
-                { data: data.map((d) => ({ ...d, _id: d.name })), additional_file: additional_file.map((d) => ({ ...d, _id: d.name })) },
-            );
-            if (!pdoc.config) return;
-            if (data.map((d) => d.name).includes('config.yaml')) return;
-            const cfg = yaml.dump(pdoc.config);
-            await problem.addTestdata(pdoc.domainId, pdoc.docId, 'config.yaml', Buffer.from(cfg));
-        });
-        return true;
-    },
-    async function _9_10() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem([], async (pdoc) => {
-            logger.info('%s/%s', pdoc.domainId, pdoc.docId);
-            const [data, additional_file] = await Promise.all([
-                storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
-                storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`),
-            ]) as any;
-            for (let i = 0; i < data.length; i++) {
-                data[i]._id = data[i].name;
-            }
-            for (let i = 0; i < additional_file.length; i++) {
-                additional_file[i]._id = additional_file[i].name;
-            }
-            return { data, additional_file };
-        });
-        return true;
-    },
-    // Move category to tag
-    async function _10_11() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem(['tag', 'category'], async (pdoc) => {
-            await document.coll.updateOne(
-                { domainId: pdoc.domainId, docId: pdoc.docId },
-                {
-                    $set: { tag: [...pdoc.tag || [], ...pdoc.category || []] },
-                    $unset: { category: '' },
-                },
-            );
-        });
-        return true;
-    },
-    // Set null tag to []
-    async function _11_12() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await db.collection('document').updateMany({ docType: 10, tag: null }, { $set: { tag: [] } });
-        return true;
-    },
-    // Update problem difficulty
-    async function _12_13() {
-        return true;
-    },
-    // Set domain owner perm
-    async function _13_14() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllDomain((ddoc) => domain.setUserRole(ddoc._id, ddoc.owner, 'root'));
-        await db.collection('domain.user').updateMany({ role: 'admin' }, { $set: { role: 'root' } });
-        return true;
-    },
-    async function _14_15() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await db.collection('domain.user').deleteMany({ uid: null });
-        await db.collection('domain.user').deleteMany({ uid: 0 });
-        return true;
-    },
-    async function _15_16() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem(['data', 'additional_file'], async (pdoc) => {
-            const $set: any = {};
-            const td = (pdoc.data || []).filter((f) => !!f.name);
-            if (JSON.stringify(td) !== JSON.stringify(pdoc.data)) $set.data = td;
-            const af = (pdoc.additional_file || []).filter((f) => !!f.name);
-            if (JSON.stringify(af) !== JSON.stringify(pdoc.additional_file)) $set.additional_file = af;
-            if (Object.keys($set).length) await problem.edit(pdoc.domainId, pdoc.docId, $set);
-        });
-        return true;
-    },
-    null,
-    null,
-    async function _18_19() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem(['content'], async (pdoc) => {
-            if (typeof pdoc.content !== 'string') {
-                await problem.edit(pdoc.domainId, pdoc.docId, { content: JSON.stringify(pdoc.content) });
-            }
-        });
-        return true;
-    },
-    null,
-    async function _20_21() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem([], async (pdoc) => {
-            let config: string;
-            try {
-                const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yaml`);
-                config = (await streamToBuffer(file)).toString('utf-8');
-                logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yaml`);
-            } catch (e) {
-                try {
-                    const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yml`);
-                    config = (await streamToBuffer(file)).toString('utf-8');
-                    logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yml`);
-                } catch (err) {
-                    try {
-                        const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.ini`);
-                        config = yaml.dump(convertIniConfig((await streamToBuffer(file)).toString('utf-8')));
-                        logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.ini`);
-                    } catch (error) {
-                        logger.warn('Config for %s/%s(%s) not found', pdoc.domainId, pdoc.docId, pdoc.pid || pdoc.docId);
-                        // no config found
-                    }
-                }
-            }
-            if (config) await problem.edit(pdoc.domainId, pdoc.docId, { config });
-        });
-        return true;
-    },
-    async function _21_22() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        const coll = db.collection('domain');
-        await iterateAllDomain(async (ddoc) => {
-            if (ddoc.join) {
-                await coll.updateOne(pick(ddoc, '_id'), { $set: { _join: ddoc.join }, $unset: { join: '' } });
-            }
-        });
-        return true;
-    },
-    async function _22_23() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllUser(async (udoc) => {
-            if (!udoc.gravatar) return;
-            await user.setById(udoc._id, { avatar: `gravatar:${udoc.gravatar}` }, { gravatar: '' });
-        });
-        return true;
-    },
-    async function _23_24() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await db.collection('oplog').updateMany({}, { $set: { type: 'delete', operateIp: '127.0.0.1' } });
-        return true;
-    },
-    async function _24_25() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllDomain(async (ddoc) => {
-            if (typeof ddoc.host === 'string') await domain.edit(ddoc._id, { host: [ddoc.host] });
-        });
-        return true;
-    },
-    async function _25_26() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllPsdoc({ rid: { $exists: true } }, async (psdoc) => {
-            const rdoc = await RecordModel.get(psdoc.domainId, psdoc.rid);
-            await document.setStatus(psdoc.domainId, document.TYPE_PROBLEM, rdoc.pid, rdoc.uid, { score: rdoc.score });
-        });
-        return true;
-    },
-    async function _26_27() {
-        const _FRESH_INSTALL_IGNORE = 1;
-        const stream = storage.client.listObjects(storage.opts.bucket, '', true);
-        await new Promise<BucketItem[]>((resolve, reject) => {
-            stream.on('data', (result) => {
-                if (result.size) {
-                    logger.debug('File found: %s %s', result.name, size(result.size));
-                    StorageModel.coll.insertOne({
-                        _id: result.name,
-                        path: result.name,
-                        size: result.size,
-                        lastModified: result.lastModified,
-                        lastUsage: new Date(),
-                        etag: result.etag,
-                        meta: {},
-                    });
-                }
-            });
-            stream.on('end', () => resolve(null));
-            stream.on('error', reject);
-        });
-        return true;
-    },
     async function _27_28() {
         const _FRESH_INSTALL_IGNORE = 1;
         const cursor = document.coll.find({ docType: document.TYPE_DISCUSSION });