core: consumer: handle database read failure

packages/hydrooj/src/model/task.ts

@@ -16,6 +16,7 @@ const argv = cac().parse();
 
 async function getFirst(query: Filter<Task>) {
     if (process.env.CI) return null;
+    try {
         const q = { ...query };
         const res = await coll.findOneAndDelete(q, { sort: { priority: -1 } });
         if (res.value) {
@@ -23,6 +24,12 @@ async function getFirst(query: Filter<Task>) {
             return res.value;
         }
         return null;
+    } catch (e) {
+        // Usually caused by the database being down
+        // Should recover once it is back
+        logger.error(e);
+        return null;
+    }
 }
 
 export class Consumer {
@@ -163,11 +170,17 @@ export async function apply(ctx: Context) {
         logger.info('No replica set found.');
         // eslint-disable-next-line no-constant-condition
         while (true) {
+            let res;
+            try {
                 // eslint-disable-next-line no-await-in-loop
-            const res = await collEvent.findOneAndUpdate(
+                res = await collEvent.findOneAndUpdate(
                     { expire: { $gt: new Date() }, ack: { $nin: [id] } },
                     { $push: { ack: id } },
                 );
+            } catch (e) {
+                logger.error(e);
+                continue;
+            }
             if (argv.options.showEvent) logger.info('Event: %o', res.value);
             // eslint-disable-next-line no-await-in-loop
             await (res.value ? handleEvent(res.value) : sleep(500));