wip

extra/reset-migrate-aggregate-table-status.js

@@ -0,0 +1,24 @@
+const { R } = require("redbean-node");
+const Database = require("../server/database");
+const args = require("args-parser")(process.argv);
+const { Settings } = require("../server/settings");
+
+const main = async () => {
+    console.log("Connecting the database");
+    Database.initDataDir(args);
+    await Database.connect(false, false, true);
+
+    console.log("Deleting all data from aggregate tables");
+    await R.exec("DELETE FROM stat_minutely");
+    await R.exec("DELETE FROM stat_hourly");
+    await R.exec("DELETE FROM stat_daily");
+
+    console.log("Resetting the aggregate table state");
+    await Settings.set("migrateAggregateTableState", "");
+
+    await Database.close();
+    console.log("Done");
+};
+
+main();
+

package.json

@@ -68,7 +68,8 @@
         "sort-contributors": "node extra/sort-contributors.js",
         "quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2",
         "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate",
-        "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X"
+        "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X",
+        "reset-migrate-aggregate-table-status": "node extra/reset-migrate-aggregate-table-status.js"
     },
     "dependencies": {
         "@grpc/grpc-js": "~1.8.22",

server/database.js

@@ -739,35 +739,26 @@ class Database {
         // Add a setting for 2.0.0-dev users to skip this migration
         if (process.env.SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE === "1") {
             log.warn("db", "SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE is set to 1, skipping aggregate table migration forever (for 2.0.0-dev users)");
-            await Settings.set("migratedAggregateTable", true);
+            await Settings.set("migrateAggregateTableState", "migrated");
         }
 
-        // Empty the aggregate table if FORCE_MIGRATE_AGGREGATE_TABLE is set to 1
+        let migrateState = await Settings.get("migrateAggregateTableState");
-        if (process.env.FORCE_MIGRATE_AGGREGATE_TABLE === "1") {
-            log.warn("db", "FORCE_MIGRATE_AGGREGATE_TABLE is set to 1, forcing aggregate table migration");
-            await R.exec("DELETE FROM stat_minutely");
-            await R.exec("DELETE FROM stat_hourly");
-            await R.exec("DELETE FROM stat_daily");
-        }
 
-        //
+        // Skip if already migrated
-        let migrated = await Settings.get("migratedAggregateTable");
+        // If it is migrating, it possibly means the migration was interrupted, or the migration is in progress
-
+        if (migrateState === "migrated") {
-        if (migrated) {
+            log.debug("db", "Migrated aggregate table already, skip");
-            log.debug("db", "Migrated, skip migration");
             return;
+        } else if (migrateState === "migrating") {
+            log.warn("db", "Aggregate table migration is already in progress, or it was interrupted");
+            throw new Error("Aggregate table migration is already in progress");
         }
 
+        await Settings.set("migrateAggregateTableState", "migrating");
+
         log.info("db", "Migrating Aggregate Table");
 
-        log.info("db", "Getting list of unique dates and monitors");
+        log.info("db", "Getting list of unique monitors");
-
-        // Get a list of unique dates from the heartbeat table, using raw sql
-        let dates = await R.getAll(`
-            SELECT DISTINCT DATE(time) AS date
-            FROM heartbeat
-            ORDER BY date ASC
-        `);
 
         // Get a list of unique monitors from the heartbeat table, using raw sql
         let monitors = await R.getAll(`
@@ -786,11 +777,23 @@ class Database {
             }
         }
 
+        let progressPercent = 0;
+        let part = 100 / monitors.length;
+        let i = 1;
         for (let monitor of monitors) {
 
-            // TODO: Get the date list for each monitor
+            // TODO: Get two or three days at the same to speed up???
-            for (let date of dates) {
+            // Get a list of unique dates from the heartbeat table, using raw sql
+            let dates = await R.getAll(`
+                SELECT DISTINCT DATE(time) AS date
+                FROM heartbeat
+                WHERE monitor_id = ?
+                ORDER BY date ASC
+            `, [
+                monitor.monitor_id
+            ]);
 
+            for (let date of dates) {
                 // New Uptime Calculator
                 let calculator = new UptimeCalculator();
                 calculator.monitorID = monitor.monitor_id;
@@ -806,16 +809,34 @@ class Database {
                 `, [ monitor.monitor_id, date.date ]);
 
                 if (heartbeats.length > 0) {
-                    log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`);
+                    log.info("db", `[DON'T STOP] Migrating monitor data ${monitor.monitor_id} - ${date.date} [${progressPercent.toFixed(2)}%][${i}/${monitors.length}]`);
                 }
 
                 for (let heartbeat of heartbeats) {
                     await calculator.update(heartbeat.status, parseFloat(heartbeat.ping), dayjs(heartbeat.time));
                 }
+
+                progressPercent += (Math.round(part / dates.length * 100) / 100);
+
+                // Lazy to fix the floating point issue, it is acceptable since it is just a progress bar
+                if (progressPercent > 100) {
+                    progressPercent = 100;
                 }
             }
 
-        await Settings.set("migratedAggregateTable", true);
+            i++;
+        }
+
+        // TODO: Remove all non-important heartbeats from heartbeat table
+        log.info("db", "[DON'T STOP] Deleting all data from heartbeat table");
+
+        await Settings.set("migrateAggregateTableState", "migrated");
+
+        if (monitors.length > 0) {
+            log.info("db", "Aggregate Table Migration Completed");
+        } else {
+            log.info("db", "No data to migrate");
+        }
     }
 
 }

server/server.js

@@ -1599,18 +1599,20 @@ let needSetup = false;
 
     await server.start();
 
-    server.httpServer.listen(port, hostname, () => {
+    server.httpServer.listen(port, hostname, async () => {
         if (hostname) {
             log.info("server", `Listening on ${hostname}:${port}`);
         } else {
             log.info("server", `Listening on ${port}`);
         }
-        startMonitors();
+        await startMonitors();
+
+        // Put this here. Start background jobs after the db and server is ready to prevent clear up during db migration.
+        await initBackgroundJobs();
+
         checkVersion.startInterval();
     });
 
-    await initBackgroundJobs();
-
     // Start cloudflared at the end if configured
     await cloudflaredAutoStart(cloudflaredToken);