Moved toke log to it's own DB collection w/ cached statistics. Tokebot statistics page-load time decreased by up to 20-30x

src/schemas/tokebot/tokeSchema.js

@@ -0,0 +1,219 @@
+/*Canopy - The next generation of stoner streaming software
+Copyright (C) 2024-2025 Rainbownapkin and the TTN Community
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.*/
+
+//NPM Imports
+const {mongoose} = require('mongoose');
+
+//Local Imports
+const config = require('./../../../config.json');
+const loggerUtils = require('../../utils/loggerUtils');
+
+/**
+ * DB Schema for single document for keeping track of a single toke
+ */
+const tokeSchema = new mongoose.Schema({
+    toke: {
+        type: mongoose.SchemaTypes.Map,
+        required: true,
+        default: new Map()
+    },
+    date: {
+        type: mongoose.SchemaTypes.Date,
+        required: true,
+        default: new Date()
+    }
+});
+
+//statics
+/**
+ * Cached map containing counts of individual toke commands
+ */
+tokeSchema.statics.tokeMap = new Map();
+
+/**
+ * Cached number of total tokes
+ */
+tokeSchema.statics.count = 0;
+
+/**
+ * Cached number of times a user has successfully ran a '!toke' command
+ * Not to be confused with tokeSchema.statics.count, which counts total amount of tokes called out
+ */
+tokeSchema.statics.commandCount = 0;
+
+/**
+ * Calculates cached toke map from existing 
+ */
+tokeSchema.statics.calculateTokeMap = async function(){
+    //Pull full toke collection
+    const tokes = await this.find();
+
+    //Drop existing toke map
+    this.tokeMap = new Map();
+
+    //Iterate through DB of tokes
+    for(const toke of tokes){
+        //Increment toke count
+        this.count++;
+
+        //For each command callout
+        for(const command of toke.toke){
+            //Increment Command Count
+            this.commandCount++;
+
+            //Pull current count of respective toke command
+            let curCount = this.tokeMap.get(command[1]);
+
+            //If this is an unset toke command
+            if(curCount == null){
+                //Set it to one
+                this.tokeMap.set(command[1], 1);
+            //Otherwise
+            }else{
+                //Increment the existing count
+                this.tokeMap.set(command[1], ++curCount);
+            }
+        }
+    }
+
+    //Display calculated toke sats for funsies
+    if(config.verbose){
+        console.log(`Processed ${this.commandCount} toke command callouts accross ${await this.estimatedDocumentCount()} tokes.`);
+    }
+}
+
+/**
+ * Tattoos toke into toke DB colleciton
+ * 
+ * We use this instead of a pre-save function as we need to fuck w/ statics
+ */
+tokeSchema.statics.tattooToke = async function(toke){
+    //Write toke to DB
+    await this.create({toke});
+
+    //Increment RAM-backed toke count
+    this.count++;
+
+    //Iterate through tokers
+    for(const curToke of toke){
+        //Pull current toke count
+        let curCount = this.tokeMap.get(curToke[1]);
+
+        //If this command hasn't been counted
+        if(curCount == null){
+            //Set new command count to one
+            this.tokeMap.set(curToke[1], 1);
+        }else{
+            //Increment current toke count and commit it to the RAM-backed tokeMap
+            this.tokeMap.set(curToke[1], ++curCount);
+        }
+
+        //Increment RAM-Backed command count
+        this.commandCount++;
+    }
+}
+
+/**
+ * Ingests legacy tokes handed over by the migration model
+ * @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
+ */
+tokeSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
+    //If migration is disabled
+    if(!config.migrate){
+        //BAIL!
+        return;
+    }
+
+    try{
+        //For each toke log
+        for(const tokeLog of rawLegacyTokes){
+            //Split and iterate toke log by new line
+            for(const tokeLine of tokeLog.split('\n')){
+                //Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
+                //You'll probably want to have migrated from cytube/fore.st to canopy by then :)
+                //Also splits tokers array off for easier processing
+                const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
+                if(splitToke != null){
+
+                    //Create empty tokers map
+                    const toke = new Map();
+
+                    //Add qoutes around strings in the tokers line 
+                    let tokersLine = splitToke[0].replaceAll('[', '["');
+                    tokersLine = tokersLine.replaceAll(']','"]');
+                    tokersLine = tokersLine.replaceAll(',','","');
+
+                    //Force feed doctored line into the JSON parser, and iterate by the array it shits out
+                    for(const toker of JSON.parse(tokersLine)){
+                        toke.set(toker,"Legacy Tokes");
+                    }
+
+                    const date = new Date(Number(splitToke[2]));
+
+                    //Push toke on to statDB
+                    this.create({
+                        toke,
+                        date
+                    });
+
+                    console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
+                }
+            }
+        }
+
+        console.log("Legacy tokes commited to server-wide database statistics file!");
+    }catch(err){
+        return loggerUtils.localExceptionHandler(err);
+    }
+}
+
+tokeSchema.statics.dropLegacyTokes = async function(){
+    try{
+        //If legacy toke dropping is disabled or migration is enabled
+        if(!config.dropLegacyTokes || config.migrate){
+            //return
+            return;
+        }
+        //Pull tokes from DB
+        const oldTokes = await this.find();
+
+        //Create temporary toke array
+        const tokes = [];
+
+        //Nuke the toke collection
+        await this.deleteMany({});
+
+        //Iterate through server toke history
+        for(const toke of oldTokes){
+            //If it's not a legacy toke or a dupe
+            if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
+                //Re-add it to the database, scraping out the old ID
+                this.create({
+                    toke: toke.toke,
+                    date: toke.date
+                });
+            }
+        }
+
+        //Tell of our success
+        console.log("Removed migration tokes!");
+    }catch(err){
+        return loggerUtils.localExceptionHandler(err);
+    }
+
+}
+
+module.exports = mongoose.model("toke", tokeSchema);