Moved toke log to it's own DB collection w/ cached statistics. Tokebot statistics page-load time decreased by up to 20-30x

src/app/channel/tokebot.js

@@ -16,8 +16,8 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.*/
 
 //Local Imports
 const tokeCommandModel = require('../../schemas/tokebot/tokeCommandSchema');
+const tokeModel = require('../../schemas/tokebot/tokeSchema');
 const {userModel} = require('../../schemas/user/userSchema');
-const statSchema = require('../../schemas/statSchema');
 
 
 /**
@@ -190,8 +190,8 @@ class tokebot{
 
             //Asynchronously tattoo the toke into the users documents within the database so that tokebot doesn't have to wait or worry about DB transactions
             userModel.tattooToke(this.tokers);
-            //Do the same for the global stat schema
+            //Do the same for the global toke statistics collection
-            statSchema.tattooToke(this.tokers);
+            tokeModel.tattooToke(this.tokers);
 
             //Set the toke cooldown
             this.cooldownCounter  = this.cooldownTime;

src/schemas/statSchema.js

@@ -44,22 +44,16 @@ const statSchema = new mongoose.Schema({
         type: mongoose.SchemaTypes.Date,
         required: true,
         default: new Date()
-    },
+    }
-    tokes: [{
-        toke: {
-            type: mongoose.SchemaTypes.Map,
-            required: true,
-            default: new Map()
-        },
-        date: {
-            type: mongoose.SchemaTypes.Date,
-            required: true,
-            default: new Date()
-        }
-    }]
 });
 
 //statics
+
+/**
+ * Set placeholder variable to hold cached firstLaunch date from stat document
+ */
+statSchema.statics.firstLaunch = null;
+
 /**
  * Get's servers sole stat document from the DB
  * @returns {Mongoose.Document} Server's sole statistics document
@@ -67,7 +61,7 @@ const statSchema = new mongoose.Schema({
 statSchema.statics.getStats = async function(){
     //Get the first document we find
     var stats = await this.findOne({});
-    
+
     if(stats){
         //If we found something then the statistics document exist and this is it,
         //So long as no one else has fucked with the database it should be the only one. (is this forshadowing for a future bug?)
@@ -96,6 +90,9 @@ statSchema.statics.incrementLaunchCount = async function(){
     stats.launchCount++;
     stats.save();
 
+    //Cache first launch
+    this.firstLaunch = stats.firstLaunch;
+
     //print bootup message to console.
     console.log(`${config.instanceName}(Powered by Canopy) initialized. This server has booted ${stats.launchCount} time${stats.launchCount == 1 ? '' : 's'}.`)
     console.log(`First booted on ${stats.firstLaunch}.`);
@@ -137,147 +134,4 @@ statSchema.statics.incrementChannelCount = async function(){
     return oldCount;
 }
 
-/**
- * Tattoo's toke to the server statistics document
- * @param {Map} toke - Tokemap handed down from Tokebot
- */
-statSchema.statics.tattooToke = async function(toke){
-    //Get the statistics document
-    const stats = await this.getStats();
-
-    //Add the toke to the stat document
-    stats.tokes.push({toke});
-
-    //Save the stat document
-    await stats.save();
-}
-
-/**
- * Ingests legacy tokes handed over by the migration model
- * @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
- */
-statSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
-    //If migration is disabled
-    if(!config.migrate){
-        //BAIL!
-        return;
-    }
-
-    try{
-        const statDB = await this.getStats();
-
-        //For each toke log
-        for(const tokeLog of rawLegacyTokes){
-            //Split and iterate toke log by new line
-            for(const tokeLine of tokeLog.split('\n')){
-                //Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
-                //You'll probably want to have migrated from cytube/fore.st to canopy by then :)
-                //Also splits tokers array off for easier processing
-                const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
-                if(splitToke != null){
-
-                    //Create empty tokers map
-                    const toke = new Map();
-
-                    //Add qoutes around strings in the tokers line 
-                    let tokersLine = splitToke[0].replaceAll('[', '["');
-                    tokersLine = tokersLine.replaceAll(']','"]');
-                    tokersLine = tokersLine.replaceAll(',','","');
-
-                    //Force feed doctored line into the JSON parser, and iterate by the array it shits out
-                    for(const toker of JSON.parse(tokersLine)){
-                        toke.set(toker,"Legacy Tokes");
-                    }
-
-                    const date = new Date(Number(splitToke[2]));
-
-                    //Push toke on to statDB
-                    statDB.tokes.push({
-                        toke,
-                        date
-                    });
-
-                    console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
-                }
-            }
-        }
-
-        //Save toke to file
-        await statDB.save();
-
-        console.log("Legacy tokes commited to server-wide database statistics file!");
-    }catch(err){
-        return loggerutils.localexceptionhandler(err);
-    }
-}
-
-statSchema.statics.dropLegacyTokes = async function(){
-    try{
-        //If legacy toke dropping is disabled or migration is enabled
-        if(!config.dropLegacyTokes || config.migrate){
-            //return
-            return;
-        }
-
-        //pull stat doc
-        const statDB = await this.getStats();
-
-        //Create temporary toke array
-        const tokes = [];
-
-        //Iterate through server toke history
-        for(const toke of statDB.tokes){
-            //If it's not a legacy toke
-            if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
-                //Add it to the temp array
-                tokes.push(toke);
-            }
-        }
-
-        //Replace the server-wide toke log with our newly doctored one
-        statDB.tokes = tokes;
-
-        //Save the stat document
-        await statDB.save();
-
-        //Tell of our success
-        console.log("Removed migration tokes!");
-    }catch(err){
-        return loggerutils.localexceptionhandler(err);
-    }
-
-}
-
-//Methods
-
-/**
- * Gets toke counts for each individual callout in a map
- * @returns {Map} Map of toke counts for each individual callout registered to the server
- */
-statSchema.methods.calculateTokeCommandCounts = async function(){
-    //Create empty map to hold toke command counts
-    const count = new Map();
-
-    //for each toke
-    this.tokes.forEach((toke) => {
-        //For each toke command called in the current toke
-        toke.toke.forEach((command) => {
-            //Get the current count for the current command
-            var curCount = count.get(command);
-
-            //if the current count is null
-            if(curCount == null){
-                //Set it to one
-                count.set(command, 1);
-            }else{
-                //Set it to ++curCount
-                count.set(command, curCount + 1);
-            }
-        });
-    });
-
-    //return the toke command count
-    return count;
-}
-
 module.exports = mongoose.model("statistics", statSchema);

src/schemas/tokebot/tokeSchema.js

@@ -0,0 +1,219 @@
+/*Canopy - The next generation of stoner streaming software
+Copyright (C) 2024-2025 Rainbownapkin and the TTN Community
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.*/
+
+//NPM Imports
+const {mongoose} = require('mongoose');
+
+//Local Imports
+const config = require('./../../../config.json');
+const loggerUtils = require('../../utils/loggerUtils');
+
+/**
+ * DB Schema for single document for keeping track of a single toke
+ */
+const tokeSchema = new mongoose.Schema({
+    toke: {
+        type: mongoose.SchemaTypes.Map,
+        required: true,
+        default: new Map()
+    },
+    date: {
+        type: mongoose.SchemaTypes.Date,
+        required: true,
+        default: new Date()
+    }
+});
+
+//statics
+/**
+ * Cached map containing counts of individual toke commands
+ */
+tokeSchema.statics.tokeMap = new Map();
+
+/**
+ * Cached number of total tokes
+ */
+tokeSchema.statics.count = 0;
+
+/**
+ * Cached number of times a user has successfully ran a '!toke' command
+ * Not to be confused with tokeSchema.statics.count, which counts total amount of tokes called out
+ */
+tokeSchema.statics.commandCount = 0;
+
+/**
+ * Calculates cached toke map from existing 
+ */
+tokeSchema.statics.calculateTokeMap = async function(){
+    //Pull full toke collection
+    const tokes = await this.find();
+
+    //Drop existing toke map
+    this.tokeMap = new Map();
+
+    //Iterate through DB of tokes
+    for(const toke of tokes){
+        //Increment toke count
+        this.count++;
+
+        //For each command callout
+        for(const command of toke.toke){
+            //Increment Command Count
+            this.commandCount++;
+
+            //Pull current count of respective toke command
+            let curCount = this.tokeMap.get(command[1]);
+
+            //If this is an unset toke command
+            if(curCount == null){
+                //Set it to one
+                this.tokeMap.set(command[1], 1);
+            //Otherwise
+            }else{
+                //Increment the existing count
+                this.tokeMap.set(command[1], ++curCount);
+            }
+        }
+    }
+
+    //Display calculated toke sats for funsies
+    if(config.verbose){
+        console.log(`Processed ${this.commandCount} toke command callouts accross ${await this.estimatedDocumentCount()} tokes.`);
+    }
+}
+
+/**
+ * Tattoos toke into toke DB colleciton
+ * 
+ * We use this instead of a pre-save function as we need to fuck w/ statics
+ */
+tokeSchema.statics.tattooToke = async function(toke){
+    //Write toke to DB
+    await this.create({toke});
+
+    //Increment RAM-backed toke count
+    this.count++;
+
+    //Iterate through tokers
+    for(const curToke of toke){
+        //Pull current toke count
+        let curCount = this.tokeMap.get(curToke[1]);
+
+        //If this command hasn't been counted
+        if(curCount == null){
+            //Set new command count to one
+            this.tokeMap.set(curToke[1], 1);
+        }else{
+            //Increment current toke count and commit it to the RAM-backed tokeMap
+            this.tokeMap.set(curToke[1], ++curCount);
+        }
+
+        //Increment RAM-Backed command count
+        this.commandCount++;
+    }
+}
+
+/**
+ * Ingests legacy tokes handed over by the migration model
+ * @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
+ */
+tokeSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
+    //If migration is disabled
+    if(!config.migrate){
+        //BAIL!
+        return;
+    }
+
+    try{
+        //For each toke log
+        for(const tokeLog of rawLegacyTokes){
+            //Split and iterate toke log by new line
+            for(const tokeLine of tokeLog.split('\n')){
+                //Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
+                //You'll probably want to have migrated from cytube/fore.st to canopy by then :)
+                //Also splits tokers array off for easier processing
+                const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
+                if(splitToke != null){
+
+                    //Create empty tokers map
+                    const toke = new Map();
+
+                    //Add qoutes around strings in the tokers line 
+                    let tokersLine = splitToke[0].replaceAll('[', '["');
+                    tokersLine = tokersLine.replaceAll(']','"]');
+                    tokersLine = tokersLine.replaceAll(',','","');
+
+                    //Force feed doctored line into the JSON parser, and iterate by the array it shits out
+                    for(const toker of JSON.parse(tokersLine)){
+                        toke.set(toker,"Legacy Tokes");
+                    }
+
+                    const date = new Date(Number(splitToke[2]));
+
+                    //Push toke on to statDB
+                    this.create({
+                        toke,
+                        date
+                    });
+
+                    console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
+                }
+            }
+        }
+
+        console.log("Legacy tokes commited to server-wide database statistics file!");
+    }catch(err){
+        return loggerUtils.localExceptionHandler(err);
+    }
+}
+
+tokeSchema.statics.dropLegacyTokes = async function(){
+    try{
+        //If legacy toke dropping is disabled or migration is enabled
+        if(!config.dropLegacyTokes || config.migrate){
+            //return
+            return;
+        }
+        //Pull tokes from DB
+        const oldTokes = await this.find();
+
+        //Create temporary toke array
+        const tokes = [];
+
+        //Nuke the toke collection
+        await this.deleteMany({});
+
+        //Iterate through server toke history
+        for(const toke of oldTokes){
+            //If it's not a legacy toke or a dupe
+            if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
+                //Re-add it to the database, scraping out the old ID
+                this.create({
+                    toke: toke.toke,
+                    date: toke.date
+                });
+            }
+        }
+
+        //Tell of our success
+        console.log("Removed migration tokes!");
+    }catch(err){
+        return loggerUtils.localExceptionHandler(err);
+    }
+
+}
+
+module.exports = mongoose.model("toke", tokeSchema);

src/schemas/user/migrationSchema.js

@@ -24,7 +24,7 @@ const {mongoose} = require('mongoose');
 const config = require('../../../config.json');
 const {userModel} = require('../user/userSchema');
 const permissionModel = require('../permissionSchema');
-const statModel = require('../statSchema');
+const tokeModel = require('../tokebot/tokeSchema');
 const loggerUtils = require('../../utils/loggerUtils');
 
 
@@ -78,7 +78,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){
     try{
         //If migration is disabled
         if(!config.migrate){
-            statModel.dropLegacyTokes();
+            await tokeModel.dropLegacyTokes();
             //BAIL!
             return;
         }
@@ -145,7 +145,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){
         await this.ingestTokeMaps(tokeMaps);
 
         //Pass toke logs over to the stat model for further ingestion
-        await statModel.ingestLegacyTokes(tokeLogs);
+        await tokeModel.ingestLegacyTokes(tokeLogs);
 
         loggerUtils.consoleWarn(`Legacy Server Migration Completed at: ${new Date().toLocaleString()}`);
     }catch(err){

src/schemas/user/userSchema.js

@@ -22,6 +22,7 @@ const {mongoose} = require('mongoose');
 const server = require('../../server');
 //DB Models
 const statModel = require('../statSchema');
+const tokeModel = require('../tokebot/tokeSchema');
 const flairModel = require('../flairSchema');
 const permissionModel = require('../permissionSchema');
 const emoteModel = require('../emoteSchema');
@@ -311,16 +312,14 @@ userSchema.statics.findProfile = async function(user, includeEmail = false){
         return null;
     //If someone's looking for tokebot
     }else if(user.user.toLowerCase() == "tokebot"){
-        //Pull statistics document from the database
-        const statDB = await statModel.getStats();
-
         //fake a profile hashtable for tokebot
         const profile = {
             id: -420,
             user: "Tokebot",
-            date: statDB.firstLaunch,
+            //Look ma, no DB calls!
-            tokes: await statDB.calculateTokeCommandCounts(),
+            date: statModel.firstLaunch,
-            tokeCount: statDB.tokes.length, 
+            tokes: tokeModel.tokeMap,
+            tokeCount: tokeModel.count, 
             img: "/nonfree/johnny.png",
             signature: "!TOKE",
             bio: "!TOKE OR DIE!"

src/server.js

@@ -42,6 +42,7 @@ const {errorMiddleware} = require('./utils/loggerUtils');
 const statModel = require('./schemas/statSchema');
 const flairModel = require('./schemas/flairSchema');
 const emoteModel = require('./schemas/emoteSchema');
+const tokeModel = require('./schemas/tokebot/tokeSchema');
 const tokeCommandModel = require('./schemas/tokebot/tokeCommandSchema');
 const migrationModel = require('./schemas/user/migrationSchema');
 //Controller
@@ -181,29 +182,43 @@ app.use(errorMiddleware);
 //Basic 404 handler
 app.use(fileNotFoundController);
 
-//Increment launch counter
+asyncKickStart();
-statModel.incrementLaunchCount();
 
-//Load default flairs
+/*Asyncronous Kickstarter function
-flairModel.loadDefaults();
+Allows us to force server startup to wait on the DB to be ready.
+Might be better if she kicked off everything at once, and ran a while loop to check when they where all done.
+This runs once at server startup, and most startups will run fairly quickly so... Not worth it?*/
+async function asyncKickStart(){
+    //Lettum fuckin' know wassup
+    console.log(`${config.instanceName}(Powered by Canopy) is booting up!`);
 
-//Load default emotes
+    //Run legacy migration
-emoteModel.loadDefaults();
+    await migrationModel.ingestLegacyDump();
 
-//Load default toke commands
+    //Calculate Toke Map
-tokeCommandModel.loadDefaults();
+    await tokeModel.calculateTokeMap();
 
-//Run legacy migration
+    //Load default toke commands
-migrationModel.ingestLegacyDump();
+    await tokeCommandModel.loadDefaults();
 
-//Kick off scheduled-jobs
+    //Load default flairs
-scheduler.kickoff();
+    await flairModel.loadDefaults();
 
-//Hand over general-namespace socket.io connections to the channel manager
+    //Load default emotes
-module.exports.channelManager = new channelManager(io)
+    await emoteModel.loadDefaults();
-module.exports.pmHandler = new pmHandler(io, module.exports.channelManager);
 
-//Listen Function
+    //Kick off scheduled-jobs
-webServer.listen(port, () => {
+    scheduler.kickoff();
-    console.log(`Opening port ${port}`);
+
-});
+    //Increment launch counter
+    await statModel.incrementLaunchCount();
+
+    //Hand over general-namespace socket.io connections to the channel manager
+    module.exports.channelManager = new channelManager(io)
+    module.exports.pmHandler = new pmHandler(io, module.exports.channelManager);
+
+    //Listen Function
+    webServer.listen(port, () => {
+        console.log(`Tokes up on port ${port}!`);
+    });
+}

src/views/partial/profile/tokeCount.ejs

@@ -19,6 +19,11 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>. %>
 </span>
 <div class="profile-item nested-dynamic-container" id="profile-tokes">
     <% profile.tokes.forEach((count, toke) => { %>
-        <p class="profile-item profile-toke" id='profile-tokes<%-toke%>'><%- toke == "Legacy Tokes" ? '<br>' : '!' %><%- toke %>: <%- count %></p>
+        <% if(toke != "Legacy Tokes"){ %>
+            <p class="profile-item profile-toke" id='profile-tokes<%-toke%>'>!<%- toke %>: <%- count %></p>
+        <% } %>
     <% }); %>
+    <% if(profile.tokes.get("Legacy Tokes") != null){ %>
+        <p class="profile-item profile-toke" id='profile-tokes-legacy-tokes'><br>Legacy Tokes: <%- profile.tokes.get("Legacy Tokes") %></p>
+    <% } %>
 </div>