From a1b602efdb32f22d827130cac208d933d6d3ad66 Mon Sep 17 00:00:00 2001 From: rainbow napkin Date: Sun, 12 Oct 2025 23:48:45 -0400 Subject: [PATCH] Moved toke log to it's own DB collection w/ cached statistics. Tokebot statistics page-load time decreased by up to 20-30x --- src/app/channel/tokebot.js | 6 +- src/schemas/statSchema.js | 168 ++---------------- src/schemas/tokebot/tokeSchema.js | 219 ++++++++++++++++++++++++ src/schemas/user/migrationSchema.js | 6 +- src/schemas/user/userSchema.js | 11 +- src/server.js | 53 ++++-- src/views/partial/profile/tokeCount.ejs | 7 +- 7 files changed, 281 insertions(+), 189 deletions(-) create mode 100644 src/schemas/tokebot/tokeSchema.js diff --git a/src/app/channel/tokebot.js b/src/app/channel/tokebot.js index 7d1f005..6e41471 100644 --- a/src/app/channel/tokebot.js +++ b/src/app/channel/tokebot.js @@ -16,8 +16,8 @@ along with this program. If not, see .*/ //Local Imports const tokeCommandModel = require('../../schemas/tokebot/tokeCommandSchema'); +const tokeModel = require('../../schemas/tokebot/tokeSchema'); const {userModel} = require('../../schemas/user/userSchema'); -const statSchema = require('../../schemas/statSchema'); /** @@ -190,8 +190,8 @@ class tokebot{ //Asynchronously tattoo the toke into the users documents within the database so that tokebot doesn't have to wait or worry about DB transactions userModel.tattooToke(this.tokers); - //Do the same for the global stat schema - statSchema.tattooToke(this.tokers); + //Do the same for the global toke statistics collection + tokeModel.tattooToke(this.tokers); //Set the toke cooldown this.cooldownCounter = this.cooldownTime; diff --git a/src/schemas/statSchema.js b/src/schemas/statSchema.js index ab737a0..d1af85e 100644 --- a/src/schemas/statSchema.js +++ b/src/schemas/statSchema.js @@ -44,22 +44,16 @@ const statSchema = new mongoose.Schema({ type: mongoose.SchemaTypes.Date, required: true, default: new Date() - }, - tokes: [{ - toke: { - type: mongoose.SchemaTypes.Map, - required: true, - default: new Map() - }, - date: { - type: mongoose.SchemaTypes.Date, - required: true, - default: new Date() - } - }] + } }); //statics + +/** + * Set placeholder variable to hold cached firstLaunch date from stat document + */ +statSchema.statics.firstLaunch = null; + /** * Get's servers sole stat document from the DB * @returns {Mongoose.Document} Server's sole statistics document @@ -67,7 +61,7 @@ const statSchema = new mongoose.Schema({ statSchema.statics.getStats = async function(){ //Get the first document we find var stats = await this.findOne({}); - + if(stats){ //If we found something then the statistics document exist and this is it, //So long as no one else has fucked with the database it should be the only one. (is this forshadowing for a future bug?) @@ -96,6 +90,9 @@ statSchema.statics.incrementLaunchCount = async function(){ stats.launchCount++; stats.save(); + //Cache first launch + this.firstLaunch = stats.firstLaunch; + //print bootup message to console. console.log(`${config.instanceName}(Powered by Canopy) initialized. This server has booted ${stats.launchCount} time${stats.launchCount == 1 ? '' : 's'}.`) console.log(`First booted on ${stats.firstLaunch}.`); @@ -137,147 +134,4 @@ statSchema.statics.incrementChannelCount = async function(){ return oldCount; } -/** - * Tattoo's toke to the server statistics document - * @param {Map} toke - Tokemap handed down from Tokebot - */ -statSchema.statics.tattooToke = async function(toke){ - //Get the statistics document - const stats = await this.getStats(); - - //Add the toke to the stat document - stats.tokes.push({toke}); - - //Save the stat document - await stats.save(); -} - -/** - * Ingests legacy tokes handed over by the migration model - * @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs - */ -statSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){ - //If migration is disabled - if(!config.migrate){ - //BAIL! - return; - } - - try{ - const statDB = await this.getStats(); - - //For each toke log - for(const tokeLog of rawLegacyTokes){ - //Split and iterate toke log by new line - for(const tokeLine of tokeLog.split('\n')){ - //Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001) - //You'll probably want to have migrated from cytube/fore.st to canopy by then :) - //Also splits tokers array off for easier processing - const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g) - if(splitToke != null){ - - //Create empty tokers map - const toke = new Map(); - - //Add qoutes around strings in the tokers line - let tokersLine = splitToke[0].replaceAll('[', '["'); - tokersLine = tokersLine.replaceAll(']','"]'); - tokersLine = tokersLine.replaceAll(',','","'); - - //Force feed doctored line into the JSON parser, and iterate by the array it shits out - for(const toker of JSON.parse(tokersLine)){ - toke.set(toker,"Legacy Tokes"); - } - - const date = new Date(Number(splitToke[2])); - - //Push toke on to statDB - statDB.tokes.push({ - toke, - date - }); - - console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`); - } - } - } - - //Save toke to file - await statDB.save(); - - console.log("Legacy tokes commited to server-wide database statistics file!"); - }catch(err){ - return loggerutils.localexceptionhandler(err); - } -} - -statSchema.statics.dropLegacyTokes = async function(){ - try{ - //If legacy toke dropping is disabled or migration is enabled - if(!config.dropLegacyTokes || config.migrate){ - //return - return; - } - - //pull stat doc - const statDB = await this.getStats(); - - //Create temporary toke array - const tokes = []; - - //Iterate through server toke history - for(const toke of statDB.tokes){ - //If it's not a legacy toke - if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){ - //Add it to the temp array - tokes.push(toke); - } - } - - //Replace the server-wide toke log with our newly doctored one - statDB.tokes = tokes; - - //Save the stat document - await statDB.save(); - - //Tell of our success - console.log("Removed migration tokes!"); - }catch(err){ - return loggerutils.localexceptionhandler(err); - } - -} - -//Methods - -/** - * Gets toke counts for each individual callout in a map - * @returns {Map} Map of toke counts for each individual callout registered to the server - */ -statSchema.methods.calculateTokeCommandCounts = async function(){ - //Create empty map to hold toke command counts - const count = new Map(); - - //for each toke - this.tokes.forEach((toke) => { - //For each toke command called in the current toke - toke.toke.forEach((command) => { - //Get the current count for the current command - var curCount = count.get(command); - - //if the current count is null - if(curCount == null){ - //Set it to one - count.set(command, 1); - }else{ - //Set it to ++curCount - count.set(command, curCount + 1); - } - }); - }); - - //return the toke command count - return count; -} - module.exports = mongoose.model("statistics", statSchema); \ No newline at end of file diff --git a/src/schemas/tokebot/tokeSchema.js b/src/schemas/tokebot/tokeSchema.js new file mode 100644 index 0000000..ca186af --- /dev/null +++ b/src/schemas/tokebot/tokeSchema.js @@ -0,0 +1,219 @@ +/*Canopy - The next generation of stoner streaming software +Copyright (C) 2024-2025 Rainbownapkin and the TTN Community + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see .*/ + +//NPM Imports +const {mongoose} = require('mongoose'); + +//Local Imports +const config = require('./../../../config.json'); +const loggerUtils = require('../../utils/loggerUtils'); + +/** + * DB Schema for single document for keeping track of a single toke + */ +const tokeSchema = new mongoose.Schema({ + toke: { + type: mongoose.SchemaTypes.Map, + required: true, + default: new Map() + }, + date: { + type: mongoose.SchemaTypes.Date, + required: true, + default: new Date() + } +}); + +//statics +/** + * Cached map containing counts of individual toke commands + */ +tokeSchema.statics.tokeMap = new Map(); + +/** + * Cached number of total tokes + */ +tokeSchema.statics.count = 0; + +/** + * Cached number of times a user has successfully ran a '!toke' command + * Not to be confused with tokeSchema.statics.count, which counts total amount of tokes called out + */ +tokeSchema.statics.commandCount = 0; + +/** + * Calculates cached toke map from existing + */ +tokeSchema.statics.calculateTokeMap = async function(){ + //Pull full toke collection + const tokes = await this.find(); + + //Drop existing toke map + this.tokeMap = new Map(); + + //Iterate through DB of tokes + for(const toke of tokes){ + //Increment toke count + this.count++; + + //For each command callout + for(const command of toke.toke){ + //Increment Command Count + this.commandCount++; + + //Pull current count of respective toke command + let curCount = this.tokeMap.get(command[1]); + + //If this is an unset toke command + if(curCount == null){ + //Set it to one + this.tokeMap.set(command[1], 1); + //Otherwise + }else{ + //Increment the existing count + this.tokeMap.set(command[1], ++curCount); + } + } + } + + //Display calculated toke sats for funsies + if(config.verbose){ + console.log(`Processed ${this.commandCount} toke command callouts accross ${await this.estimatedDocumentCount()} tokes.`); + } +} + +/** + * Tattoos toke into toke DB colleciton + * + * We use this instead of a pre-save function as we need to fuck w/ statics + */ +tokeSchema.statics.tattooToke = async function(toke){ + //Write toke to DB + await this.create({toke}); + + //Increment RAM-backed toke count + this.count++; + + //Iterate through tokers + for(const curToke of toke){ + //Pull current toke count + let curCount = this.tokeMap.get(curToke[1]); + + //If this command hasn't been counted + if(curCount == null){ + //Set new command count to one + this.tokeMap.set(curToke[1], 1); + }else{ + //Increment current toke count and commit it to the RAM-backed tokeMap + this.tokeMap.set(curToke[1], ++curCount); + } + + //Increment RAM-Backed command count + this.commandCount++; + } +} + +/** + * Ingests legacy tokes handed over by the migration model + * @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs + */ +tokeSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){ + //If migration is disabled + if(!config.migrate){ + //BAIL! + return; + } + + try{ + //For each toke log + for(const tokeLog of rawLegacyTokes){ + //Split and iterate toke log by new line + for(const tokeLine of tokeLog.split('\n')){ + //Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001) + //You'll probably want to have migrated from cytube/fore.st to canopy by then :) + //Also splits tokers array off for easier processing + const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g) + if(splitToke != null){ + + //Create empty tokers map + const toke = new Map(); + + //Add qoutes around strings in the tokers line + let tokersLine = splitToke[0].replaceAll('[', '["'); + tokersLine = tokersLine.replaceAll(']','"]'); + tokersLine = tokersLine.replaceAll(',','","'); + + //Force feed doctored line into the JSON parser, and iterate by the array it shits out + for(const toker of JSON.parse(tokersLine)){ + toke.set(toker,"Legacy Tokes"); + } + + const date = new Date(Number(splitToke[2])); + + //Push toke on to statDB + this.create({ + toke, + date + }); + + console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`); + } + } + } + + console.log("Legacy tokes commited to server-wide database statistics file!"); + }catch(err){ + return loggerUtils.localExceptionHandler(err); + } +} + +tokeSchema.statics.dropLegacyTokes = async function(){ + try{ + //If legacy toke dropping is disabled or migration is enabled + if(!config.dropLegacyTokes || config.migrate){ + //return + return; + } + //Pull tokes from DB + const oldTokes = await this.find(); + + //Create temporary toke array + const tokes = []; + + //Nuke the toke collection + await this.deleteMany({}); + + //Iterate through server toke history + for(const toke of oldTokes){ + //If it's not a legacy toke or a dupe + if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){ + //Re-add it to the database, scraping out the old ID + this.create({ + toke: toke.toke, + date: toke.date + }); + } + } + + //Tell of our success + console.log("Removed migration tokes!"); + }catch(err){ + return loggerUtils.localExceptionHandler(err); + } + +} + +module.exports = mongoose.model("toke", tokeSchema); \ No newline at end of file diff --git a/src/schemas/user/migrationSchema.js b/src/schemas/user/migrationSchema.js index 8ae54f7..b03b1dc 100644 --- a/src/schemas/user/migrationSchema.js +++ b/src/schemas/user/migrationSchema.js @@ -24,7 +24,7 @@ const {mongoose} = require('mongoose'); const config = require('../../../config.json'); const {userModel} = require('../user/userSchema'); const permissionModel = require('../permissionSchema'); -const statModel = require('../statSchema'); +const tokeModel = require('../tokebot/tokeSchema'); const loggerUtils = require('../../utils/loggerUtils'); @@ -78,7 +78,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){ try{ //If migration is disabled if(!config.migrate){ - statModel.dropLegacyTokes(); + await tokeModel.dropLegacyTokes(); //BAIL! return; } @@ -145,7 +145,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){ await this.ingestTokeMaps(tokeMaps); //Pass toke logs over to the stat model for further ingestion - await statModel.ingestLegacyTokes(tokeLogs); + await tokeModel.ingestLegacyTokes(tokeLogs); loggerUtils.consoleWarn(`Legacy Server Migration Completed at: ${new Date().toLocaleString()}`); }catch(err){ diff --git a/src/schemas/user/userSchema.js b/src/schemas/user/userSchema.js index bf7af99..dd13b24 100644 --- a/src/schemas/user/userSchema.js +++ b/src/schemas/user/userSchema.js @@ -22,6 +22,7 @@ const {mongoose} = require('mongoose'); const server = require('../../server'); //DB Models const statModel = require('../statSchema'); +const tokeModel = require('../tokebot/tokeSchema'); const flairModel = require('../flairSchema'); const permissionModel = require('../permissionSchema'); const emoteModel = require('../emoteSchema'); @@ -311,16 +312,14 @@ userSchema.statics.findProfile = async function(user, includeEmail = false){ return null; //If someone's looking for tokebot }else if(user.user.toLowerCase() == "tokebot"){ - //Pull statistics document from the database - const statDB = await statModel.getStats(); - //fake a profile hashtable for tokebot const profile = { id: -420, user: "Tokebot", - date: statDB.firstLaunch, - tokes: await statDB.calculateTokeCommandCounts(), - tokeCount: statDB.tokes.length, + //Look ma, no DB calls! + date: statModel.firstLaunch, + tokes: tokeModel.tokeMap, + tokeCount: tokeModel.count, img: "/nonfree/johnny.png", signature: "!TOKE", bio: "!TOKE OR DIE!" diff --git a/src/server.js b/src/server.js index 8a13955..c1b2e2f 100644 --- a/src/server.js +++ b/src/server.js @@ -42,6 +42,7 @@ const {errorMiddleware} = require('./utils/loggerUtils'); const statModel = require('./schemas/statSchema'); const flairModel = require('./schemas/flairSchema'); const emoteModel = require('./schemas/emoteSchema'); +const tokeModel = require('./schemas/tokebot/tokeSchema'); const tokeCommandModel = require('./schemas/tokebot/tokeCommandSchema'); const migrationModel = require('./schemas/user/migrationSchema'); //Controller @@ -181,29 +182,43 @@ app.use(errorMiddleware); //Basic 404 handler app.use(fileNotFoundController); -//Increment launch counter -statModel.incrementLaunchCount(); +asyncKickStart(); -//Load default flairs -flairModel.loadDefaults(); +/*Asyncronous Kickstarter function +Allows us to force server startup to wait on the DB to be ready. +Might be better if she kicked off everything at once, and ran a while loop to check when they where all done. +This runs once at server startup, and most startups will run fairly quickly so... Not worth it?*/ +async function asyncKickStart(){ + //Lettum fuckin' know wassup + console.log(`${config.instanceName}(Powered by Canopy) is booting up!`); -//Load default emotes -emoteModel.loadDefaults(); + //Run legacy migration + await migrationModel.ingestLegacyDump(); -//Load default toke commands -tokeCommandModel.loadDefaults(); + //Calculate Toke Map + await tokeModel.calculateTokeMap(); -//Run legacy migration -migrationModel.ingestLegacyDump(); + //Load default toke commands + await tokeCommandModel.loadDefaults(); -//Kick off scheduled-jobs -scheduler.kickoff(); + //Load default flairs + await flairModel.loadDefaults(); -//Hand over general-namespace socket.io connections to the channel manager -module.exports.channelManager = new channelManager(io) -module.exports.pmHandler = new pmHandler(io, module.exports.channelManager); + //Load default emotes + await emoteModel.loadDefaults(); -//Listen Function -webServer.listen(port, () => { - console.log(`Opening port ${port}`); -}); \ No newline at end of file + //Kick off scheduled-jobs + scheduler.kickoff(); + + //Increment launch counter + await statModel.incrementLaunchCount(); + + //Hand over general-namespace socket.io connections to the channel manager + module.exports.channelManager = new channelManager(io) + module.exports.pmHandler = new pmHandler(io, module.exports.channelManager); + + //Listen Function + webServer.listen(port, () => { + console.log(`Tokes up on port ${port}!`); + }); +} \ No newline at end of file diff --git a/src/views/partial/profile/tokeCount.ejs b/src/views/partial/profile/tokeCount.ejs index 9c1ab81..ccf9ac5 100644 --- a/src/views/partial/profile/tokeCount.ejs +++ b/src/views/partial/profile/tokeCount.ejs @@ -19,6 +19,11 @@ along with this program. If not, see . %>
<% profile.tokes.forEach((count, toke) => { %> -

<%- toke == "Legacy Tokes" ? '
' : '!' %><%- toke %>: <%- count %>

+ <% if(toke != "Legacy Tokes"){ %> +

!<%- toke %>: <%- count %>

+ <% } %> <% }); %> + <% if(profile.tokes.get("Legacy Tokes") != null){ %> +


Legacy Tokes: <%- profile.tokes.get("Legacy Tokes") %>

+ <% } %>
\ No newline at end of file