rainbownapkin/canopy
1
0
Fork 0
Code Issues 2 Pull requests Actions Packages Projects Releases Wiki Activity

Moved toke log to it's own DB collection w/ cached statistics. Tokebot statistics page-load time decreased by up to 20-30x

Browse source
This commit is contained in:
rainbow napkin 2025-10-12 23:48:45 -04:00
parent 42ad17072f
commit a1b602efdb
7 changed files with 281 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/app/channel/tokebot.js
View file

@ -16,8 +16,8 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.*/
//Local Imports
const tokeCommandModel = require('../../schemas/tokebot/tokeCommandSchema');
const tokeModel = require('../../schemas/tokebot/tokeSchema');
const {userModel} = require('../../schemas/user/userSchema');
const statSchema = require('../../schemas/statSchema');
/**
@ -190,8 +190,8 @@ class tokebot{
//Asynchronously tattoo the toke into the users documents within the database so that tokebot doesn't have to wait or worry about DB transactions
userModel.tattooToke(this.tokers);
//Do the same for the global stat schema
statSchema.tattooToke(this.tokers);
//Do the same for the global toke statistics collection
tokeModel.tattooToke(this.tokers);
//Set the toke cooldown
this.cooldownCounter = this.cooldownTime;

164
src/schemas/statSchema.js
View file

@ -44,22 +44,16 @@ const statSchema = new mongoose.Schema({
type: mongoose.SchemaTypes.Date,
required: true,
default: new Date()
},
tokes: [{
toke: {
type: mongoose.SchemaTypes.Map,
required: true,
default: new Map()
},
date: {
type: mongoose.SchemaTypes.Date,
required: true,
default: new Date()
}
}]
});
//statics
/**
* Set placeholder variable to hold cached firstLaunch date from stat document
*/
statSchema.statics.firstLaunch = null;
/**
* Get's servers sole stat document from the DB
* @returns {Mongoose.Document} Server's sole statistics document
@ -96,6 +90,9 @@ statSchema.statics.incrementLaunchCount = async function(){
stats.launchCount++;
stats.save();
//Cache first launch
this.firstLaunch = stats.firstLaunch;
//print bootup message to console.
console.log(`${config.instanceName}(Powered by Canopy) initialized. This server has booted ${stats.launchCount} time${stats.launchCount == 1 ? '' : 's'}.`)
console.log(`First booted on ${stats.firstLaunch}.`);
@ -137,147 +134,4 @@ statSchema.statics.incrementChannelCount = async function(){
return oldCount;
}
/**
* Tattoo's toke to the server statistics document
* @param {Map} toke - Tokemap handed down from Tokebot
*/
statSchema.statics.tattooToke = async function(toke){
//Get the statistics document
const stats = await this.getStats();
//Add the toke to the stat document
stats.tokes.push({toke});
//Save the stat document
await stats.save();
}
/**
* Ingests legacy tokes handed over by the migration model
* @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
*/
statSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
//If migration is disabled
if(!config.migrate){
//BAIL!
return;
}
try{
const statDB = await this.getStats();
//For each toke log
for(const tokeLog of rawLegacyTokes){
//Split and iterate toke log by new line
for(const tokeLine of tokeLog.split('\n')){
//Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
//You'll probably want to have migrated from cytube/fore.st to canopy by then :)
//Also splits tokers array off for easier processing
const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
if(splitToke != null){
//Create empty tokers map
const toke = new Map();
//Add qoutes around strings in the tokers line
let tokersLine = splitToke[0].replaceAll('[', '["');
tokersLine = tokersLine.replaceAll(']','"]');
tokersLine = tokersLine.replaceAll(',','","');
//Force feed doctored line into the JSON parser, and iterate by the array it shits out
for(const toker of JSON.parse(tokersLine)){
toke.set(toker,"Legacy Tokes");
}
const date = new Date(Number(splitToke[2]));
//Push toke on to statDB
statDB.tokes.push({
toke,
date
});
console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
}
}
}
//Save toke to file
await statDB.save();
console.log("Legacy tokes commited to server-wide database statistics file!");
}catch(err){
return loggerutils.localexceptionhandler(err);
}
}
statSchema.statics.dropLegacyTokes = async function(){
try{
//If legacy toke dropping is disabled or migration is enabled
if(!config.dropLegacyTokes || config.migrate){
//return
return;
}
//pull stat doc
const statDB = await this.getStats();
//Create temporary toke array
const tokes = [];
//Iterate through server toke history
for(const toke of statDB.tokes){
//If it's not a legacy toke
if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
//Add it to the temp array
tokes.push(toke);
}
}
//Replace the server-wide toke log with our newly doctored one
statDB.tokes = tokes;
//Save the stat document
await statDB.save();
//Tell of our success
console.log("Removed migration tokes!");
}catch(err){
return loggerutils.localexceptionhandler(err);
}
}
//Methods
/**
* Gets toke counts for each individual callout in a map
* @returns {Map} Map of toke counts for each individual callout registered to the server
*/
statSchema.methods.calculateTokeCommandCounts = async function(){
//Create empty map to hold toke command counts
const count = new Map();
//for each toke
this.tokes.forEach((toke) => {
//For each toke command called in the current toke
toke.toke.forEach((command) => {
//Get the current count for the current command
var curCount = count.get(command);
//if the current count is null
if(curCount == null){
//Set it to one
count.set(command, 1);
}else{
//Set it to ++curCount
count.set(command, curCount + 1);
}
});
});
//return the toke command count
return count;
}
module.exports = mongoose.model("statistics", statSchema);

219
src/schemas/tokebot/tokeSchema.js Normal file
View file

@ -0,0 +1,219 @@
/*Canopy - The next generation of stoner streaming software
Copyright (C) 2024-2025 Rainbownapkin and the TTN Community
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.*/
//NPM Imports
const {mongoose} = require('mongoose');
//Local Imports
const config = require('./../../../config.json');
const loggerUtils = require('../../utils/loggerUtils');
/**
* DB Schema for single document for keeping track of a single toke
*/
const tokeSchema = new mongoose.Schema({
toke: {
type: mongoose.SchemaTypes.Map,
required: true,
default: new Map()
},
date: {
type: mongoose.SchemaTypes.Date,
required: true,
default: new Date()
}
});
//statics
/**
* Cached map containing counts of individual toke commands
*/
tokeSchema.statics.tokeMap = new Map();
/**
* Cached number of total tokes
*/
tokeSchema.statics.count = 0;
/**
* Cached number of times a user has successfully ran a '!toke' command
* Not to be confused with tokeSchema.statics.count, which counts total amount of tokes called out
*/
tokeSchema.statics.commandCount = 0;
/**
* Calculates cached toke map from existing
*/
tokeSchema.statics.calculateTokeMap = async function(){
//Pull full toke collection
const tokes = await this.find();
//Drop existing toke map
this.tokeMap = new Map();
//Iterate through DB of tokes
for(const toke of tokes){
//Increment toke count
this.count++;
//For each command callout
for(const command of toke.toke){
//Increment Command Count
this.commandCount++;
//Pull current count of respective toke command
let curCount = this.tokeMap.get(command[1]);
//If this is an unset toke command
if(curCount == null){
//Set it to one
this.tokeMap.set(command[1], 1);
//Otherwise
}else{
//Increment the existing count
this.tokeMap.set(command[1], ++curCount);
}
}
}
//Display calculated toke sats for funsies
if(config.verbose){
console.log(`Processed ${this.commandCount} toke command callouts accross ${await this.estimatedDocumentCount()} tokes.`);
}
}
/**
* Tattoos toke into toke DB colleciton
*
* We use this instead of a pre-save function as we need to fuck w/ statics
*/
tokeSchema.statics.tattooToke = async function(toke){
//Write toke to DB
await this.create({toke});
//Increment RAM-backed toke count
this.count++;
//Iterate through tokers
for(const curToke of toke){
//Pull current toke count
let curCount = this.tokeMap.get(curToke[1]);
//If this command hasn't been counted
if(curCount == null){
//Set new command count to one
this.tokeMap.set(curToke[1], 1);
}else{
//Increment current toke count and commit it to the RAM-backed tokeMap
this.tokeMap.set(curToke[1], ++curCount);
}
//Increment RAM-Backed command count
this.commandCount++;
}
}
/**
* Ingests legacy tokes handed over by the migration model
* @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
*/
tokeSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
//If migration is disabled
if(!config.migrate){
//BAIL!
return;
}
try{
//For each toke log
for(const tokeLog of rawLegacyTokes){
//Split and iterate toke log by new line
for(const tokeLine of tokeLog.split('\n')){
//Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
//You'll probably want to have migrated from cytube/fore.st to canopy by then :)
//Also splits tokers array off for easier processing
const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
if(splitToke != null){
//Create empty tokers map
const toke = new Map();
//Add qoutes around strings in the tokers line
let tokersLine = splitToke[0].replaceAll('[', '["');
tokersLine = tokersLine.replaceAll(']','"]');
tokersLine = tokersLine.replaceAll(',','","');
//Force feed doctored line into the JSON parser, and iterate by the array it shits out
for(const toker of JSON.parse(tokersLine)){
toke.set(toker,"Legacy Tokes");
}
const date = new Date(Number(splitToke[2]));
//Push toke on to statDB
this.create({
toke,
date
});
console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
}
}
}
console.log("Legacy tokes commited to server-wide database statistics file!");
}catch(err){
return loggerUtils.localExceptionHandler(err);
}
}
tokeSchema.statics.dropLegacyTokes = async function(){
try{
//If legacy toke dropping is disabled or migration is enabled
if(!config.dropLegacyTokes || config.migrate){
//return
return;
}
//Pull tokes from DB
const oldTokes = await this.find();
//Create temporary toke array
const tokes = [];
//Nuke the toke collection
await this.deleteMany({});
//Iterate through server toke history
for(const toke of oldTokes){
//If it's not a legacy toke or a dupe
if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
//Re-add it to the database, scraping out the old ID
this.create({
toke: toke.toke,
date: toke.date
});
}
}
//Tell of our success
console.log("Removed migration tokes!");
}catch(err){
return loggerUtils.localExceptionHandler(err);
}
}
module.exports = mongoose.model("toke", tokeSchema);

6
src/schemas/user/migrationSchema.js
View file

@ -24,7 +24,7 @@ const {mongoose} = require('mongoose');
const config = require('../../../config.json');
const {userModel} = require('../user/userSchema');
const permissionModel = require('../permissionSchema');
const statModel = require('../statSchema');
const tokeModel = require('../tokebot/tokeSchema');
const loggerUtils = require('../../utils/loggerUtils');
@ -78,7 +78,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){
try{
//If migration is disabled
if(!config.migrate){
statModel.dropLegacyTokes();
await tokeModel.dropLegacyTokes();
//BAIL!
return;
}
@ -145,7 +145,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){
await this.ingestTokeMaps(tokeMaps);
//Pass toke logs over to the stat model for further ingestion
await statModel.ingestLegacyTokes(tokeLogs);
await tokeModel.ingestLegacyTokes(tokeLogs);
loggerUtils.consoleWarn(`Legacy Server Migration Completed at: ${new Date().toLocaleString()}`);
}catch(err){

11
src/schemas/user/userSchema.js
View file

@ -22,6 +22,7 @@ const {mongoose} = require('mongoose');
const server = require('../../server');
//DB Models
const statModel = require('../statSchema');
const tokeModel = require('../tokebot/tokeSchema');
const flairModel = require('../flairSchema');
const permissionModel = require('../permissionSchema');
const emoteModel = require('../emoteSchema');
@ -311,16 +312,14 @@ userSchema.statics.findProfile = async function(user, includeEmail = false){
return null;
//If someone's looking for tokebot
}else if(user.user.toLowerCase() == "tokebot"){
//Pull statistics document from the database
const statDB = await statModel.getStats();
//fake a profile hashtable for tokebot
const profile = {
id: -420,
user: "Tokebot",
date: statDB.firstLaunch,
tokes: await statDB.calculateTokeCommandCounts(),
tokeCount: statDB.tokes.length,
//Look ma, no DB calls!
date: statModel.firstLaunch,
tokes: tokeModel.tokeMap,
tokeCount: tokeModel.count,
img: "/nonfree/johnny.png",
signature: "!TOKE",
bio: "!TOKE OR DIE!"

39
src/server.js
View file

@ -42,6 +42,7 @@ const {errorMiddleware} = require('./utils/loggerUtils');
const statModel = require('./schemas/statSchema');
const flairModel = require('./schemas/flairSchema');
const emoteModel = require('./schemas/emoteSchema');
const tokeModel = require('./schemas/tokebot/tokeSchema');
const tokeCommandModel = require('./schemas/tokebot/tokeCommandSchema');
const migrationModel = require('./schemas/user/migrationSchema');
//Controller
@ -181,29 +182,43 @@ app.use(errorMiddleware);
//Basic 404 handler
app.use(fileNotFoundController);
//Increment launch counter
statModel.incrementLaunchCount();
asyncKickStart();
//Load default flairs
flairModel.loadDefaults();
//Load default emotes
emoteModel.loadDefaults();
//Load default toke commands
tokeCommandModel.loadDefaults();
/*Asyncronous Kickstarter function
Allows us to force server startup to wait on the DB to be ready.
Might be better if she kicked off everything at once, and ran a while loop to check when they where all done.
This runs once at server startup, and most startups will run fairly quickly so... Not worth it?*/
async function asyncKickStart(){
//Lettum fuckin' know wassup
console.log(`${config.instanceName}(Powered by Canopy) is booting up!`);
//Run legacy migration
migrationModel.ingestLegacyDump();
await migrationModel.ingestLegacyDump();
//Calculate Toke Map
await tokeModel.calculateTokeMap();
//Load default toke commands
await tokeCommandModel.loadDefaults();
//Load default flairs
await flairModel.loadDefaults();
//Load default emotes
await emoteModel.loadDefaults();
//Kick off scheduled-jobs
scheduler.kickoff();
//Increment launch counter
await statModel.incrementLaunchCount();
//Hand over general-namespace socket.io connections to the channel manager
module.exports.channelManager = new channelManager(io)
module.exports.pmHandler = new pmHandler(io, module.exports.channelManager);
//Listen Function
webServer.listen(port, () => {
console.log(`Opening port ${port}`);
console.log(`Tokes up on port ${port}!`);
});
}

7
src/views/partial/profile/tokeCount.ejs
View file

@ -19,6 +19,11 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. %>
</span>
<div class="profile-item nested-dynamic-container" id="profile-tokes">
<% profile.tokes.forEach((count, toke) => { %>
<p class="profile-item profile-toke" id='profile-tokes<%-toke%>'><%- toke == "Legacy Tokes" ? '<br>' : '!' %><%- toke %>: <%- count %></p>
<% if(toke != "Legacy Tokes"){ %>
<p class="profile-item profile-toke" id='profile-tokes<%-toke%>'>!<%- toke %>: <%- count %></p>
<% } %>
<% }); %>
<% if(profile.tokes.get("Legacy Tokes") != null){ %>
<p class="profile-item profile-toke" id='profile-tokes-legacy-tokes'><br>Legacy Tokes: <%- profile.tokes.get("Legacy Tokes") %></p>
<% } %>
</div>