rainbownapkin/canopy
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases Packages Wiki Activity Actions

Moved toke log to it's own DB collection w/ cached statistics. Tokebot statistics page-load time decreased by up to 20-30x

Browse source
This commit is contained in:
rainbow napkin 2025-10-12 23:48:45 -04:00
parent 42ad17072f
commit a1b602efdb
7 changed files with 281 additions and 189 deletions
Download patch file Download diff file Expand all files Collapse all files

168
src/schemas/statSchema.js
View file

@ -44,22 +44,16 @@ const statSchema = new mongoose.Schema({
type: mongoose.SchemaTypes.Date,
required: true,
default: new Date()
},
tokes: [{
toke: {
type: mongoose.SchemaTypes.Map,
required: true,
default: new Map()
},
date: {
type: mongoose.SchemaTypes.Date,
required: true,
default: new Date()
}
}]
}
});
//statics
/**
* Set placeholder variable to hold cached firstLaunch date from stat document
*/
statSchema.statics.firstLaunch = null;
/**
* Get's servers sole stat document from the DB
* @returns {Mongoose.Document} Server's sole statistics document
@ -67,7 +61,7 @@ const statSchema = new mongoose.Schema({
statSchema.statics.getStats = async function(){
//Get the first document we find
var stats = await this.findOne({});
if(stats){
//If we found something then the statistics document exist and this is it,
//So long as no one else has fucked with the database it should be the only one. (is this forshadowing for a future bug?)
@ -96,6 +90,9 @@ statSchema.statics.incrementLaunchCount = async function(){
stats.launchCount++;
stats.save();
//Cache first launch
this.firstLaunch = stats.firstLaunch;
//print bootup message to console.
console.log(`${config.instanceName}(Powered by Canopy) initialized. This server has booted ${stats.launchCount} time${stats.launchCount == 1 ? '' : 's'}.`)
console.log(`First booted on ${stats.firstLaunch}.`);
@ -137,147 +134,4 @@ statSchema.statics.incrementChannelCount = async function(){
return oldCount;
}
/**
* Tattoo's toke to the server statistics document
* @param {Map} toke - Tokemap handed down from Tokebot
*/
statSchema.statics.tattooToke = async function(toke){
//Get the statistics document
const stats = await this.getStats();
//Add the toke to the stat document
stats.tokes.push({toke});
//Save the stat document
await stats.save();
}
/**
* Ingests legacy tokes handed over by the migration model
* @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
*/
statSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
//If migration is disabled
if(!config.migrate){
//BAIL!
return;
}
try{
const statDB = await this.getStats();
//For each toke log
for(const tokeLog of rawLegacyTokes){
//Split and iterate toke log by new line
for(const tokeLine of tokeLog.split('\n')){
//Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
//You'll probably want to have migrated from cytube/fore.st to canopy by then :)
//Also splits tokers array off for easier processing
const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
if(splitToke != null){
//Create empty tokers map
const toke = new Map();
//Add qoutes around strings in the tokers line
let tokersLine = splitToke[0].replaceAll('[', '["');
tokersLine = tokersLine.replaceAll(']','"]');
tokersLine = tokersLine.replaceAll(',','","');
//Force feed doctored line into the JSON parser, and iterate by the array it shits out
for(const toker of JSON.parse(tokersLine)){
toke.set(toker,"Legacy Tokes");
}
const date = new Date(Number(splitToke[2]));
//Push toke on to statDB
statDB.tokes.push({
toke,
date
});
console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
}
}
}
//Save toke to file
await statDB.save();
console.log("Legacy tokes commited to server-wide database statistics file!");
}catch(err){
return loggerutils.localexceptionhandler(err);
}
}
statSchema.statics.dropLegacyTokes = async function(){
try{
//If legacy toke dropping is disabled or migration is enabled
if(!config.dropLegacyTokes || config.migrate){
//return
return;
}
//pull stat doc
const statDB = await this.getStats();
//Create temporary toke array
const tokes = [];
//Iterate through server toke history
for(const toke of statDB.tokes){
//If it's not a legacy toke
if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
//Add it to the temp array
tokes.push(toke);
}
}
//Replace the server-wide toke log with our newly doctored one
statDB.tokes = tokes;
//Save the stat document
await statDB.save();
//Tell of our success
console.log("Removed migration tokes!");
}catch(err){
return loggerutils.localexceptionhandler(err);
}
}
//Methods
/**
* Gets toke counts for each individual callout in a map
* @returns {Map} Map of toke counts for each individual callout registered to the server
*/
statSchema.methods.calculateTokeCommandCounts = async function(){
//Create empty map to hold toke command counts
const count = new Map();
//for each toke
this.tokes.forEach((toke) => {
//For each toke command called in the current toke
toke.toke.forEach((command) => {
//Get the current count for the current command
var curCount = count.get(command);
//if the current count is null
if(curCount == null){
//Set it to one
count.set(command, 1);
}else{
//Set it to ++curCount
count.set(command, curCount + 1);
}
});
});
//return the toke command count
return count;
}
module.exports = mongoose.model("statistics", statSchema);

219
src/schemas/tokebot/tokeSchema.js Normal file
View file

@ -0,0 +1,219 @@
/*Canopy - The next generation of stoner streaming software
Copyright (C) 2024-2025 Rainbownapkin and the TTN Community
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.*/
//NPM Imports
const {mongoose} = require('mongoose');
//Local Imports
const config = require('./../../../config.json');
const loggerUtils = require('../../utils/loggerUtils');
/**
* DB Schema for single document for keeping track of a single toke
*/
const tokeSchema = new mongoose.Schema({
toke: {
type: mongoose.SchemaTypes.Map,
required: true,
default: new Map()
},
date: {
type: mongoose.SchemaTypes.Date,
required: true,
default: new Date()
}
});
//statics
/**
* Cached map containing counts of individual toke commands
*/
tokeSchema.statics.tokeMap = new Map();
/**
* Cached number of total tokes
*/
tokeSchema.statics.count = 0;
/**
* Cached number of times a user has successfully ran a '!toke' command
* Not to be confused with tokeSchema.statics.count, which counts total amount of tokes called out
*/
tokeSchema.statics.commandCount = 0;
/**
* Calculates cached toke map from existing
*/
tokeSchema.statics.calculateTokeMap = async function(){
//Pull full toke collection
const tokes = await this.find();
//Drop existing toke map
this.tokeMap = new Map();
//Iterate through DB of tokes
for(const toke of tokes){
//Increment toke count
this.count++;
//For each command callout
for(const command of toke.toke){
//Increment Command Count
this.commandCount++;
//Pull current count of respective toke command
let curCount = this.tokeMap.get(command[1]);
//If this is an unset toke command
if(curCount == null){
//Set it to one
this.tokeMap.set(command[1], 1);
//Otherwise
}else{
//Increment the existing count
this.tokeMap.set(command[1], ++curCount);
}
}
}
//Display calculated toke sats for funsies
if(config.verbose){
console.log(`Processed ${this.commandCount} toke command callouts accross ${await this.estimatedDocumentCount()} tokes.`);
}
}
/**
* Tattoos toke into toke DB colleciton
*
* We use this instead of a pre-save function as we need to fuck w/ statics
*/
tokeSchema.statics.tattooToke = async function(toke){
//Write toke to DB
await this.create({toke});
//Increment RAM-backed toke count
this.count++;
//Iterate through tokers
for(const curToke of toke){
//Pull current toke count
let curCount = this.tokeMap.get(curToke[1]);
//If this command hasn't been counted
if(curCount == null){
//Set new command count to one
this.tokeMap.set(curToke[1], 1);
}else{
//Increment current toke count and commit it to the RAM-backed tokeMap
this.tokeMap.set(curToke[1], ++curCount);
}
//Increment RAM-Backed command count
this.commandCount++;
}
}
/**
* Ingests legacy tokes handed over by the migration model
* @param {Array} rawLegacyTokes - List of strings containing contents of legacy cytube/fore.st toke logs
*/
tokeSchema.statics.ingestLegacyTokes = async function(rawLegacyTokes){
//If migration is disabled
if(!config.migrate){
//BAIL!
return;
}
try{
//For each toke log
for(const tokeLog of rawLegacyTokes){
//Split and iterate toke log by new line
for(const tokeLine of tokeLog.split('\n')){
//Ensure line is a valid toke log line (this will break if your tokes take place after 12:46:40PM on Nov 20th 2286... Or before 21:46:40 Sep 08 2001)
//You'll probably want to have migrated from cytube/fore.st to canopy by then :)
//Also splits tokers array off for easier processing
const splitToke = tokeLine.match(/^\[.+\]|,[0-9]{1,4},|[0-9]{13}$/g)
if(splitToke != null){
//Create empty tokers map
const toke = new Map();
//Add qoutes around strings in the tokers line
let tokersLine = splitToke[0].replaceAll('[', '["');
tokersLine = tokersLine.replaceAll(']','"]');
tokersLine = tokersLine.replaceAll(',','","');
//Force feed doctored line into the JSON parser, and iterate by the array it shits out
for(const toker of JSON.parse(tokersLine)){
toke.set(toker,"Legacy Tokes");
}
const date = new Date(Number(splitToke[2]));
//Push toke on to statDB
this.create({
toke,
date
});
console.log(`Adding legacy toke: ${tokersLine} from: ${date.toLocaleString()}`);
}
}
}
console.log("Legacy tokes commited to server-wide database statistics file!");
}catch(err){
return loggerUtils.localExceptionHandler(err);
}
}
tokeSchema.statics.dropLegacyTokes = async function(){
try{
//If legacy toke dropping is disabled or migration is enabled
if(!config.dropLegacyTokes || config.migrate){
//return
return;
}
//Pull tokes from DB
const oldTokes = await this.find();
//Create temporary toke array
const tokes = [];
//Nuke the toke collection
await this.deleteMany({});
//Iterate through server toke history
for(const toke of oldTokes){
//If it's not a legacy toke or a dupe
if(Array.from(toke.toke)[0][1] != "Legacy Tokes"){
//Re-add it to the database, scraping out the old ID
this.create({
toke: toke.toke,
date: toke.date
});
}
}
//Tell of our success
console.log("Removed migration tokes!");
}catch(err){
return loggerUtils.localExceptionHandler(err);
}
}
module.exports = mongoose.model("toke", tokeSchema);

6
src/schemas/user/migrationSchema.js
View file

@ -24,7 +24,7 @@ const {mongoose} = require('mongoose');
const config = require('../../../config.json');
const {userModel} = require('../user/userSchema');
const permissionModel = require('../permissionSchema');
const statModel = require('../statSchema');
const tokeModel = require('../tokebot/tokeSchema');
const loggerUtils = require('../../utils/loggerUtils');
@ -78,7 +78,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){
try{
//If migration is disabled
if(!config.migrate){
statModel.dropLegacyTokes();
await tokeModel.dropLegacyTokes();
//BAIL!
return;
}
@ -145,7 +145,7 @@ migrationSchema.statics.ingestLegacyDump = async function(){
await this.ingestTokeMaps(tokeMaps);
//Pass toke logs over to the stat model for further ingestion
await statModel.ingestLegacyTokes(tokeLogs);
await tokeModel.ingestLegacyTokes(tokeLogs);
loggerUtils.consoleWarn(`Legacy Server Migration Completed at: ${new Date().toLocaleString()}`);
}catch(err){

11
src/schemas/user/userSchema.js
View file

@ -22,6 +22,7 @@ const {mongoose} = require('mongoose');
const server = require('../../server');
//DB Models
const statModel = require('../statSchema');
const tokeModel = require('../tokebot/tokeSchema');
const flairModel = require('../flairSchema');
const permissionModel = require('../permissionSchema');
const emoteModel = require('../emoteSchema');
@ -311,16 +312,14 @@ userSchema.statics.findProfile = async function(user, includeEmail = false){
return null;
//If someone's looking for tokebot
}else if(user.user.toLowerCase() == "tokebot"){
//Pull statistics document from the database
const statDB = await statModel.getStats();
//fake a profile hashtable for tokebot
const profile = {
id: -420,
user: "Tokebot",
date: statDB.firstLaunch,
tokes: await statDB.calculateTokeCommandCounts(),
tokeCount: statDB.tokes.length,
//Look ma, no DB calls!
date: statModel.firstLaunch,
tokes: tokeModel.tokeMap,
tokeCount: tokeModel.count,
img: "/nonfree/johnny.png",
signature: "!TOKE",
bio: "!TOKE OR DIE!"