Remove old flatfile chandump storage
This commit is contained in:
Calvin Montgomery
parent
e3a9915b45
commit
106065184f
11 changed files with 42 additions and 312 deletions
|
|
@ -1,4 +1,3 @@
|
|||
import { FileStore } from './filestore';
|
||||
import { DatabaseStore } from './dbstore';
|
||||
import Config from '../config';
|
||||
import Promise from 'bluebird';
|
||||
|
|
@ -26,11 +25,12 @@ export function save(id, channelName, data) {
|
|||
}
|
||||
|
||||
function loadChannelStore() {
|
||||
switch (Config.get('channel-storage.type')) {
|
||||
case 'database':
|
||||
return new DatabaseStore();
|
||||
case 'file':
|
||||
default:
|
||||
return new FileStore();
|
||||
if (Config.get('channel-storage.type') === 'file') {
|
||||
throw new Error(
|
||||
'channel-storage type "file" is no longer supported. Please see ' +
|
||||
'NEWS.md for instructions on upgrading.'
|
||||
);
|
||||
}
|
||||
|
||||
return new DatabaseStore();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,78 +0,0 @@
|
|||
import Promise from 'bluebird';
|
||||
import { stat } from 'fs';
|
||||
import * as fs from 'graceful-fs';
|
||||
import path from 'path';
|
||||
import { ChannelStateSizeError } from '../errors';
|
||||
|
||||
const readFileAsync = Promise.promisify(fs.readFile);
|
||||
const writeFileAsync = Promise.promisify(fs.writeFile);
|
||||
const readdirAsync = Promise.promisify(fs.readdir);
|
||||
const statAsync = Promise.promisify(stat);
|
||||
const SIZE_LIMIT = 1048576;
|
||||
const CHANDUMP_DIR = path.resolve(__dirname, '..', '..', 'chandump');
|
||||
|
||||
export class FileStore {
|
||||
filenameForChannel(channelName) {
|
||||
return path.join(CHANDUMP_DIR, channelName);
|
||||
}
|
||||
|
||||
load(id, channelName) {
|
||||
const filename = this.filenameForChannel(channelName);
|
||||
return statAsync(filename).then(stats => {
|
||||
if (stats.size > SIZE_LIMIT) {
|
||||
return Promise.reject(
|
||||
new ChannelStateSizeError(
|
||||
'Channel state file is too large',
|
||||
{
|
||||
limit: SIZE_LIMIT,
|
||||
actual: stats.size
|
||||
}
|
||||
)
|
||||
);
|
||||
} else {
|
||||
return readFileAsync(filename);
|
||||
}
|
||||
}).then(fileContents => {
|
||||
try {
|
||||
return JSON.parse(fileContents);
|
||||
} catch (e) {
|
||||
return Promise.reject(new Error('Channel state file is not valid JSON: ' + e));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async save(id, channelName, data) {
|
||||
let original;
|
||||
try {
|
||||
original = await this.load(id, channelName);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
throw error;
|
||||
} else {
|
||||
original = {};
|
||||
}
|
||||
}
|
||||
|
||||
Object.keys(data).forEach(key => {
|
||||
original[key] = data[key];
|
||||
});
|
||||
|
||||
const filename = this.filenameForChannel(channelName);
|
||||
const fileContents = new Buffer(JSON.stringify(original), 'utf8');
|
||||
if (fileContents.length > SIZE_LIMIT) {
|
||||
throw new ChannelStateSizeError(
|
||||
'Channel state size is too large',
|
||||
{
|
||||
limit: SIZE_LIMIT,
|
||||
actual: fileContents.length
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return await writeFileAsync(filename, fileContents);
|
||||
}
|
||||
|
||||
listChannels() {
|
||||
return readdirAsync(CHANDUMP_DIR);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
import Config from '../config';
|
||||
import Promise from 'bluebird';
|
||||
import db from '../database';
|
||||
import { FileStore } from './filestore';
|
||||
import { DatabaseStore } from './dbstore';
|
||||
import { sanitizeHTML } from '../xss';
|
||||
import { ChannelNotFoundError } from '../errors';
|
||||
|
||||
const lookupAsync = Promise.promisify(require('../database/channels').lookup);
|
||||
|
||||
/* eslint no-console: off */
|
||||
|
||||
const EXPECTED_KEYS = [
|
||||
'chatbuffer',
|
||||
'chatmuted',
|
||||
'css',
|
||||
'emotes',
|
||||
'filters',
|
||||
'js',
|
||||
'motd',
|
||||
'openPlaylist',
|
||||
'opts',
|
||||
'permissions',
|
||||
'playlist',
|
||||
'poll'
|
||||
];
|
||||
|
||||
function fixOldChandump(data) {
|
||||
const converted = {};
|
||||
EXPECTED_KEYS.forEach(key => {
|
||||
converted[key] = data[key];
|
||||
});
|
||||
|
||||
if (data.queue) {
|
||||
converted.playlist = {
|
||||
pl: data.queue.map(item => {
|
||||
return {
|
||||
media: {
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
seconds: item.seconds,
|
||||
duration: item.duration,
|
||||
type: item.type,
|
||||
meta: {}
|
||||
},
|
||||
queueby: item.queueby,
|
||||
temp: item.temp
|
||||
};
|
||||
}),
|
||||
pos: data.position,
|
||||
time: data.currentTime
|
||||
};
|
||||
}
|
||||
|
||||
if (data.hasOwnProperty('openqueue')) {
|
||||
converted.openPlaylist = data.openqueue;
|
||||
}
|
||||
|
||||
if (data.hasOwnProperty('playlistLock')) {
|
||||
converted.openPlaylist = !data.playlistLock;
|
||||
}
|
||||
|
||||
if (data.chatbuffer) {
|
||||
converted.chatbuffer = data.chatbuffer.map(entry => {
|
||||
return {
|
||||
username: entry.username,
|
||||
msg: entry.msg,
|
||||
meta: entry.meta || {
|
||||
addClass: entry.msgclass ? entry.msgclass : undefined
|
||||
},
|
||||
time: entry.time
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
if (data.motd && data.motd.motd) {
|
||||
converted.motd = sanitizeHTML(data.motd.motd).replace(/\n/g, '<br>\n');
|
||||
}
|
||||
|
||||
if (data.opts && data.opts.customcss) {
|
||||
converted.opts.externalcss = data.opts.customcss;
|
||||
}
|
||||
|
||||
if (data.opts && data.opts.customjs) {
|
||||
converted.opts.externaljs = data.opts.customjs;
|
||||
}
|
||||
|
||||
if (data.filters && data.filters.length > 0 && Array.isArray(data.filters[0])) {
|
||||
converted.filters = data.filters.map(filter => {
|
||||
let [source, replace, active] = filter;
|
||||
return {
|
||||
source: source,
|
||||
replace: replace,
|
||||
flags: 'g',
|
||||
active: active,
|
||||
filterlinks: false
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
return converted;
|
||||
}
|
||||
|
||||
function migrate(src, dest, opts) {
|
||||
const chanPath = Config.get('channel-path');
|
||||
|
||||
return src.listChannels().then(names => {
|
||||
return Promise.reduce(names, (_, name) => {
|
||||
// A long time ago there was a bug where CyTube would save a different
|
||||
// chandump depending on the capitalization of the channel name in the URL.
|
||||
// This was fixed, but there are still some really old chandumps with
|
||||
// uppercase letters in the name.
|
||||
//
|
||||
// If another chandump exists which is all lowercase, then that one is
|
||||
// canonical. Otherwise, it's safe to load the existing capitalization,
|
||||
// convert it, and save.
|
||||
if (name !== name.toLowerCase()) {
|
||||
if (names.indexOf(name.toLowerCase()) >= 0) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
let id;
|
||||
return lookupAsync(name).then(chan => {
|
||||
id = chan.id;
|
||||
return src.load(id, name);
|
||||
}).then(data => {
|
||||
data = fixOldChandump(data);
|
||||
Object.keys(data).forEach(key => {
|
||||
if (opts.keyWhitelist.length > 0 &&
|
||||
opts.keyWhitelist.indexOf(key) < 0) {
|
||||
delete data[key];
|
||||
} else if (opts.keyBlacklist.length > 0 &&
|
||||
opts.keyBlacklist.indexOf(key) >= 0) {
|
||||
delete data[key];
|
||||
}
|
||||
});
|
||||
return dest.save(id, name, data);
|
||||
}).then(() => {
|
||||
console.log(`Migrated /${chanPath}/${name}`);
|
||||
}).catch(ChannelNotFoundError, _err => {
|
||||
console.log(`Skipping /${chanPath}/${name} (not present in the database)`);
|
||||
}).catch(err => {
|
||||
console.error(`Failed to migrate /${chanPath}/${name}: ${err.stack}`);
|
||||
});
|
||||
}, 0);
|
||||
});
|
||||
}
|
||||
|
||||
function loadOpts(argv) {
|
||||
const opts = {
|
||||
keyWhitelist: [],
|
||||
keyBlacklist: []
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '-w') {
|
||||
opts.keyWhitelist = (argv[i+1] || '').split(',');
|
||||
i++;
|
||||
} else if (argv[i] === '-b') {
|
||||
opts.keyBlacklist = (argv[i+1] || '').split(',');
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
return opts;
|
||||
}
|
||||
|
||||
function main() {
|
||||
Config.load('config.yaml');
|
||||
db.init();
|
||||
const src = new FileStore();
|
||||
const dest = new DatabaseStore();
|
||||
const opts = loadOpts(process.argv.slice(2));
|
||||
|
||||
Promise.delay(1000).then(() => {
|
||||
return migrate(src, dest, opts);
|
||||
}).then(() => {
|
||||
console.log('Migration complete');
|
||||
process.exit(0);
|
||||
}).catch(err => {
|
||||
console.error(`Migration failed: ${err.stack}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -174,23 +174,6 @@ Channel.prototype.initModules = function () {
|
|||
self.logger.log("[init] Loaded modules: " + inited.join(", "));
|
||||
};
|
||||
|
||||
Channel.prototype.getDiskSize = function (cb) {
|
||||
if (this._getDiskSizeTimeout > Date.now()) {
|
||||
return cb(null, this._cachedDiskSize);
|
||||
}
|
||||
|
||||
var self = this;
|
||||
var file = path.join(__dirname, "..", "..", "chandump", self.uniqueName);
|
||||
fs.stat(file, function (err, stats) {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
self._cachedDiskSize = stats.size;
|
||||
cb(null, self._cachedDiskSize);
|
||||
});
|
||||
};
|
||||
|
||||
Channel.prototype.loadState = function () {
|
||||
/* Don't load from disk if not registered */
|
||||
if (!this.is(Flags.C_REGISTERED)) {
|
||||
|
|
|
|||
|
|
@ -65,9 +65,6 @@ var defaults = {
|
|||
"channel-blacklist": [],
|
||||
"channel-path": "r",
|
||||
"channel-save-interval": 5,
|
||||
"channel-storage": {
|
||||
type: "file"
|
||||
},
|
||||
"max-channels-per-user": 5,
|
||||
"max-accounts-per-ip": 5,
|
||||
"guest-login-delay": 60,
|
||||
|
|
@ -436,6 +433,10 @@ function preprocessConfig(cfg) {
|
|||
'bucket-capacity': cfg.io.throttle['in-rate-limit']
|
||||
}, cfg.io.throttle);
|
||||
|
||||
if (!cfg['channel-storage']) {
|
||||
cfg['channel-storage'] = { type: undefined };
|
||||
}
|
||||
|
||||
return cfg;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
var db = require("../database");
|
||||
var valid = require("../utilities").isValidChannelName;
|
||||
var fs = require("fs");
|
||||
var path = require("path");
|
||||
var Flags = require("../flags");
|
||||
var util = require("../utilities");
|
||||
import { createMySQLDuplicateKeyUpdate } from '../util/on-duplicate-key-update';
|
||||
|
|
@ -199,14 +197,6 @@ module.exports = {
|
|||
}
|
||||
});
|
||||
|
||||
fs.unlink(path.join(__dirname, "..", "..", "chandump", name),
|
||||
function (err) {
|
||||
if (err && err.code !== "ENOENT") {
|
||||
LOGGER.error("Deleting chandump failed:");
|
||||
LOGGER.error(err);
|
||||
}
|
||||
});
|
||||
|
||||
callback(err, !err);
|
||||
});
|
||||
},
|
||||
|
|
|
|||
|
|
@ -15,11 +15,6 @@ module.exports = {
|
|||
exists || fs.mkdirSync(chanlogpath);
|
||||
});
|
||||
|
||||
var chandumppath = path.join(__dirname, "../chandump");
|
||||
fs.exists(chandumppath, function (exists) {
|
||||
exists || fs.mkdirSync(chandumppath);
|
||||
});
|
||||
|
||||
var gdvttpath = path.join(__dirname, "../google-drive-subtitles");
|
||||
fs.exists(gdvttpath, function (exists) {
|
||||
exists || fs.mkdirSync(gdvttpath);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ const LOGGER = require('@calzoneman/jsli')('setuid');
|
|||
|
||||
var needPermissionsFixed = [
|
||||
path.join(__dirname, "..", "chanlogs"),
|
||||
path.join(__dirname, "..", "chandump"),
|
||||
path.join(__dirname, "..", "google-drive-subtitles")
|
||||
];
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue