diff --git a/integration_test/testutil/config.js b/integration_test/testutil/config.js index 0b681cb5..aa28b573 100644 --- a/integration_test/testutil/config.js +++ b/integration_test/testutil/config.js @@ -1,4 +1,4 @@ -const loadFromToml = require('cytube-common/lib/configuration/configloader').loadFromToml; +const loadFromToml = require('../../lib/configuration/configloader').loadFromToml; const path = require('path'); class IntegrationTestConfig { diff --git a/package.json b/package.json index 128c7b6d..b6eec028 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "cookie-parser": "^1.4.0", "create-error": "^0.3.1", "csrf": "^3.0.0", - "cytube-common": "git://github.com/CyTube/cytube-common", "cytube-mediaquery": "git://github.com/CyTube/mediaquery", "cytubefilters": "git://github.com/calzoneman/cytubefilters#67c7c69a", "express": "^4.13.3", diff --git a/src/config.js b/src/config.js index 92a4ef8b..89295b42 100644 --- a/src/config.js +++ b/src/config.js @@ -4,7 +4,7 @@ var nodemailer = require("nodemailer"); var net = require("net"); var YAML = require("yamljs"); -import { loadFromToml } from 'cytube-common/lib/configuration/configloader'; +import { loadFromToml } from './configuration/configloader'; import { CamoConfig } from './configuration/camoconfig'; import { PrometheusConfig } from './configuration/prometheusconfig'; diff --git a/src/configuration/configloader.js b/src/configuration/configloader.js new file mode 100644 index 00000000..604151fb --- /dev/null +++ b/src/configuration/configloader.js @@ -0,0 +1,19 @@ +import toml from 'toml'; +import fs from 'fs'; + +/** @module cytube-common/configuration/configloader */ + +/** + * Load a toml file and pass the results to a configuration + * constructor. + * + * @param {function} constructor Constructor to call with the loaded data + * @param {string} filename Path to the toml file to load + * @returns {Object} Configuration object constructed from the provided constructor + * @throws {SyntaxError} Errors propagated from toml.parse() + */ +export function loadFromToml(constructor, filename) { + const rawContents = fs.readFileSync(filename).toString('utf8'); + const configData = toml.parse(rawContents); + return new (constructor)(configData); +} diff --git a/src/counters.js b/src/counters.js index 3c780893..b2ec44ca 100644 --- a/src/counters.js +++ b/src/counters.js @@ -1,7 +1,7 @@ import io from 'socket.io'; import Socket from 'socket.io/lib/socket'; -import * as Metrics from 'cytube-common/lib/metrics/metrics'; -import { JSONFileMetricsReporter } from 'cytube-common/lib/metrics/jsonfilemetricsreporter'; +import * as Metrics from './metrics/metrics'; +import { JSONFileMetricsReporter } from './metrics/jsonfilemetricsreporter'; const LOGGER = require('@calzoneman/jsli')('counters'); diff --git a/src/database.js b/src/database.js index 772afe9c..2a16ad5d 100644 --- a/src/database.js +++ b/src/database.js @@ -4,7 +4,7 @@ var Config = require("./config"); var tables = require("./database/tables"); var net = require("net"); var util = require("./utilities"); -import * as Metrics from 'cytube-common/lib/metrics/metrics'; +import * as Metrics from './metrics/metrics'; import knex from 'knex'; import { GlobalBanDB } from './db/globalban'; diff --git a/src/metrics/jsonfilemetricsreporter.js b/src/metrics/jsonfilemetricsreporter.js new file mode 100644 index 00000000..c94a7f71 --- /dev/null +++ b/src/metrics/jsonfilemetricsreporter.js @@ -0,0 +1,73 @@ +import fs from 'fs'; + +/** MetricsReporter that records metrics as JSON objects in a file, one per line */ +class JSONFileMetricsReporter { + /** + * Create a new JSONFileMetricsReporter that writes to the given file path. + * + * @param {string} filename file path to write to + */ + constructor(filename) { + this.writeStream = fs.createWriteStream(filename, { flags: 'a' }); + this.metrics = {}; + this.timers = {}; + } + + /** + * @see {@link module:cytube-common/metrics/metrics.incCounter} + */ + incCounter(counter, value) { + if (!this.metrics.hasOwnProperty(counter)) { + this.metrics[counter] = 0; + } + + this.metrics[counter] += value; + } + + /** + * Add a time metric + * + * @param {string} timer name of the timer + * @param {number} ms milliseconds to record + */ + addTime(timer, ms) { + if (!this.timers.hasOwnProperty(timer)) { + this.timers[timer] = { + totalTime: 0, + count: 0, + p100: 0 + }; + } + + this.timers[timer].totalTime += ms; + this.timers[timer].count++; + if (ms > this.timers[timer].p100) { + this.timers[timer].p100 = ms; + } + } + + /** + * @see {@link module:cytube-common/metrics/metrics.addProperty} + */ + addProperty(property, value) { + this.metrics[property] = value; + } + + report() { + for (const timer in this.timers) { + this.metrics[timer+':avg'] = this.timers[timer].totalTime / this.timers[timer].count; + this.metrics[timer+':count'] = this.timers[timer].count; + this.metrics[timer+':p100'] = this.timers[timer].p100; + } + + const line = JSON.stringify(this.metrics) + '\n'; + try { + this.writeStream.write(line); + } finally { + this.metrics = {}; + this.timers = {}; + } + } +} + +export { JSONFileMetricsReporter }; diff --git a/src/metrics/metrics.js b/src/metrics/metrics.js new file mode 100644 index 00000000..0088b5b8 --- /dev/null +++ b/src/metrics/metrics.js @@ -0,0 +1,132 @@ +import os from 'os'; + +/** @module cytube-common/metrics/metrics */ + +const MEM_RSS = 'memory:rss'; +const LOAD_1MIN = 'load:1min'; +const TIMESTAMP = 'time'; +const logger = require('@calzoneman/jsli')('metrics'); + +var delegate = null; +var reportInterval = null; +var reportHooks = []; +let warnedNoReporter = false; + +function warnNoReporter() { + if (!warnedNoReporter) { + warnedNoReporter = true; + logger.warn('No metrics reporter configured. Metrics will not be recorded.'); + } +} + +/** + * Increment a metrics counter by the specified amount. + * + * @param {string} counter name of the counter to increment + * @param {number} value optional value to increment by (default 1) + */ +export function incCounter(counter, amount = 1) { + if (delegate === null) { + warnNoReporter(); + } else { + delegate.incCounter(counter, amount); + } +} + +/** + * Start a timer. Returns a handle to use to end the timer. + * + * @param {string} timer name + * @return {object} timer handle + */ +export function startTimer(timer) { + return { + timer: timer, + hrtime: process.hrtime() + }; +} + +/** + * Stop a timer and record the time (as an average) + * + * @param {object} handle timer handle to Stop + */ +export function stopTimer(handle) { + if (delegate === null) { + warnNoReporter(); + return; + } + const [seconds, ns] = process.hrtime(handle.hrtime); + delegate.addTime(handle.timer, seconds*1e3 + ns/1e6); +} + +/** + * Add a property to the current metrics period. + * + * @param {string} property property name to add + * @param {any} property value + */ +export function addProperty(property, value) { + if (delegate === null) { + warnNoReporter(); + } else { + delegate.addProperty(property, value); + } +} + +/** + * Set the metrics reporter to record to. + * + * @param {MetricsReporter} reporter reporter to record metrics to + */ +export function setReporter(reporter) { + delegate = reporter; +} + +/** + * Set the interval at which to report metrics. + * + * @param {number} interval time in milliseconds between successive reports + */ +export function setReportInterval(interval) { + clearInterval(reportInterval); + if (!isNaN(interval) && interval >= 0) { + reportInterval = setInterval(reportLoop, interval); + } +} + +/** + * Add a callback to add additional metrics before reporting. + * + * @param {function(metricsReporter)} hook callback to be invoked before reporting + */ +export function addReportHook(hook) { + reportHooks.push(hook); +} + +/** + * Force metrics to be reported right now. + */ +export function flush() { + reportLoop(); +} + +function addDefaults() { + addProperty(MEM_RSS, process.memoryUsage().rss / 1048576); + addProperty(LOAD_1MIN, os.loadavg()[0]); + addProperty(TIMESTAMP, new Date()); +} + +function reportLoop() { + if (delegate !== null) { + try { + addDefaults(); + reportHooks.forEach(hook => { + hook(delegate); + }); + delegate.report(); + } catch (error) { + logger.error(error.stack); + } + } +} diff --git a/src/partition/partitionchannelindex.js b/src/partition/partitionchannelindex.js index d7fa579f..9d739118 100644 --- a/src/partition/partitionchannelindex.js +++ b/src/partition/partitionchannelindex.js @@ -1,6 +1,6 @@ import Promise from 'bluebird'; import uuid from 'uuid'; -import { runLuaScript } from 'cytube-common/lib/redis/lualoader'; +import { runLuaScript } from '../redis/lualoader'; import path from 'path'; const LOGGER = require('@calzoneman/jsli')('partitionchannelindex'); diff --git a/src/partition/partitionmodule.js b/src/partition/partitionmodule.js index 7d3c30e1..4811db69 100644 --- a/src/partition/partitionmodule.js +++ b/src/partition/partitionmodule.js @@ -1,8 +1,8 @@ -import { loadFromToml } from 'cytube-common/lib/configuration/configloader'; +import { loadFromToml } from '../configuration/configloader'; import { PartitionConfig } from './partitionconfig'; import { PartitionDecider } from './partitiondecider'; import { PartitionClusterClient } from '../io/cluster/partitionclusterclient'; -import RedisClientProvider from 'cytube-common/lib/redis/redisclientprovider'; +import RedisClientProvider from '../redis/redisclientprovider'; import LegacyConfig from '../config'; import path from 'path'; import { AnnouncementRefresher } from './announcementrefresher'; diff --git a/src/redis/lualoader.js b/src/redis/lualoader.js new file mode 100644 index 00000000..0a3ad5a3 --- /dev/null +++ b/src/redis/lualoader.js @@ -0,0 +1,44 @@ +import fs from 'fs'; +import logger from '../logger'; + +const CACHE = {}; +const EVALSHA_CACHE = {}; + +export function loadLuaScript(filename) { + if (CACHE.hasOwnProperty(filename)) { + return CACHE[filename]; + } + + CACHE[filename] = fs.readFileSync(filename).toString('utf8'); + return CACHE[filename]; +} + +function loadAndExecuteScript(redisClient, filename, args) { + return redisClient.scriptAsync('load', loadLuaScript(filename)) + .then(sha => { + EVALSHA_CACHE[filename] = sha; + logger.debug(`Cached ${filename} as ${sha}`); + return runEvalSha(redisClient, filename, args); + }); +} + +function runEvalSha(redisClient, filename, args) { + const evalInput = args.slice(); + evalInput.unshift(EVALSHA_CACHE[filename]) + return redisClient.evalshaAsync.apply(redisClient, evalInput); +} + +export function runLuaScript(redisClient, filename, args) { + if (EVALSHA_CACHE.hasOwnProperty(filename)) { + return runEvalSha(redisClient, filename, args).catch(error => { + if (error.code === 'NOSCRIPT') { + logger.warn(`Got NOSCRIPT error for ${filename}, reloading script`); + return loadAndExecuteScript(redisClient, filename, args); + } else { + throw error; + } + }); + } else { + return loadAndExecuteScript(redisClient, filename, args); + } +} diff --git a/src/redis/redisclientprovider.js b/src/redis/redisclientprovider.js new file mode 100644 index 00000000..7007c9dd --- /dev/null +++ b/src/redis/redisclientprovider.js @@ -0,0 +1,49 @@ +import clone from 'clone'; +import redis from 'redis'; +import Promise from 'bluebird'; +Promise.promisifyAll(redis.RedisClient.prototype); +Promise.promisifyAll(redis.Multi.prototype); + +/** + * Provider for RedisClients. + */ +class RedisClientProvider { + /** + * Create a new RedisClientProvider. + * + * @param {Object} redisConfig default configuration to use + * @see {@link https://www.npmjs.com/package/redis} + */ + constructor(redisConfig) { + this.redisConfig = redisConfig; + } + + /** + * Get a RedisClient. + * + * @param {Object} options optional override configuration for the RedisClient + * @return {RedisClient} redis client using the provided configuration + */ + get(options = {}) { + const config = clone(this.redisConfig); + for (const key in options) { + config[key] = options[key]; + } + + const client = redis.createClient(config); + client.on('error', this._defaultErrorHandler); + + return client; + } + + /** + * Handle an 'error' event from a provided client. + * + * @param {Error} err error from the client + * @private + */ + _defaultErrorHandler(err) { + } +} + +export default RedisClientProvider diff --git a/test/metrics/metricstest.js b/test/metrics/metricstest.js new file mode 100644 index 00000000..717370cb --- /dev/null +++ b/test/metrics/metricstest.js @@ -0,0 +1,33 @@ +var assert = require('assert'); +var JSONFileMetricsReporter = require('../../lib/metrics/jsonfilemetricsreporter').JSONFileMetricsReporter; +var Metrics = require('../../lib/metrics/metrics'); +var os = require('os'); +var fs = require('fs'); +var path = require('path'); + +describe('JSONFileMetricsReporter', function () { + describe('#report', function () { + it('reports metrics to file', function (done) { + const outfile = path.resolve(os.tmpdir(), + 'metrics' + Math.random() + '.txt'); + const reporter = new JSONFileMetricsReporter(outfile); + Metrics.setReporter(reporter); + Metrics.incCounter('abc'); + Metrics.incCounter('abc'); + Metrics.incCounter('def', 10); + Metrics.addProperty('foo', { bar: 'baz' }); + Metrics.flush(); + + setTimeout(function () { + const contents = String(fs.readFileSync(outfile)); + const data = JSON.parse(contents); + assert.strictEqual(data.abc, 2); + assert.strictEqual(data.def, 10); + assert.deepStrictEqual(data.foo, { bar: 'baz' }); + + fs.unlinkSync(outfile); + done(); + }, 100); + }); + }); +});