import sio from 'socket.io'; import db from '../database'; import User from '../user'; import Server from '../server'; import Config from '../config'; const cookieParser = require("cookie-parser")(Config.get("http.cookie-secret")); import typecheck from 'json-typecheck'; import { isTorExit } from '../tor'; import session from '../session'; import counters from '../counters'; import { verifyIPSessionCookie } from '../web/middleware/ipsessioncookie'; import Promise from 'bluebird'; const verifySession = Promise.promisify(session.verifySession); const getAliases = Promise.promisify(db.getAliases); import { CachingGlobalBanlist } from './globalban'; import proxyaddr from 'proxy-addr'; import { Counter, Gauge } from 'prom-client'; import Socket from 'socket.io/lib/socket'; import { TokenBucket } from '../util/token-bucket'; import http from 'http'; const LOGGER = require('@calzoneman/jsli')('ioserver'); const rateLimitExceeded = new Counter({ name: 'cytube_socketio_rate_limited_total', help: 'Number of socket.io connections rejected due to exceeding rate limit' }); const connLimitExceeded = new Counter({ name: 'cytube_socketio_conn_limited_total', help: 'Number of socket.io connections rejected due to exceeding conn limit' }); const authFailureCount = new Counter({ name: 'cytube_socketio_auth_error_total', help: 'Number of failed authentications from session middleware' }); class IOServer { constructor(options = { proxyTrustFn: proxyaddr.compile('127.0.0.1') }) { ({ proxyTrustFn: this.proxyTrustFn } = options); this.ipThrottle = new Map(); this.ipCount = new Map(); } // Map proxied sockets to the real IP address via X-Forwarded-For // If the resulting address is a known Tor exit, flag it as such ipProxyMiddleware(socket, next) { if (!socket.context) socket.context = {}; try { socket.handshake.connection = { remoteAddress: socket.handshake.address }; socket.context.ipAddress = proxyaddr( socket.handshake, this.proxyTrustFn ); if (!socket.context.ipAddress) { throw new Error( `Assertion failed: unexpected IP ${socket.context.ipAddress}` ); } } catch (error) { LOGGER.warn('Rejecting socket - proxyaddr failed: %s', error); next(new Error('Could not determine IP address')); return; } if (isTorExit(socket.context.ipAddress)) { socket.context.torConnection = true; } next(); } // Reject global banned IP addresses ipBanMiddleware(socket, next) { if (isIPGlobalBanned(socket.context.ipAddress)) { LOGGER.info('Rejecting %s - banned', socket.context.ipAddress); next(new Error('You are banned from the server')); return; } next(); } // Rate limit connection attempts by IP address ipThrottleMiddleware(socket, next) { if (!this.ipThrottle.has(socket.context.ipAddress)) { this.ipThrottle.set(socket.context.ipAddress, new TokenBucket(5, 0.1)); } const bucket = this.ipThrottle.get(socket.context.ipAddress); if (bucket.throttle()) { rateLimitExceeded.inc(1); LOGGER.info('Rejecting %s - exceeded connection rate limit', socket.context.ipAddress); next(new Error('Rate limit exceeded')); return; } next(); } /* TODO: see https://github.com/calzoneman/sync/issues/724 ipConnectionLimitMiddleware(socket, next) { const ip = socket.context.ipAddress; const count = this.ipCount.get(ip) || 0; if (count >= Config.get('io.ip-connection-limit')) { // TODO: better error message would be nice next(new Error('Too many connections from your IP address')); return; } this.ipCount.set(ip, count + 1); console.log(ip, this.ipCount.get(ip)); socket.once('disconnect', () => { console.log('Disconnect event has fired for', socket.id); this.ipCount.set(ip, this.ipCount.get(ip) - 1); }); next(); } */ checkIPLimit(socket) { const ip = socket.context.ipAddress; const count = this.ipCount.get(ip) || 0; if (count >= Config.get('io.ip-connection-limit')) { connLimitExceeded.inc(1); LOGGER.info( 'Rejecting %s - exceeded connection count limit', ip ); socket.emit('kick', { reason: 'Too many connections from your IP address' }); socket.disconnect(true); return false; } this.ipCount.set(ip, count + 1); socket.once('disconnect', () => { const newCount = (this.ipCount.get(ip) || 1) - 1; if (newCount === 0) { this.ipCount.delete(ip); } else { this.ipCount.set(ip, newCount); } }); return true; } // Parse cookies cookieParsingMiddleware(socket, next) { const req = socket.handshake; if (req.headers.cookie) { cookieParser(req, null, () => next()); } else { req.cookies = {}; req.signedCookies = {}; next(); } } // Determine session age from ip-session cookie // (Used for restricting chat) ipSessionCookieMiddleware(socket, next) { const cookie = socket.handshake.signedCookies['ip-session']; if (!cookie) { socket.context.ipSessionFirstSeen = new Date(); next(); return; } const sessionMatch = verifyIPSessionCookie(socket.context.ipAddress, cookie); if (sessionMatch) { socket.context.ipSessionFirstSeen = sessionMatch.date; } else { socket.context.ipSessionFirstSeen = new Date(); } next(); } // Match login cookie against the DB, look up aliases authUserMiddleware(socket, next) { socket.context.aliases = []; const promises = []; const auth = socket.handshake.signedCookies.auth; if (auth) { promises.push(verifySession(auth).then(user => { socket.context.user = Object.assign({}, user); }).catch(_error => { authFailureCount.inc(1); LOGGER.warn('Unable to verify session for %s - ignoring auth', socket.context.ipAddress); })); } promises.push(getAliases(socket.context.ipAddress).then(aliases => { socket.context.aliases = aliases; }).catch(_error => { LOGGER.warn('Unable to load aliases for %s', socket.context.ipAddress); })); Promise.all(promises).then(() => next()); } handleConnection(socket) { if (!this.checkIPLimit(socket)) { return; } this.setRateLimiter(socket); emitMetrics(socket); LOGGER.info('Accepted socket from %s', socket.context.ipAddress); counters.add('socket.io:accept', 1); socket.once('disconnect', (reason, reasonDetail) => { LOGGER.info( '%s disconnected (%s%s)', socket.context.ipAddress, reason, reasonDetail ? ` - ${reasonDetail}` : '' ); counters.add('socket.io:disconnect', 1); }); const user = new User(socket, socket.context.ipAddress, socket.context.user); if (socket.context.user) { db.recordVisit(socket.context.ipAddress, user.getName()); } const announcement = Server.getServer().announcement; if (announcement !== null) { socket.emit('announcement', announcement); } } setRateLimiter(socket) { const refillRate = () => Config.get('io.throttle.in-rate-limit'); const capacity = () => Config.get('io.throttle.bucket-capacity'); socket._inRateLimit = new TokenBucket(capacity, refillRate); socket.on('cytube:count-event', () => { if (socket._inRateLimit.throttle()) { LOGGER.warn( 'Kicking client %s: exceeded in-rate-limit of %d', socket.context.ipAddress, refillRate() ); socket.emit('kick', { reason: 'Rate limit exceeded' }); socket.disconnect(); } }); } initSocketIO() { patchSocketMetrics(); patchTypecheckedFunctions(); const io = this.io = sio.instance = sio(); io.use(this.ipProxyMiddleware.bind(this)); io.use(this.ipBanMiddleware.bind(this)); io.use(this.ipThrottleMiddleware.bind(this)); //io.use(this.ipConnectionLimitMiddleware.bind(this)); io.use(this.cookieParsingMiddleware.bind(this)); io.use(this.ipSessionCookieMiddleware.bind(this)); io.use(this.authUserMiddleware.bind(this)); io.on('connection', this.handleConnection.bind(this)); } bindTo(servers) { if (!this.io) { throw new Error('Cannot bind: socket.io has not been initialized yet'); } const engineOpts = { /* * Set ping timeout to 2 minutes to avoid spurious reconnects * during transient network issues. The default of 5 minutes * is too aggressive. * * https://github.com/calzoneman/sync/issues/780 */ pingTimeout: 120000, /* * Per `ws` docs: "Note that Node.js has a variety of issues with * high-performance compression, where increased concurrency, * especially on Linux, can lead to catastrophic memory * fragmentation and slow performance." * * CyTube's frames are ordinarily quite small, so there's not much * point in compressing them. */ perMessageDeflate: false, httpCompression: false, /* * Default is 10MB. * Even 1MiB seems like a generous limit... */ maxHttpBufferSize: 1 << 20 }; servers.forEach(server => { this.io.attach(server, engineOpts); }); } } const incomingEventCount = new Counter({ name: 'cytube_socketio_incoming_events_total', help: 'Number of received socket.io events from clients' }); const outgoingPacketCount = new Counter({ name: 'cytube_socketio_outgoing_packets_total', help: 'Number of outgoing socket.io packets to clients' }); function patchSocketMetrics() { const onevent = Socket.prototype.onevent; const packet = Socket.prototype.packet; const emit = require('events').EventEmitter.prototype.emit; Socket.prototype.onevent = function patchedOnevent() { onevent.apply(this, arguments); incomingEventCount.inc(1); emit.call(this, 'cytube:count-event'); }; Socket.prototype.packet = function patchedPacket() { packet.apply(this, arguments); outgoingPacketCount.inc(1); }; } /* TODO: remove this crap */ function patchTypecheckedFunctions() { Socket.prototype.typecheckedOn = function typecheckedOn(msg, template, cb) { this.on(msg, (data, ack) => { typecheck(data, template, (err, data) => { if (err) { this.emit("errorMsg", { msg: "Unexpected error for message " + msg + ": " + err.message }); } else { cb(data, ack); } }); }); }; Socket.prototype.typecheckedOnce = function typecheckedOnce(msg, template, cb) { this.once(msg, data => { typecheck(data, template, (err, data) => { if (err) { this.emit("errorMsg", { msg: "Unexpected error for message " + msg + ": " + err.message }); } else { cb(data); } }); }); }; } let globalIPBanlist = null; function isIPGlobalBanned(ip) { if (globalIPBanlist === null) { globalIPBanlist = new CachingGlobalBanlist(db.getGlobalBanDB()); globalIPBanlist.refreshCache(); globalIPBanlist.startCacheTimer(60 * 1000); } return globalIPBanlist.isIPGlobalBanned(ip); } const promSocketCount = new Gauge({ name: 'cytube_sockets_num_connected', help: 'Gauge of connected socket.io clients', labelNames: ['transport'] }); const promSocketAccept = new Counter({ name: 'cytube_sockets_accepts_total', help: 'Counter for number of connections accepted. Excludes rejected connections.' }); const promSocketDisconnect = new Counter({ name: 'cytube_sockets_disconnects_total', help: 'Counter for number of connections disconnected.' }); const promSocketReconnect = new Counter({ name: 'cytube_sockets_reconnects_total', help: 'Counter for number of reconnects detected.' }); function emitMetrics(sock) { try { let closed = false; let transportName = sock.client.conn.transport.name; promSocketCount.inc({ transport: transportName }); promSocketAccept.inc(1); sock.client.conn.on('upgrade', newTransport => { try { // Sanity check if (!closed && newTransport.name !== transportName) { promSocketCount.dec({ transport: transportName }); transportName = newTransport.name; promSocketCount.inc({ transport: transportName }); } } catch (error) { LOGGER.error('Error emitting transport upgrade metrics for socket (ip=%s): %s', sock.context.ipAddress, error.stack); } }); sock.once('disconnect', () => { try { closed = true; promSocketCount.dec({ transport: transportName }); promSocketDisconnect.inc(1); } catch (error) { LOGGER.error('Error emitting disconnect metrics for socket (ip=%s): %s', sock.context.ipAddress, error.stack); } }); sock.once('reportReconnect', () => { try { promSocketReconnect.inc(1, new Date()); } catch (error) { LOGGER.error('Error emitting reconnect metrics for socket (ip=%s): %s', sock.context.ipAddress, error.stack); } }); } catch (error) { LOGGER.error('Error emitting metrics for socket (ip=%s): %s', sock.context.ipAddress, error.stack); } } let instance = null; module.exports = { init: function (srv, webConfig) { if (instance !== null) { throw new Error('ioserver.init: already initialized'); } const ioServer = instance = new IOServer({ proxyTrustFn: proxyaddr.compile(webConfig.getTrustedProxies()) }); ioServer.initSocketIO(); const uniqueListenAddresses = new Set(); const servers = []; Config.get("listen").forEach(function (bind) { if (!bind.io) { return; } const id = bind.ip + ":" + bind.port; if (uniqueListenAddresses.has(id)) { LOGGER.warn("Ignoring duplicate listen address %s", id); return; } if (srv.servers.hasOwnProperty(id)) { servers.push(srv.servers[id]); } else { const server = http.createServer().listen(bind.port, bind.ip); servers.push(server); server.on("error", error => { if (error.code === "EADDRINUSE") { LOGGER.fatal( "Could not bind %s: address already in use. Check " + "whether another application has already bound this " + "port, or whether another instance of this server " + "is running.", id ); process.exit(1); } }); } uniqueListenAddresses.add(id); }); ioServer.bindTo(servers); }, IOServer: IOServer }; /* Clean out old rate limiters */ setInterval(function () { if (instance == null) return; let cleaned = 0; const keys = instance.ipThrottle.keys(); for (const key of keys) { if (instance.ipThrottle.get(key).lastRefill < Date.now() - 60000) { const bucket = instance.ipThrottle.delete(key); for (const k in bucket) delete bucket[k]; cleaned++; } } if (cleaned > 0) { LOGGER.info('Cleaned up %d stale IP throttle token buckets', cleaned); } }, 5 * 60 * 1000);