diff --git a/changelog b/changelog deleted file mode 100644 index 82d53fef..00000000 --- a/changelog +++ /dev/null @@ -1,14 +0,0 @@ -3.0.0 - -* Changed web interface to Bootstrap 3 -* Replaced hacky POST AJAX with dynamic pages rendered via Jade -* Replaced AJAX login with HTTP-only cookies -* Refactored database code -* Refactored webserver code -* Updated username validation to allow accented letters (e.g. á, ö) - -Planned: - -* Support IPv6 -* Better user playlist management -* Better channel settings interface diff --git a/import.js b/import.js deleted file mode 100644 index 6571a554..00000000 --- a/import.js +++ /dev/null @@ -1,493 +0,0 @@ -/** - * Utility for importing a CyTube 2.4.6 database to 3.0 - */ - -var mysql = require("mysql"); -var AsyncQueue = require("./lib/asyncqueue"); -var tables = require("./lib/database/tables"); - -var olddb = { - host: "", - user: "", - password: "", - database: "" -}; - -var newdb = { - host: "", - user: "", - password: "", - database: "" -}; - -var oldpool; -var newpool; - -function query(pool, query, sub, callback) { - // 2nd argument is optional - if (typeof sub === "function") { - callback = sub; - sub = false; - } - - if (typeof callback !== "function") { - callback = function () { }; - } - - pool.getConnection(function (err, conn) { - if (err) { - console.log("[ERROR] DB connection failed: " + err); - callback("Database failure", null); - } else { - function cback(err, res) { - if (err) { - console.log("[ERROR] DB query failed: " + query); - if (sub) { - console.log("[ERROR] Substitutions: " + sub); - } - console.log("[ERROR] " + err); - callback("Database failure", null); - } else { - callback(null, res); - } - conn.release(); - } - - if (sub) { - conn.query(query, sub, cback); - } else { - conn.query(query, cback); - } - } - }); -}; - -var queryOld; -var queryNew; - -function chain(/* arguments */) { - var args = Array.prototype.slice.call(arguments); - var cb = args.pop(); - var next = function () { - if (args.length > 0) { - args.shift()(next); - } else { - cb(); - } - }; - - next(); -} - -/** - * Imports entries from the registrations table of 2.4.6 to the users table of 3.0 - */ -function importUsers(cb) { - console.log("[INFO] Importing users"); - var insert = "INSERT INTO `users` (`name`, `password`, `global_rank`, " + - "`email`, `profile`, `time`) VALUES (?, ?, ?, ?, ?, ?)"; - queryOld("SELECT * FROM `registrations`", function (err, rows) { - if (err) { - cb(err); - return; - } - - rows.sort(function (a, b) { - return a.id - b.id; - }); - - var aq = new AsyncQueue(); - rows.forEach(function (r) { - var data = [r.uname, r.pw, r.global_rank, r.email, - JSON.stringify({ image: r.profile_image, text: r.profile_text }), - Date.now()]; - aq.queue(function (lock) { - queryNew(insert, data, function (err) { - if (!err) { - console.log("Imported user " + r.uname); - } - lock.release(); - }); - }); - }); - - aq.queue(function (lock) { - lock.release(); - cb(); - }); - }); -} - -/** - * Imports channel registration entries from `channels` table - */ -function importChannelRegistrations(cb) { - var insert = "INSERT INTO `channels` (`name`, `owner`, `time`) VALUES (?, ?, ?)"; - - queryOld("SELECT * FROM channels", function (err, rows) { - if (err) { - cb(err); - return; - } - - rows.sort(function (a, b) { - return a.id - b.id; - }); - - var aq = new AsyncQueue(); - rows.forEach(function (r) { - var data = [r.name, r.owner, Date.now()]; - aq.queue(function (lock) { - queryNew(insert, data, function (err) { - if (!err) { - console.log("Imported channel record " + r.name + " (" + r.owner + ")"); - } - lock.release(); - }); - }); - }); - - aq.queue(function (lock) { - lock.release(); - cb(); - }); - }); -} - -/** - * Imports ranks/bans/library - */ -function importChannelTables(cb) { - console.log("Importing channel ranks, libraries, bans"); - queryOld("SELECT * FROM `channels`", function (err, rows) { - if (err) { - cb(err); - return; - } - - var aq = new AsyncQueue(); - - rows.forEach(function (r) { - aq.queue(function (lock) { - console.log("Creating channel tables for "+r.name); - tables.createChannelTables(r.name, queryNew, function () { - copyChannelTables(r.name, function () { - lock.release(); - }); - }); - }); - }); - - aq.queue(function (lock) { - lock.release(); - cb(); - }); - }); -} - -function copyChannelTables(name, cb) { - var copyRanks = function () { - queryOld("SELECT * FROM `chan_"+name+"_ranks`", function (err, rows) { - if (err) { - cb(err); - return; - } - - rows = rows.filter(function (r) { - return r.rank > 1; - }); - - rows = rows.map(function (r) { - if (r.rank === 10) { - r.rank = 5; - } else if (r.rank > 3 && r.rank < 10) { - r.rank = 4; - } - return [r.name, r.rank]; - }); - - if (rows.length === 0) { - console.log("`chan_"+name+"_ranks` is empty"); - copyLibrary(); - return; - } - - console.log("Copying `chan_"+name+"_ranks`"); - queryNew("INSERT INTO `chan_"+name+"_ranks` VALUES ?", [rows], copyLibrary); - }); - }; - - var copyLibrary = function () { - queryOld("SELECT * FROM `chan_"+name+"_library`", function (err, rows) { - if (err) { - cb(err); - return; - } - - rows = rows.map(function (r) { - return [r.id, r.title, r.seconds, r.type]; - }); - - if (rows.length === 0) { - console.log("`chan_"+name+"_library` is empty"); - copyBans(); - return; - } - - var subs = []; - while (rows.length > 1000) { - subs.push(rows.slice(0, 1000)); - rows = rows.slice(1000); - } - - if (rows.length > 0) { - subs.push(rows); - } - - if (subs.length > 1) { - console.log("`chan_"+name+"_library` is >1000 rows, requires multiple inserts"); - } - - var aq = new AsyncQueue(); - subs.forEach(function (s) { - aq.queue(function (lock) { - console.log("Copying `chan_"+name+"_library`"); - queryNew("INSERT INTO `chan_"+name+"_library` VALUES ?", - [s], function () { - lock.release(); - }); - }); - }); - - aq.queue(function (lock) { - lock.release(); - copyBans(); - }); - }); - }; - - var copyBans = function () { - queryOld("SELECT * FROM `chan_"+name+"_bans`", function (err, rows) { - if (err) { - cb(err); - return; - } - - rows = rows.map(function (r) { - return [r.id, r.ip, r.name, r.bannedby, r.reason]; - }); - - if (rows.length === 0) { - console.log("`chan_"+name+"_bans` is empty"); - cb(); - return; - } - - console.log("Copying `chan_"+name+"_bans`"); - queryNew("INSERT INTO `chan_"+name+"_bans` VALUES ?", [rows], cb); - }); - }; - - copyRanks(); -} - -function importGlobalBans(cb) { - console.log("Importing global bans"); - queryOld("SELECT * FROM `global_bans`", function (err, bans) { - if (err) { - cb(err); - return; - } - - bans = bans.map(function (b) { - return [b.ip, b.reason]; - }); - queryNew("INSERT INTO `global_bans` VALUES ?", [bans], cb); - }); -} - -function importUserPlaylists(cb) { - console.log("Importing user playlists"); - queryOld("SELECT * FROM `user_playlists`", function (err, pls) { - if (err) { - cb(err); - return; - } - - pls = pls.map(function (pl) { - return [pl.user, pl.name, pl.contents, pl.count, pl.duration]; - }); - var subs = []; - while (pls.length > 10) { - subs.push(pls.slice(0, 10)); - pls = pls.slice(10); - } - - if (pls.length > 0) { - subs.push(pls); - } - - var aq = new AsyncQueue(); - subs.forEach(function (s) { - aq.queue(function (lock) { - queryNew("INSERT INTO `user_playlists` VALUES ?", [s], function () { - lock.release(); - }); - }); - }); - - aq.queue(function (lock) { - lock.release(); - cb(); - }); - }); -} - -function importAliases(cb) { - console.log("Importing aliases"); - queryOld("SELECT * FROM `aliases`", function (err, aliases) { - if (err) { - cb(err); - return; - } - - aliases = aliases.map(function (al) { - return [al.visit_id, al.ip, al.name, al.time]; - }); - - var subs = []; - while (aliases.length > 1000) { - subs.push(aliases.slice(0, 1000)); - aliases = aliases.slice(1000); - } - - if (aliases.length > 0) { - subs.push(aliases); - } - - var aq = new AsyncQueue(); - subs.forEach(function (s) { - aq.queue(function (lock) { - queryNew("INSERT INTO `aliases` VALUES ?", [s], function () { - lock.release(); - }); - }); - }); - - aq.queue(function (lock) { - lock.release(); - cb(); - }); - }); -} - -function main() { - var aq = new AsyncQueue(); - var readline = require("readline"); - console.log("This script will generate a lot of text output, both informational and " + - "possibly errors. I recommend running it as `node import.js | " + - "tee import.log` or similar to pipe output to a log file for easy reading"); - var rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }); - aq.queue(function (lock) { - rl.question("2.x host: ", function (host) { - olddb.host = host; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("2.x username: ", function (user) { - olddb.user = user; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("2.x password: ", function (pw) { - olddb.password = pw; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("2.x database: ", function (db) { - olddb.database = db; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("3.0 host: ", function (host) { - newdb.host = host; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("3.0 username: ", function (user) { - newdb.user = user; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("3.0 password: ", function (pw) { - newdb.password = pw; - lock.release(); - }); - }); - aq.queue(function (lock) { - rl.question("3.0 database: ", function (db) { - newdb.database = db; - lock.release(); - }); - }); - aq.queue(function (lock) { - oldpool = mysql.createPool(olddb); - newpool = mysql.createPool(newdb); - queryOld = query.bind(this, oldpool); - queryNew = query.bind(this, newpool); - startImport(); - }); -} - -function startImport() { - tables.init(queryNew, function (err) { - if (!err) { - var aq = new AsyncQueue(); - aq.queue(function (lock) { - importUsers(function () { - lock.release(); - }); - }); - aq.queue(function (lock) { - importChannelRegistrations(function () { - lock.release(); }); - }); - aq.queue(function (lock) { - importChannelTables(function () { - lock.release(); - }); - }); - aq.queue(function (lock) { - importGlobalBans(function () { - lock.release(); - }); - }); - aq.queue(function (lock) { - importUserPlaylists(function () { - lock.release(); - }); - }); - aq.queue(function (lock) { - importAliases(function () { - lock.release(); - }); - }); - aq.queue(function (lock) { - process.exit(0); - }); - } else { - console.log("[ERROR] Aborting due to errors initializing tables"); - } - }); -} - -main(); diff --git a/update.js b/update.js deleted file mode 100644 index 8986b3fc..00000000 --- a/update.js +++ /dev/null @@ -1,180 +0,0 @@ -var Config = require("./lib/config.js"); -var Database = require("./lib/database.js"); - -var updates = { - "2013-08-20-utf8fix": fixDBUnicode, - "2013-08-21-banfix": fixChannelBanKey -}; - -var x = {}; -Config.load(x, "cfg.json", function () { - var db = new Database(x.cfg); - - var u = process.argv[2]; - if(!(u in updates)) { - console.log("Usage: node update.js "); - console.log("Available updates: "); - for(var k in updates) { - console.log(" " + k); - } - process.exit(0); - } - - var fn = updates[u]; - fn(db); -}); - -/* - 2013-08-20 - - This function iterates over tables in the database and converts the - encoding on each to UTF-8. - - Furthermore, it does the following to convert channel libraries in - a way such that UTF-8 titles stored in other encodings (e.g. latin1) - are preserved as UTF-8: - 1. Change the `title` column to BLOB (unencoded) - 2. Change the table character set to utf8 - 3. Change the `title` column to VARCHAR(255) CHARACTER SET utf8 - - This corrects an encoding issue that was exposed when switching to - node-mysql. mysql-libmysqlclient ignored database encoding and assumed - the data was UTF-8. - -*/ - -function fixDBUnicode(db) { - db.query("SHOW TABLES", function (err, res) { - if(err) { - console.log(err); - return; - } - - var libs = []; - var waiting = res.length; - res.forEach(function (r) { - var k = Object.keys(r)[0]; - if(r[k].match(/^chan_[\w-_]{1,30}_library$/)) { - libs.push(r[k]); - waiting--; - return; - } else if(r[k] == "user_playlists") { - waiting--; - return; - } - db.query("ALTER TABLE `" + r[k] + "` CONVERT TO CHARACTER SET utf8", function (err, _) { - waiting--; - if(err) - console.log("FAIL: " + r[k]); - else - console.log("Fixed " + r[k]); - }); - }); - var s1int = setInterval(function () { - if(waiting == 0) { - clearInterval(s1int); - waiting = libs.length + 1; - libs.forEach(function (lib) { - db.query("ALTER TABLE `"+lib+"` MODIFY title BLOB", function (err, _) { - if(err) { - console.log(err); - waiting--; - return; - } - db.query("ALTER TABLE `"+lib+"` CHARACTER SET utf8", function (err, _) { - if(err) { - console.log(err); - waiting--; - return; - } - db.query("ALTER TABLE `"+lib+"` MODIFY title VARCHAR(255) CHARACTER SET utf8", function (err, _) { - waiting--; - if(err) { - console.log(err); - return; - } - console.log("Fixed " + lib); - }); - }); - }); - }); - db.query("ALTER TABLE user_playlists MODIFY contents MEDIUMBLOB", function (err, _) { - if(err) { - console.log(err); - waiting--; - return; - } - db.query("ALTER TABLE user_playlists CHARACTER SET utf8", function (err, _) { - if(err) { - console.log(err); - waiting--; - return; - } - db.query("ALTER TABLE user_playlists MODIFY contents MEDIUMTEXT CHARACTER SET utf8", function (err, _) { - waiting--; - if(err) { - console.log(err); - return; - } - console.log("Fixed user_playlists"); - }); - }); - }); - setInterval(function () { - if(waiting == 0) { - console.log("Done"); - process.exit(0); - } - }, 1000); - } - }, 1000); - }); -} - -/* - 2013-08-21 - - This function iterates over channel ban tables and corrects the - PRIMARY KEY. Previously, the table was defined with PRIMARY KEY (ip), - but in reality, (ip, name) should be pairwise unique. - - This corrects the issue where only one name ban can exist in the table - because of the `ip` field "*" being unique. - -*/ - -function fixChannelBanKey(db) { - db.query("SHOW TABLES", function (err, res) { - if(err) { - console.log("SEVERE: SHOW TABLES failed"); - return; - } - - var count = res.length; - res.forEach(function (r) { - var k = Object.keys(r)[0]; - - if(!r[k].match(/^chan_[\w-_]{1,30}_bans$/)) { - count--; - return; - } - - db.query("ALTER TABLE `" + r[k] + "` DROP PRIMARY KEY, ADD PRIMARY KEY (ip, name)", function (err, res) { - count--; - if(err) { - console.log("FAILED: " + r[k]); - return; - } - - console.log("Fixed " + r[k]); - }); - }); - - setInterval(function () { - if(count == 0) { - console.log("Done"); - process.exit(0); - } - }, 1000); - }); -} diff --git a/version.py b/version.py deleted file mode 100644 index fdbeb5d3..00000000 --- a/version.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python3 - -import re -import sys - -PKGVER = re.compile(r'"version":(?:\s+)?"(\d)\.(\d)\.(\d)"') -SERVER = re.compile(r'VERSION(?:\s+)?=(?:\s+)"(\d)\.(\d)\.(\d)";') -OP = "patch" - -if len(sys.argv) > 1 and sys.argv[1] in ["patch", "minor", "major"]: - OP = sys.argv[1] - -pkgbuf = [] -with open("package.json") as f: - for line in f: - m = PKGVER.search(line) - sb = [] - if m: - sb.append(line[:m.start(1)]) - if OP == "major": - sb.append(str(int(line[m.start(1):m.end(1)])+1)) - else: - sb.append(line[m.start(1):m.end(1)]) - sb.append(line[m.end(1):m.start(2)]) - if OP == "minor": - sb.append(str(int(line[m.start(2):m.end(2)])+1)) - elif OP == "major": - sb.append("0") - else: - sb.append(line[m.start(2):m.end(2)]) - sb.append(line[m.end(2):m.start(3)]) - if OP == "patch": - sb.append(str(int(line[m.start(3):m.end(3)])+1)) - elif OP == "minor" or OP == "major": - sb.append("0") - sb.append(line[m.end(3):]) - pkgbuf.append("".join(sb)) - print("package.json: {} => {}".format(line, "".join(sb))) - else: - pkgbuf.append(line) -with open("package.json", "w") as f: - f.write("".join(pkgbuf)) - -serbuf = [] -with open("lib/server.js") as f: - for line in f: - m = SERVER.search(line) - sb = [] - if m: - sb.append(line[:m.start(1)]) - if OP == "major": - sb.append(str(int(line[m.start(1):m.end(1)])+1)) - else: - sb.append(line[m.start(1):m.end(1)]) - sb.append(line[m.end(1):m.start(2)]) - if OP == "minor": - sb.append(str(int(line[m.start(2):m.end(2)])+1)) - elif OP == "major": - sb.append("0") - else: - sb.append(line[m.start(2):m.end(2)]) - sb.append(line[m.end(2):m.start(3)]) - if OP == "patch": - sb.append(str(int(line[m.start(3):m.end(3)])+1)) - elif OP == "minor" or OP == "major": - sb.append("0") - sb.append(line[m.end(3):]) - serbuf.append("".join(sb)) - print("server.js: {} => {}".format(line, "".join(sb))) - else: - serbuf.append(line) -with open("lib/server.js", "w") as f: - f.write("".join(serbuf))