rainbownapkin/fore.st
1
0
Fork 0
Code Issues 13 Pull requests Actions Packages Projects Releases Wiki Activity

Added bulk-queueing from Internet Archive.

Browse source
This commit is contained in:
rainbownapkin 2024-10-29 18:44:01 -04:00
parent 9057ed2899
commit 02dc12e646
11 changed files with 190 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
README.md
View file

@ -1,4 +1,4 @@
fore.st - Pineapple Express += 2 (v1.1.2)
fore.st - Pineapple Express += 3 (v1.1.3)
======
fore.st is the server software for ourfore.st, a community based chat & synced video embedding site tailored to service
@ -24,8 +24,8 @@ You can reach out by bugging rainbownapkin on the ttn discord or ourfore.st, you
- Thanks to calzoneman for making [cytube](https://github.com/calzoneman/sync), that saved our asses.
- Thanks to the core TTN community and everyone else who's ever used it, I was only there for the last handful of years but it was an absolute fuckin' ride. You guys are the best, it isn't TTN but I hope this at least help fills the gap.
## Pineapple Express += 2 1.1.2 Release Notes
- Fixed busted video providers with YT-DLP integration (youtube/dailymotion)
## Pineapple Express += 3 1.1.3 Release Notes
- Add bulk-queueing from Internet Archive
## License
Original fore.st code is provided under the Affero General Public License v3 in order to prevent fore.st being used in proprietary software.

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "fore.st",
"version": "1.1.2",
"version": "1.1.3",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "fore.st",
"version": "1.1.2",
"version": "1.1.3",
"hasInstallScript": true,
"dependencies": {
"@calzoneman/jsli": "^2.0.1",

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "fore.st",
"version": "1.1.2",
"version": "1.1.3",
"description": "fore.st: A fork of cytube tailored for the TTN Community",
"main": "index.js",
"directories": {

7
patchnotes.md
View file

@ -1,2 +1,5 @@
dev goals for 1.1.2 pineapple Express += 2:
--Fix broken video providers (dailymotion/youtube)
dev goals for 1.1.3 pineapple Express += 3:
-- Add bulk-queueing from Internet Archive
- automagically pull most web-compatible version of all videos within a specific upload to IA
- allow queueing admin to filter directory by video length in minutes
- allow for bulk naming

12
src/channel/autobump.js
View file

@ -107,11 +107,13 @@ function loadLists(cb, callp){
bumplists = new Map();//create new map to load lists into, this clears the variable as well as lets the channel know whether or not they have been loaded yet.
item.forEach(function(list, i){
if(list != configFolder.slice(bumpFolder.length, configFolder.length - 1) && list.slice(list.length - 5) === ".bump"){
loadList("bumps/" + list);
}
});
if(item != null){
item.forEach(function(list, i){
if(list != configFolder.slice(bumpFolder.length, configFolder.length - 1) && list.slice(list.length - 5) === ".bump"){
loadList("bumps/" + list);
}
});
}
if(cb != null){//lil' nasty but it calls loadconfig after loading lists :P
if(callp != null){

50
src/channel/playlist.js
View file

@ -67,6 +67,7 @@ const TYPE_QUEUE = {
pos: "string",
title: "string,boolean,optional",
duration: "number,optional",
minDuration: "number,optional",
temp: "boolean,optional",
subtitle: "string"
};
@ -445,7 +446,7 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
* Specifying a custom title is currently only allowed for custom media
* and raw files
*/
if (typeof data.title !== "string" || (data.type !== "cu" && data.type !== "fi")) {
if (typeof data.title !== "string" || (data.type !== "cu" && data.type !== "fi" && data.type !== "ia")) {
data.title = false;
}
@ -466,9 +467,9 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
}
/* Certain media types require special permission to add */
if (data.type === "yp" && !perms.canAddList(user)) {
if ((data.type === "yp" || data.type === "ia") && !perms.canAddList(user)) {
user.socket.emit("queueFail", {
msg: "You don't have permission to add playlists",
msg: "You don't have permission to bulk queue",
link: link,
id: id
});
@ -504,6 +505,11 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
duration = !isNaN(data.duration) ? data.duration : undefined;
}
var minDuration = undefined;
if (typeof data.minDuration === "number") {
minDuration = !isNaN(data.minDuration) ? data.minDuration : 0;
}
var limit = {
burst: 3,
sustained: 1
@ -537,17 +543,20 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
type: data.type,
pos: data.pos,
title: data.title,
subtitle: data.subtitle,
subtitle: data.subtitle,
link: link,
temp: temp,
shouldAddToLibrary: true,//for now chan library will act as a history
queueby: queueby,
duration: duration,
minDuration: minDuration,
maxlength: maxlength
};
if (data.type === "yp") {
this.queueYouTubePlaylist(user, data);
} else if (data.type === "ia") {
this.queueBulkIA(user, data);
} else {
this.queueStandard(user, data);
}
@ -576,7 +585,38 @@ PlaylistModule.prototype.queueStandard = function (user, data) {
lock.release();
self.channel.refCounter.unref("PlaylistModule::queueStandard");
});
}, data.minDuration);
});
};
PlaylistModule.prototype.queueBulkIA = function (user, data) {
var error = function (what) {
user.socket.emit("queueFail", {
msg: what,
link: data.link,
id: data.id
});
};
const self = this;
this.channel.refCounter.ref("PlaylistModule::queueBulkIA");
this.semaphore.queue(function (lock) {
InfoGetter.getMedia(data.id, data.type, function (err, vids) {
if (err) {
error(XSS.sanitizeText(String(err)));
self.channel.refCounter.unref("PlaylistModule::queueBulkIA");
return lock.release();
}
vids.forEach(function(media){
self._addItem(media, data, user, null, data.minDuration);
});
self.channel.refCounter.unref("PlaylistModule::queueBulkIA");
lock.release();
}, data.minDuration);
});
};
@ -1117,7 +1157,7 @@ PlaylistModule.prototype._addItem = function (media, data, user, cb, abump) {
queueby: data.queueby
});
if (data.title && (media.type === "cu" || media.type === "fi")) {
if (data.title && (media.type === "cu" || media.type === "fi" || media.type === "ia")) {
media.setTitle(data.title);
}

92
src/get-info.js
View file

@ -231,7 +231,6 @@ var Getters = {
});
console.log(videos);
callback(null, videos);
}catch(err){
callback(err.message || err, null);
@ -660,18 +659,99 @@ var Getters = {
callback,
"As of July 2020, Mixer is no longer in service."
);
},
/*Internet Archive Bulk Grabber*/
ia: function(id, minDuration, callback){
try{
//Get metadata on the directory
var options = {
host: "archive.org",
port: 443,
path: "/metadata/" + id,
method: "GET",
timeout: 1000
};
//pull the URL
urlRetrieve(https, options, function (status, data) {
//if we fucked up
if(status !== 200) {
return callback("Archive.org HTTPS error code: " + status, null);
}
//Parse the dump
var dump = JSON.parse(data);
//if we have files
if(dump.files != null){
var vids = new Array;
var derivative = new Array;
var media = new Array;
//sift through files to find .mp4's
dump.files.forEach(function(file){
//Skip out on videos that dont meet the requested minimum duration
if(file.length >= minDuration){
//if its a standard .mp4 (either MPEG, or h.264)
if(file.format == "h.264" || file.format == "MPEG4"){
//add the file to the video array
vids.push(file);
//if it's been transcoded by archive.org to ensure web-compatibility
}else if(file.format == "h.264 IA"){
//add the file to the derivative array
derivative.push(file);
}
}
});
//chose derivatives over originals to save on bandwith and ensure web-compatibility, even if they don't always look as nice :P
derivative.forEach(function(file){
//sift through standard files to find matching originals
for(i = 0; i < vids.length; i++){
//if we have a match
if(vids[i].name == file.original){
//replace the file with the correct derivative
vids[i] = file;
}
}
});
//Lets try this just using info from IA's api. It would take up way less fucking time, even if we can't fill in everything...
vids = vids.map(function (file) {
return new Media(`https://${dump.d1}${dump.dir}/${file.name}`, dump.metadata.title, file.length, "fi", {codec: "mov/h264"});
});
process.nextTick(callback, false, vids);
//if we fucked up some other way
}else if(dump.error != null){
return callback(`Archive.org error: ${dump.error}`);
}else{
return callback("Unkown metadata error from archive.org!");
}
});
} catch (err) {
callback(err.message);
}
}
};
module.exports = {
Getters: Getters,
getMedia: function (id, type, callback) {
if(type in this.Getters) {
LOGGER.info("Looking up %s:%s", type, id);
lookupCounter.labels(type).inc(1, new Date());
getMedia: function (id, type, callback, minDuration) {
if (type in this.Getters) {
LOGGER.info("Looking up %s:%s", type, id);
lookupCounter.labels(type).inc(1, new Date());
if(type == "ia"){
this.Getters.ia(id, minDuration, callback);
}else{
this.Getters[type](id, callback);
}
} else {
callback("Unknown media type '" + type + "'", null);
callback("Unknown media type '" + type + "'", null);
}
},
getRawCopy: async function (id, cb){

2
templates/about.pug
View file

@ -56,4 +56,4 @@ block content
| No spamming submit channel or chat
p.
Comments? Questions? Feature requests? DMCA Notices? <a href="mailto:ourforest@420blaze.it">Email us!</a>
h4 fore.st version: Pineapple Express += 2 (v1.1.2)
h4 fore.st version: Pineapple Express += 3 (v1.1.3)

2
www/js/data.js
View file

@ -37,7 +37,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
var CL_VERSION = "1.1.2";
var CL_VERSION = "1.1.3";
var GS_VERSION = 1.7; // Google Drive Userscript
var CLIENT = {

46
www/js/ui.js
View file

@ -686,6 +686,7 @@ function queue(pos, src) {
var duration = undefined;
var title = undefined;
var subtitle = "";
var minDuration = 0;
if (data.type === "fi") {
if (data.id.match(/^http:/)) {
Callbacks.queueFail({
@ -716,7 +717,12 @@ function queue(pos, src) {
// Raw files allow title overrides since the ffprobe tag data
// is not always correct.
title = $("#addfromurl-title-val").val();
subtitle = $("#addfromurl-subtitle-val").val();
subtitle = "";
}else{
title = $("#addfromurl-title-val").val();
minDuration = $("#addfromurl-duration-val").val();
//Convert minDuration to a number that represents seconds instead of a string which represents minutes
minDuration = (minDuration == "" ? 0 : (Number.parseInt(minDuration) * 60));
}
if (data.id == null || data.type == null) {
@ -731,6 +737,7 @@ function queue(pos, src) {
pos: pos,
duration: duration,
title: title,
minDuration: minDuration,
temp: addTemp,
link: link,
subtitle: subtitle
@ -775,10 +782,12 @@ $("#mediaurl").keyup(function(ev) {
queue("end", "url");
} else {
var editTitle = false;
var editDur = false;
try {
if (parseMediaLink($("#mediaurl").val()).type === "fi") {
editTitle = true;
}
editTitle = (parseMediaLink($("#mediaurl").val()).type === "fi" || parseMediaLink($("#mediaurl").val()).type === "ia");
editDur = (parseMediaLink($("#mediaurl").val()).type === "ia");
} catch (error) {
}
@ -795,8 +804,8 @@ $("#mediaurl").keyup(function(ev) {
$("<input/>").addClass("form-control")//create title field
.attr("type", "text")//the attributes
.attr("id", "addfromurl-title-val")
.attr("placeholder", "Alternate Title")
.attr("style", "display: none; width: 100%;")
.attr("placeholder", "Alternate Title")
.attr("style", "display: none; width: 100%;")
.keydown(function (ev) {
if (ev.keyCode === 13) {
queue("end", "url");
@ -804,17 +813,20 @@ $("#mediaurl").keyup(function(ev) {
})
.appendTo($("#addfromurl-title")).show("blind");//append and show
$("<input/>").addClass("form-control")//create title field
.attr("type", "text")//the attributes
.attr("id", "addfromurl-subtitle-val")
.attr("placeholder", "Alternate Subtitle Track")
.attr("style", "display: none; width: 100%;")
.keydown(function (ev) {
if (ev.keyCode === 13) {
queue("end", "url");
}
})
.appendTo($("#addfromurl-title")).show("blind");//append and show
if(editDur){
$("<input/>").addClass("form-control")//create title field
.attr("type", "text")//the attributes
.attr("id", "addfromurl-duration-val")
.attr("placeholder", "Minimum Duration Filter")
.attr("style", "display: none; width: 100%;")
.keydown(function (ev) {
if (ev.keyCode === 13) {
queue("end", "url");
}
})
.appendTo($("#addfromurl-title")).show("blind");//append and show
}
}
} else {
$("#addfromurl-title").hide("blind");

12
www/js/util.js
View file

@ -1615,6 +1615,13 @@ function parseMediaLink(url) {
};
}
if((m = url.match(/archive\.org\/(?:details|download)\/([a-zA-Z0-9_-]+)(?!.|\/)/))){
return{
id: m[1],
type: "ia"
}
}
/* Shorthand URIs */
// So we still trim DailyMotion URLs
if((m = url.match(/^dm:([^\?&#_]+)/))) {
@ -3445,7 +3452,7 @@ function startQueueSpinner(data) {
}
var id = data.id;
if (data.type === "yp") {
if (data.type === "yp" || data.type === "ia") {
id = "$any";
}
@ -3493,7 +3500,8 @@ function stopQueueSpinner(data) {
$("#queueprogress").data("queue-id") === data.id);
shouldRemove = shouldRemove || data === null;
shouldRemove = shouldRemove || $("#queueprogress").data("queue-id") === "$any";
if (shouldRemove) {
//This is a gross way to fix the issue with IA but it works, and it's not like cytube was a pretty codebase anywho...
if (shouldRemove) {
$("#queueprogress").remove();
}
}