diff --git a/README.md b/README.md
index 5e76ab4e..a80558bb 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-fore.st - Pineapple Express += 2 (v1.1.2)
+fore.st - Pineapple Express += 3 (v1.1.3)
======
fore.st is the server software for ourfore.st, a community based chat & synced video embedding site tailored to service
@@ -24,8 +24,8 @@ You can reach out by bugging rainbownapkin on the ttn discord or ourfore.st, you
- Thanks to calzoneman for making [cytube](https://github.com/calzoneman/sync), that saved our asses.
- Thanks to the core TTN community and everyone else who's ever used it, I was only there for the last handful of years but it was an absolute fuckin' ride. You guys are the best, it isn't TTN but I hope this at least help fills the gap.
-## Pineapple Express += 2 1.1.2 Release Notes
- - Fixed busted video providers with YT-DLP integration (youtube/dailymotion)
+## Pineapple Express += 3 1.1.3 Release Notes
+ - Add bulk-queueing from Internet Archive
## License
Original fore.st code is provided under the Affero General Public License v3 in order to prevent fore.st being used in proprietary software.
diff --git a/package-lock.json b/package-lock.json
index 8848b04d..d92f05c6 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "fore.st",
- "version": "1.1.2",
+ "version": "1.1.3",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "fore.st",
- "version": "1.1.2",
+ "version": "1.1.3",
"hasInstallScript": true,
"dependencies": {
"@calzoneman/jsli": "^2.0.1",
diff --git a/package.json b/package.json
index 7557966c..3a58e6b5 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "fore.st",
- "version": "1.1.2",
+ "version": "1.1.3",
"description": "fore.st: A fork of cytube tailored for the TTN Community",
"main": "index.js",
"directories": {
diff --git a/patchnotes.md b/patchnotes.md
index 1beb8470..a08db3e0 100644
--- a/patchnotes.md
+++ b/patchnotes.md
@@ -1,2 +1,5 @@
-dev goals for 1.1.2 pineapple Express += 2:
- --Fix broken video providers (dailymotion/youtube)
\ No newline at end of file
+dev goals for 1.1.3 pineapple Express += 3:
+ -- Add bulk-queueing from Internet Archive
+ - automagically pull most web-compatible version of all videos within a specific upload to IA
+ - allow queueing admin to filter directory by video length in minutes
+ - allow for bulk naming
\ No newline at end of file
diff --git a/src/channel/autobump.js b/src/channel/autobump.js
index 2be7a7f3..882a3df2 100644
--- a/src/channel/autobump.js
+++ b/src/channel/autobump.js
@@ -107,11 +107,13 @@ function loadLists(cb, callp){
bumplists = new Map();//create new map to load lists into, this clears the variable as well as lets the channel know whether or not they have been loaded yet.
- item.forEach(function(list, i){
- if(list != configFolder.slice(bumpFolder.length, configFolder.length - 1) && list.slice(list.length - 5) === ".bump"){
- loadList("bumps/" + list);
- }
- });
+ if(item != null){
+ item.forEach(function(list, i){
+ if(list != configFolder.slice(bumpFolder.length, configFolder.length - 1) && list.slice(list.length - 5) === ".bump"){
+ loadList("bumps/" + list);
+ }
+ });
+ }
if(cb != null){//lil' nasty but it calls loadconfig after loading lists :P
if(callp != null){
diff --git a/src/channel/playlist.js b/src/channel/playlist.js
index cd1ec86c..d2fe1ad3 100644
--- a/src/channel/playlist.js
+++ b/src/channel/playlist.js
@@ -67,6 +67,7 @@ const TYPE_QUEUE = {
pos: "string",
title: "string,boolean,optional",
duration: "number,optional",
+ minDuration: "number,optional",
temp: "boolean,optional",
subtitle: "string"
};
@@ -445,7 +446,7 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
* Specifying a custom title is currently only allowed for custom media
* and raw files
*/
- if (typeof data.title !== "string" || (data.type !== "cu" && data.type !== "fi")) {
+ if (typeof data.title !== "string" || (data.type !== "cu" && data.type !== "fi" && data.type !== "ia")) {
data.title = false;
}
@@ -466,9 +467,9 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
}
/* Certain media types require special permission to add */
- if (data.type === "yp" && !perms.canAddList(user)) {
+ if ((data.type === "yp" || data.type === "ia") && !perms.canAddList(user)) {
user.socket.emit("queueFail", {
- msg: "You don't have permission to add playlists",
+ msg: "You don't have permission to bulk queue",
link: link,
id: id
});
@@ -504,6 +505,11 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
duration = !isNaN(data.duration) ? data.duration : undefined;
}
+ var minDuration = undefined;
+ if (typeof data.minDuration === "number") {
+ minDuration = !isNaN(data.minDuration) ? data.minDuration : 0;
+ }
+
var limit = {
burst: 3,
sustained: 1
@@ -537,17 +543,20 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
type: data.type,
pos: data.pos,
title: data.title,
- subtitle: data.subtitle,
+ subtitle: data.subtitle,
link: link,
temp: temp,
shouldAddToLibrary: true,//for now chan library will act as a history
queueby: queueby,
duration: duration,
+ minDuration: minDuration,
maxlength: maxlength
};
if (data.type === "yp") {
this.queueYouTubePlaylist(user, data);
+ } else if (data.type === "ia") {
+ this.queueBulkIA(user, data);
} else {
this.queueStandard(user, data);
}
@@ -576,7 +585,38 @@ PlaylistModule.prototype.queueStandard = function (user, data) {
lock.release();
self.channel.refCounter.unref("PlaylistModule::queueStandard");
});
+ }, data.minDuration);
+ });
+};
+
+PlaylistModule.prototype.queueBulkIA = function (user, data) {
+ var error = function (what) {
+ user.socket.emit("queueFail", {
+ msg: what,
+ link: data.link,
+ id: data.id
});
+ };
+
+ const self = this;
+ this.channel.refCounter.ref("PlaylistModule::queueBulkIA");
+ this.semaphore.queue(function (lock) {
+ InfoGetter.getMedia(data.id, data.type, function (err, vids) {
+ if (err) {
+ error(XSS.sanitizeText(String(err)));
+ self.channel.refCounter.unref("PlaylistModule::queueBulkIA");
+ return lock.release();
+ }
+
+
+ vids.forEach(function(media){
+ self._addItem(media, data, user, null, data.minDuration);
+ });
+
+
+ self.channel.refCounter.unref("PlaylistModule::queueBulkIA");
+ lock.release();
+ }, data.minDuration);
});
};
@@ -1117,7 +1157,7 @@ PlaylistModule.prototype._addItem = function (media, data, user, cb, abump) {
queueby: data.queueby
});
- if (data.title && (media.type === "cu" || media.type === "fi")) {
+ if (data.title && (media.type === "cu" || media.type === "fi" || media.type === "ia")) {
media.setTitle(data.title);
}
diff --git a/src/get-info.js b/src/get-info.js
index 2ce57153..37d750b5 100644
--- a/src/get-info.js
+++ b/src/get-info.js
@@ -231,7 +231,6 @@ var Getters = {
});
- console.log(videos);
callback(null, videos);
}catch(err){
callback(err.message || err, null);
@@ -660,18 +659,99 @@ var Getters = {
callback,
"As of July 2020, Mixer is no longer in service."
);
+ },
+
+ /*Internet Archive Bulk Grabber*/
+ ia: function(id, minDuration, callback){
+ try{
+ //Get metadata on the directory
+ var options = {
+ host: "archive.org",
+ port: 443,
+ path: "/metadata/" + id,
+ method: "GET",
+ timeout: 1000
+ };
+
+ //pull the URL
+ urlRetrieve(https, options, function (status, data) {
+ //if we fucked up
+ if(status !== 200) {
+ return callback("Archive.org HTTPS error code: " + status, null);
+ }
+
+ //Parse the dump
+ var dump = JSON.parse(data);
+
+
+ //if we have files
+ if(dump.files != null){
+ var vids = new Array;
+ var derivative = new Array;
+ var media = new Array;
+
+ //sift through files to find .mp4's
+ dump.files.forEach(function(file){
+ //Skip out on videos that dont meet the requested minimum duration
+ if(file.length >= minDuration){
+ //if its a standard .mp4 (either MPEG, or h.264)
+ if(file.format == "h.264" || file.format == "MPEG4"){
+ //add the file to the video array
+ vids.push(file);
+ //if it's been transcoded by archive.org to ensure web-compatibility
+ }else if(file.format == "h.264 IA"){
+ //add the file to the derivative array
+ derivative.push(file);
+ }
+ }
+ });
+
+ //chose derivatives over originals to save on bandwith and ensure web-compatibility, even if they don't always look as nice :P
+ derivative.forEach(function(file){
+ //sift through standard files to find matching originals
+ for(i = 0; i < vids.length; i++){
+ //if we have a match
+ if(vids[i].name == file.original){
+ //replace the file with the correct derivative
+ vids[i] = file;
+ }
+ }
+ });
+
+ //Lets try this just using info from IA's api. It would take up way less fucking time, even if we can't fill in everything...
+ vids = vids.map(function (file) {
+ return new Media(`https://${dump.d1}${dump.dir}/${file.name}`, dump.metadata.title, file.length, "fi", {codec: "mov/h264"});
+ });
+
+ process.nextTick(callback, false, vids);
+
+ //if we fucked up some other way
+ }else if(dump.error != null){
+ return callback(`Archive.org error: ${dump.error}`);
+ }else{
+ return callback("Unkown metadata error from archive.org!");
+ }
+ });
+
+ } catch (err) {
+ callback(err.message);
+ }
}
};
module.exports = {
Getters: Getters,
- getMedia: function (id, type, callback) {
- if(type in this.Getters) {
- LOGGER.info("Looking up %s:%s", type, id);
- lookupCounter.labels(type).inc(1, new Date());
+ getMedia: function (id, type, callback, minDuration) {
+ if (type in this.Getters) {
+ LOGGER.info("Looking up %s:%s", type, id);
+ lookupCounter.labels(type).inc(1, new Date());
+ if(type == "ia"){
+ this.Getters.ia(id, minDuration, callback);
+ }else{
this.Getters[type](id, callback);
+ }
} else {
- callback("Unknown media type '" + type + "'", null);
+ callback("Unknown media type '" + type + "'", null);
}
},
getRawCopy: async function (id, cb){
diff --git a/templates/about.pug b/templates/about.pug
index 6d342991..c5e7ec9b 100644
--- a/templates/about.pug
+++ b/templates/about.pug
@@ -56,4 +56,4 @@ block content
| No spamming submit channel or chat
p.
Comments? Questions? Feature requests? DMCA Notices? Email us!
- h4 fore.st version: Pineapple Express += 2 (v1.1.2)
+ h4 fore.st version: Pineapple Express += 3 (v1.1.3)
diff --git a/www/js/data.js b/www/js/data.js
index ca85bb7b..9dea5cd9 100644
--- a/www/js/data.js
+++ b/www/js/data.js
@@ -37,7 +37,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
-var CL_VERSION = "1.1.2";
+var CL_VERSION = "1.1.3";
var GS_VERSION = 1.7; // Google Drive Userscript
var CLIENT = {
diff --git a/www/js/ui.js b/www/js/ui.js
index 5ed78f3b..326a74d3 100644
--- a/www/js/ui.js
+++ b/www/js/ui.js
@@ -686,6 +686,7 @@ function queue(pos, src) {
var duration = undefined;
var title = undefined;
var subtitle = "";
+ var minDuration = 0;
if (data.type === "fi") {
if (data.id.match(/^http:/)) {
Callbacks.queueFail({
@@ -716,7 +717,12 @@ function queue(pos, src) {
// Raw files allow title overrides since the ffprobe tag data
// is not always correct.
title = $("#addfromurl-title-val").val();
- subtitle = $("#addfromurl-subtitle-val").val();
+ subtitle = "";
+ }else{
+ title = $("#addfromurl-title-val").val();
+ minDuration = $("#addfromurl-duration-val").val();
+ //Convert minDuration to a number that represents seconds instead of a string which represents minutes
+ minDuration = (minDuration == "" ? 0 : (Number.parseInt(minDuration) * 60));
}
if (data.id == null || data.type == null) {
@@ -731,6 +737,7 @@ function queue(pos, src) {
pos: pos,
duration: duration,
title: title,
+ minDuration: minDuration,
temp: addTemp,
link: link,
subtitle: subtitle
@@ -775,10 +782,12 @@ $("#mediaurl").keyup(function(ev) {
queue("end", "url");
} else {
var editTitle = false;
+ var editDur = false;
try {
- if (parseMediaLink($("#mediaurl").val()).type === "fi") {
- editTitle = true;
- }
+ editTitle = (parseMediaLink($("#mediaurl").val()).type === "fi" || parseMediaLink($("#mediaurl").val()).type === "ia");
+
+
+ editDur = (parseMediaLink($("#mediaurl").val()).type === "ia");
} catch (error) {
}
@@ -795,8 +804,8 @@ $("#mediaurl").keyup(function(ev) {
$("").addClass("form-control")//create title field
.attr("type", "text")//the attributes
.attr("id", "addfromurl-title-val")
- .attr("placeholder", "Alternate Title")
- .attr("style", "display: none; width: 100%;")
+ .attr("placeholder", "Alternate Title")
+ .attr("style", "display: none; width: 100%;")
.keydown(function (ev) {
if (ev.keyCode === 13) {
queue("end", "url");
@@ -804,17 +813,20 @@ $("#mediaurl").keyup(function(ev) {
})
.appendTo($("#addfromurl-title")).show("blind");//append and show
- $("").addClass("form-control")//create title field
- .attr("type", "text")//the attributes
- .attr("id", "addfromurl-subtitle-val")
- .attr("placeholder", "Alternate Subtitle Track")
- .attr("style", "display: none; width: 100%;")
- .keydown(function (ev) {
- if (ev.keyCode === 13) {
- queue("end", "url");
- }
- })
- .appendTo($("#addfromurl-title")).show("blind");//append and show
+ if(editDur){
+ $("").addClass("form-control")//create title field
+ .attr("type", "text")//the attributes
+ .attr("id", "addfromurl-duration-val")
+ .attr("placeholder", "Minimum Duration Filter")
+ .attr("style", "display: none; width: 100%;")
+ .keydown(function (ev) {
+ if (ev.keyCode === 13) {
+ queue("end", "url");
+ }
+ })
+ .appendTo($("#addfromurl-title")).show("blind");//append and show
+ }
+
}
} else {
$("#addfromurl-title").hide("blind");
diff --git a/www/js/util.js b/www/js/util.js
index 9729bd4d..1a165c5c 100644
--- a/www/js/util.js
+++ b/www/js/util.js
@@ -1615,6 +1615,13 @@ function parseMediaLink(url) {
};
}
+ if((m = url.match(/archive\.org\/(?:details|download)\/([a-zA-Z0-9_-]+)(?!.|\/)/))){
+ return{
+ id: m[1],
+ type: "ia"
+ }
+ }
+
/* Shorthand URIs */
// So we still trim DailyMotion URLs
if((m = url.match(/^dm:([^\?&#_]+)/))) {
@@ -3445,7 +3452,7 @@ function startQueueSpinner(data) {
}
var id = data.id;
- if (data.type === "yp") {
+ if (data.type === "yp" || data.type === "ia") {
id = "$any";
}
@@ -3493,7 +3500,8 @@ function stopQueueSpinner(data) {
$("#queueprogress").data("queue-id") === data.id);
shouldRemove = shouldRemove || data === null;
shouldRemove = shouldRemove || $("#queueprogress").data("queue-id") === "$any";
- if (shouldRemove) {
+ //This is a gross way to fix the issue with IA but it works, and it's not like cytube was a pretty codebase anywho...
+ if (shouldRemove) {
$("#queueprogress").remove();
}
}