Added bulk-queueing from Internet Archive.

README.md

@@ -1,4 +1,4 @@
-fore.st - Pineapple Express += 2 (v1.1.2)
+fore.st - Pineapple Express += 3 (v1.1.3)
 ======
 
 fore.st is the server software for ourfore.st, a community based chat & synced video embedding site tailored to service
@@ -24,8 +24,8 @@ You can reach out by bugging rainbownapkin on the ttn discord or ourfore.st, you
   - Thanks to calzoneman for making [cytube](https://github.com/calzoneman/sync), that saved our asses.
   - Thanks to the core TTN community and everyone else who's ever used it, I was only there for the last handful of years but it was an absolute fuckin' ride. You guys are the best, it isn't TTN but I hope this at least help fills the gap.
 
-## Pineapple Express += 2 1.1.2 Release Notes
+## Pineapple Express += 3 1.1.3 Release Notes
- - Fixed busted video providers with YT-DLP integration (youtube/dailymotion)
+ - Add bulk-queueing from Internet Archive
 
 ## License
 Original fore.st code is provided under the Affero General Public License v3 in order to prevent fore.st being used in proprietary software.

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "fore.st",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "fore.st",
-      "version": "1.1.2",
+      "version": "1.1.3",
       "hasInstallScript": true,
       "dependencies": {
         "@calzoneman/jsli": "^2.0.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fore.st",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "fore.st: A fork of cytube tailored for the TTN Community",
   "main": "index.js",
   "directories": {

patchnotes.md

@@ -1,2 +1,5 @@
-dev goals for 1.1.2 pineapple Express += 2:
+dev goals for 1.1.3 pineapple Express += 3:
- --Fix broken video providers (dailymotion/youtube)
+ -- Add bulk-queueing from Internet Archive
+  - automagically pull most web-compatible version of all videos within a specific upload to IA
+  - allow queueing admin to filter directory by video length in minutes
+  - allow for bulk naming

src/channel/autobump.js

@@ -107,11 +107,13 @@ function loadLists(cb, callp){
 
 		bumplists = new Map();//create new map to load lists into, this clears the variable as well as lets the channel know whether or not they have been loaded yet.
 
+		if(item != null){
 			item.forEach(function(list, i){
 				if(list != configFolder.slice(bumpFolder.length, configFolder.length - 1) && list.slice(list.length - 5) === ".bump"){
 					loadList("bumps/" + list);
 				}
 			});
+		}
 		
 		if(cb != null){//lil' nasty but it calls loadconfig after loading lists :P
 			if(callp != null){

src/channel/playlist.js

@@ -67,6 +67,7 @@ const TYPE_QUEUE = {
     pos: "string",
     title: "string,boolean,optional",
     duration: "number,optional",
+    minDuration: "number,optional",
     temp: "boolean,optional",
     subtitle: "string"
 };
@@ -445,7 +446,7 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
      * Specifying a custom title is currently only allowed for custom media
      * and raw files
      */
-    if (typeof data.title !== "string" || (data.type !== "cu" && data.type !== "fi")) {
+    if (typeof data.title !== "string" || (data.type !== "cu" && data.type !== "fi" && data.type !== "ia")) {
         data.title = false;
     }
     
@@ -466,9 +467,9 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
     }
 
     /* Certain media types require special permission to add */
-    if (data.type === "yp" && !perms.canAddList(user)) {
+    if ((data.type === "yp" || data.type === "ia") && !perms.canAddList(user)) {
         user.socket.emit("queueFail", {
-            msg: "You don't have permission to add playlists",
+            msg: "You don't have permission to bulk queue",
             link: link,
             id: id
         });
@@ -504,6 +505,11 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
         duration = !isNaN(data.duration) ? data.duration : undefined;
     }
 
+    var minDuration = undefined;
+    if (typeof data.minDuration === "number") {
+        minDuration = !isNaN(data.minDuration) ? data.minDuration : 0;
+    }
+
     var limit = {
         burst: 3,
         sustained: 1
@@ -543,11 +549,14 @@ PlaylistModule.prototype.handleQueue = function (user, data) {
         shouldAddToLibrary: true,//for now chan library will act as a history
         queueby: queueby,
         duration: duration,
+        minDuration: minDuration,
         maxlength: maxlength
     };
 
     if (data.type === "yp") {
         this.queueYouTubePlaylist(user, data);
+    } else if (data.type === "ia") {
+        this.queueBulkIA(user, data);
     } else {
         this.queueStandard(user, data);
     }
@@ -576,7 +585,38 @@ PlaylistModule.prototype.queueStandard = function (user, data) {
                 lock.release();
                 self.channel.refCounter.unref("PlaylistModule::queueStandard");
             });
+        }, data.minDuration);
     });
+};
+
+PlaylistModule.prototype.queueBulkIA = function (user, data) {
+    var error = function (what) {
+        user.socket.emit("queueFail", {
+            msg: what,
+            link: data.link,
+            id: data.id
+        });
+    };
+
+    const self = this;
+    this.channel.refCounter.ref("PlaylistModule::queueBulkIA");
+    this.semaphore.queue(function (lock) {
+        InfoGetter.getMedia(data.id, data.type, function (err, vids) {
+            if (err) {
+                error(XSS.sanitizeText(String(err)));
+                self.channel.refCounter.unref("PlaylistModule::queueBulkIA");
+                return lock.release();
+            }
+
+
+            vids.forEach(function(media){
+                self._addItem(media, data, user, null, data.minDuration);
+            });
+
+
+            self.channel.refCounter.unref("PlaylistModule::queueBulkIA");
+            lock.release();
+        }, data.minDuration);
     });
 };
 
@@ -1117,7 +1157,7 @@ PlaylistModule.prototype._addItem = function (media, data, user, cb, abump) {
         queueby: data.queueby
     });
 
-    if (data.title && (media.type === "cu" || media.type === "fi")) {
+    if (data.title && (media.type === "cu" || media.type === "fi" || media.type === "ia")) {
         media.setTitle(data.title);
     }
 

src/get-info.js

@@ -231,7 +231,6 @@ var Getters = {
             });
 
 
-            console.log(videos);
             callback(null, videos);
         }catch(err){
             callback(err.message || err, null);
@@ -660,16 +659,97 @@ var Getters = {
             callback,
             "As of July 2020, Mixer is no longer in service."
         );
+    },
+
+    /*Internet Archive Bulk Grabber*/
+    ia: function(id, minDuration, callback){
+        try{
+        //Get metadata on the directory
+        var options = {
+            host: "archive.org",
+            port: 443,
+            path: "/metadata/" + id,
+            method: "GET",
+            timeout: 1000
+        };
+
+        //pull the URL
+        urlRetrieve(https, options, function (status, data) {
+            //if we fucked up
+            if(status !== 200) {
+                return callback("Archive.org HTTPS error code: " + status, null);
+            }
+            
+            //Parse the dump
+            var dump = JSON.parse(data);
+
+
+            //if we have files
+            if(dump.files != null){
+                var vids = new Array;
+                var derivative = new Array;
+                var media = new Array;
+
+                //sift through files to find .mp4's
+                dump.files.forEach(function(file){
+                    //Skip out on videos that dont meet the requested minimum duration
+                    if(file.length >= minDuration){
+                        //if its a standard .mp4 (either MPEG, or h.264)
+                        if(file.format == "h.264" || file.format == "MPEG4"){
+                            //add the file to the video array
+                            vids.push(file);
+                        //if it's been transcoded by archive.org to ensure web-compatibility
+                        }else if(file.format == "h.264 IA"){
+                            //add the file to the derivative array
+                            derivative.push(file);
+                        }
+                    }
+                });
+
+                //chose derivatives over originals to save on bandwith and ensure web-compatibility, even if they don't always look as nice :P
+                derivative.forEach(function(file){
+                    //sift through standard files to find matching originals
+                    for(i = 0; i < vids.length; i++){
+                        //if we have a match
+                        if(vids[i].name == file.original){
+                            //replace the file with the correct derivative
+                            vids[i] = file;
+                        }
+                    }
+                });
+
+                //Lets try this just using info from IA's api. It would take up way less fucking time, even if we can't fill in everything...
+                vids = vids.map(function (file) {
+                    return new Media(`https://${dump.d1}${dump.dir}/${file.name}`, dump.metadata.title, file.length, "fi", {codec: "mov/h264"});
+                });
+
+                process.nextTick(callback, false, vids);
+
+            //if we fucked up some other way
+            }else if(dump.error != null){
+                return callback(`Archive.org error: ${dump.error}`);
+            }else{
+                return callback("Unkown metadata error from archive.org!");
+            }
+        });
+
+        } catch (err) {
+            callback(err.message);
+        }
     }
 };
 
 module.exports = {
     Getters: Getters,
-    getMedia: function (id, type, callback) {
+    getMedia: function (id, type, callback, minDuration) {
         if (type in this.Getters) {
         LOGGER.info("Looking up %s:%s", type, id);
         lookupCounter.labels(type).inc(1, new Date());
+        if(type == "ia"){
+            this.Getters.ia(id, minDuration, callback);
+        }else{
             this.Getters[type](id, callback);
+        }
         } else {
         callback("Unknown media type '" + type + "'", null);
         }

templates/about.pug

@@ -56,4 +56,4 @@ block content
           | No spamming submit channel or chat 
       p.
         Comments? Questions? Feature requests? DMCA Notices? <a href="mailto:ourforest@420blaze.it">Email us!</a>
-      h4 fore.st version: Pineapple Express += 2 (v1.1.2)
+      h4 fore.st version: Pineapple Express += 3 (v1.1.3)

www/js/data.js

@@ -37,7 +37,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
-var CL_VERSION = "1.1.2";
+var CL_VERSION = "1.1.3";
 var GS_VERSION = 1.7; // Google Drive Userscript
 
 var CLIENT = {

www/js/ui.js

@@ -686,6 +686,7 @@ function queue(pos, src) {
             var duration = undefined;
             var title = undefined;
             var subtitle = "";
+            var minDuration = 0;
             if (data.type === "fi") {
                 if (data.id.match(/^http:/)) {
                     Callbacks.queueFail({
@@ -716,7 +717,12 @@ function queue(pos, src) {
                 // Raw files allow title overrides since the ffprobe tag data
                 // is not always correct.
                 title = $("#addfromurl-title-val").val();
-                subtitle = $("#addfromurl-subtitle-val").val();
+                subtitle = "";
+            }else{
+                title = $("#addfromurl-title-val").val();
+                minDuration = $("#addfromurl-duration-val").val();
+                //Convert minDuration to a number that represents seconds instead of a string which represents minutes
+                minDuration = (minDuration == "" ? 0 : (Number.parseInt(minDuration) * 60));
             }
 
             if (data.id == null || data.type == null) {
@@ -731,6 +737,7 @@ function queue(pos, src) {
                     pos: pos,
                     duration: duration,
                     title: title,
+                    minDuration: minDuration,
                     temp: addTemp,
                     link: link,
 		    subtitle: subtitle
@@ -775,10 +782,12 @@ $("#mediaurl").keyup(function(ev) {
         queue("end", "url");
     } else {
         var editTitle = false;
+        var editDur = false;
         try {
-            if (parseMediaLink($("#mediaurl").val()).type === "fi") {
+            editTitle = (parseMediaLink($("#mediaurl").val()).type === "fi" || parseMediaLink($("#mediaurl").val()).type === "ia");
-                editTitle = true;
+
-            }
+
+            editDur =  (parseMediaLink($("#mediaurl").val()).type === "ia");
         } catch (error) {
         }
 
@@ -804,10 +813,11 @@ $("#mediaurl").keyup(function(ev) {
                     })
                     .appendTo($("#addfromurl-title")).show("blind");//append and show
 
+                if(editDur){
                     $("<input/>").addClass("form-control")//create title field
                         .attr("type", "text")//the attributes
-                    .attr("id", "addfromurl-subtitle-val")
+                        .attr("id", "addfromurl-duration-val")
-		    .attr("placeholder", "Alternate Subtitle Track")
+                        .attr("placeholder", "Minimum Duration Filter")
                         .attr("style", "display: none; width: 100%;")
                         .keydown(function (ev) {
                             if (ev.keyCode === 13) {
@@ -816,6 +826,8 @@ $("#mediaurl").keyup(function(ev) {
                         })
                         .appendTo($("#addfromurl-title")).show("blind");//append and show
                 }
+
+            }
         } else {
             $("#addfromurl-title").hide("blind");
             $("#addfromurl-title").remove();//otherwise remove

www/js/util.js

@@ -1615,6 +1615,13 @@ function parseMediaLink(url) {
         };
     }
 
+    if((m = url.match(/archive\.org\/(?:details|download)\/([a-zA-Z0-9_-]+)(?!.|\/)/))){
+        return{
+            id: m[1],
+            type: "ia"
+        }
+    }
+
     /*  Shorthand URIs  */
     // So we still trim DailyMotion URLs
     if((m = url.match(/^dm:([^\?&#_]+)/))) {
@@ -3445,7 +3452,7 @@ function startQueueSpinner(data) {
     }
 
     var id = data.id;
-    if (data.type === "yp") {
+    if (data.type === "yp" || data.type === "ia") {
         id = "$any";
     }
 
@@ -3493,6 +3500,7 @@ function stopQueueSpinner(data) {
                         $("#queueprogress").data("queue-id") === data.id);
     shouldRemove = shouldRemove || data === null;
     shouldRemove = shouldRemove || $("#queueprogress").data("queue-id") === "$any";
+    //This is a gross way to fix the issue with IA but it works, and it's not like cytube was a pretty codebase anywho...
     if (shouldRemove) { 
         $("#queueprogress").remove();
     }