Refactored WNE Extraction.

skyscraper8/Program.cs

@@ -106,7 +106,9 @@ namespace skyscraper5
                 if (args[0].ToLowerInvariant().EndsWith(".m3u8"))
                 {
                     M3U8Stream m3U8Stream = new M3U8Stream(args[0]);
-                    SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext());
+                    DataStorage dataStorage = new InMemoryScraperStorage();
+                    ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("."));
+                    SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
                     skyscraperContext.InitalizeFilterChain();
                     skyscraperContext.IngestFromStream(m3U8Stream);
                     return;

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"",
+      "commandLineArgs": "F:\\alpha\\2026-03\\reuters-telstar-session3\\reuters3.m3u8",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/ReutersWne/IWneHandler.cs

@@ -11,6 +11,6 @@ namespace skyscraper8.ReutersWne
 		void OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value);
 		void OnWneStoryDetect(uint embeddedSessionId);
 		void OnWneStoryFail(uint sessionId);
-		void OnWneStoryProgress(WneStoryProgress wneStoryProgress)
+		void OnWneStoryProgress(WneStoryProgress wneStoryProgress);
 	}
 }

skyscraper8/ReutersWne/ReutersWneExtractor.cs

@@ -20,6 +20,16 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
         //TODO: remember current time and skyscraper context
         if (wneStories == null)
 	        wneStories = new Dictionary<uint, WneStory>();
+
+		IWneHandler contextableWneHandler = skyscraperContext as IWneHandler;
+		if (contextableWneHandler != null)
+		{
+			this.Handler = contextableWneHandler;
+		}
+		else
+		{
+			_logger.ErrorFormat("The provided SkyscraperContext does not support handling WNE stories.");
+		}
     }
 
     public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet)
@@ -151,6 +161,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 	    if (currentStory.Corrupted)
 	    {
 		    OnError("Story #{0} is corrupted and can not be recovered.", sessionId);
+			currentStory.Dispose();
 		    wneStories.Remove(sessionId);
 		    return false;
 	    }
@@ -169,7 +180,8 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 
     private void DeliverFile(WneStory story)
     {
-		Handler.OnWneStoryComplete(new WneStoryProgress(story), story.GetStream());
+		Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream());
+		_logger.InfoFormat("Extracted file: {0}", story.DestinationFileName);
 
 		/*
 	    //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId);
@@ -351,7 +363,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 				}
 				
 				wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16));
-				Handler.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
+				Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
 				return true;
 			}
 		}
@@ -520,10 +532,10 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 		    loggedErrors = new HashSet<string>();
 
 	    string unpacked = String.Format(message, args);
-	    unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
 	    if (!loggedErrors.Contains(unpacked))
 	    {
 		    loggedErrors.Add(unpacked);
+            unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
             _logger.Warn(unpacked);
 	    }
     }

skyscraper8/ReutersWne/WneStory.cs

@@ -36,7 +36,14 @@ internal class WneStory : IDisposable
             {
                 throw new NotImplementedException("Can't set the file size when a block was already delivered");
             }
-            preallocated = new MemoryStream(new byte[_fileSize]);
+            if (preallocated == null)
+            {
+                preallocated = new MemoryStream(new byte[value]);
+            }
+            else if (preallocated.Length != value)
+            {
+                preallocated.SetLength(value);
+            }
             _fileSize = value;
         }
     }
@@ -47,6 +54,8 @@ internal class WneStory : IDisposable
     {
         if (preallocated != null)
         {
+            int writeBlockLen = (int)Math.Min(slice.Length, preallocated.Length - preallocated.Position);
+            slice = slice.Slice(0, writeBlockLen);
             preallocated.Write(slice);
             ExpectedPayloadBlock++;
             CaughtPayloadBytes += slice.Length;
@@ -65,6 +74,10 @@ internal class WneStory : IDisposable
 
     public bool IsEmpty()
     {
+        if (preallocated != null)
+        {
+            return preallocated.Position != preallocated.Length;
+        }
         if (payloadBlocks == null)
         {
             return true;
@@ -106,6 +119,8 @@ internal class WneStory : IDisposable
 
     public void Dispose()
     {
+        preallocated?.Dispose();
+        preallocated = null;
         payloadBlocks?.Clear();
         payloadBlocks = null;
         Delivered = true;
@@ -122,10 +137,12 @@ struct WneStoryProgress
 {
     internal WneStoryProgress(WneStory source)
     {
-        this.Filename = source.SourceFileName;
+        this.Filename = source.DestinationFileName;
         this.SessionId = source.SessionId;
 		this.FileSize = source.FileSize;
         this.FileCaught = source.CaughtPayloadBytes;
+        if (this.Filename.StartsWith("\\"))
+            this.Filename = this.Filename.Substring(1);
     }
 
 	public string Filename { get; }

skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs

@@ -246,5 +246,9 @@ namespace skyscraper5.Skyscraper.Scraper
 		void OnRcs2Tbtp(ushort interactiveNetworkId, Tbtp tbtp);
 		void OnRcs2Sct(ushort interactiveNetworkId, Sct sct);
 		void OnRcs2Spt(ushort interactiveNetworkId, Spt spt);
+        void WneStoryDetect(uint embeddedSessionId);
+        void WneStoryError(uint sessionId);
+        void WneStoryProgress(uint sessionId, string filename, long fileCaught, uint fileSize);
+        void WneStoryComplete(uint sessionId, string filename);
     }
 }

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -72,6 +72,7 @@ using skyscraper8.Ietf.Rfc4236_ULE;
 using skyscraper8.InteractionChannel.Model;
 using skyscraper8.InteractionChannel.Model2;
 using skyscraper8.InteractionChannel.Model2.Descriptors;
+using skyscraper8.ReutersWne;
 using skyscraper8.Ses;
 using skyscraper8.Skyscraper.Net;
 using skyscraper8.Skyscraper.Scraper;
@@ -99,7 +100,7 @@ namespace skyscraper5.Skyscraper.Scraper
         UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
         IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
 		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
-        InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler
+        InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
         public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@@ -3495,5 +3496,29 @@ namespace skyscraper5.Skyscraper.Scraper
 				LogEvent(SkyscraperContextEvent.Tim, String.Format(" -> {0}", ts.ToString()));
 			}
 		}
+
+        void IWneHandler.OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value)
+        {
+            UiJunction?.WneStoryComplete(wneStoryProgress.SessionId, wneStoryProgress.Filename);
+            if (!ObjectStorage.TestForWneStory(wneStoryProgress.SessionId,wneStoryProgress.Filename))
+            {
+                ObjectStorage.StoreWneStory(wneStoryProgress.SessionId, wneStoryProgress.Filename, value);
+            }
+        }
+
+        public void OnWneStoryDetect(uint embeddedSessionId)
+        {
+            UiJunction?.WneStoryDetect(embeddedSessionId);
+        }
+
+        public void OnWneStoryFail(uint sessionId)
+        {
+            UiJunction?.WneStoryError(sessionId);
+        }
+
+        void IWneHandler.OnWneStoryProgress(WneStoryProgress wneStoryProgress)
+        {
+            UiJunction?.WneStoryProgress(wneStoryProgress.SessionId, wneStoryProgress.Filename, wneStoryProgress.FileCaught, wneStoryProgress.FileSize);
+        }
     }
 }

skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs

@@ -1788,5 +1788,29 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
             RfSpectrumData result = RfSpectrumData.LoadFromStream(fileStream);
             return result;
 		}
+
+        public bool TestForWneStory(uint sessionId, string filename)
+        {
+            string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
+            FileInfo fi = new FileInfo(outFilename);
+            return fi.Exists;
+        }
+
+        public void StoreWneStory(uint sessionId, string filename, Stream value)
+        {
+            string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
+            FileInfo fi = new FileInfo(outFilename);
+            fi.Directory.EnsureExists();
+
+            if (value.CanSeek)
+                value.Position = 0;
+
+            FileStream fileStream = fi.OpenWrite();
+            value.CopyTo(fileStream);
+            fileStream.Flush();
+            fileStream.Close();
+
+            value.Dispose();
+        }
     }
 }

skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs

@@ -168,5 +168,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 		{
 			throw new NotImplementedException();
 		}
+
+        public bool TestForWneStory(uint sessionId, string filename)
+        {
+			return true;
+        }
+
+        public void StoreWneStory(uint sessionId, string filename, Stream value)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs

@@ -45,5 +45,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
         bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber);
         void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci);
 		byte[] DvbNipGetFile(string path);
+        bool TestForWneStory(uint sessionId, string filename);
+        void StoreWneStory(uint sessionId, string filename, Stream value);
     }
 }

skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs

@@ -236,5 +236,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
 			RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer));
 			return rfSpectrumData;
 		}
+
+        public bool TestForWneStory(uint sessionId, string filename)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void StoreWneStory(uint sessionId, string filename, Stream value)
+        {
+            throw new NotImplementedException();
+        }
     }
 }