Refactored WNE Extraction.

skyscraper8/Program.cs

@@ -106,7 +106,9 @@ namespace skyscraper5
                 if (args[0].ToLowerInvariant().EndsWith(".m3u8"))
                 {
                     M3U8Stream m3U8Stream = new M3U8Stream(args[0]);
-                    SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext());
+                    DataStorage dataStorage = new InMemoryScraperStorage();
+                    ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("."));
+                    SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
                     skyscraperContext.InitalizeFilterChain();
                     skyscraperContext.IngestFromStream(m3U8Stream);
                     return;

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"",
+      "commandLineArgs": "F:\\alpha\\2026-03\\reuters-telstar-session3\\reuters3.m3u8",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/ReutersWne/IWneHandler.cs

@@ -11,6 +11,6 @@ namespace skyscraper8.ReutersWne
 		void OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value);
 		void OnWneStoryDetect(uint embeddedSessionId);
 		void OnWneStoryFail(uint sessionId);
-		void OnWneStoryProgress(WneStoryProgress wneStoryProgress)
+		void OnWneStoryProgress(WneStoryProgress wneStoryProgress);
 	}
 }

skyscraper8/ReutersWne/ReutersWneExtractor.cs

@@ -20,6 +20,16 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
         //TODO: remember current time and skyscraper context
         if (wneStories == null)
 	        wneStories = new Dictionary<uint, WneStory>();
+
+		IWneHandler contextableWneHandler = skyscraperContext as IWneHandler;
+		if (contextableWneHandler != null)
+		{
+			this.Handler = contextableWneHandler;
+		}
+		else
+		{
+			_logger.ErrorFormat("The provided SkyscraperContext does not support handling WNE stories.");
+		}
     }
 
     public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet)
@@ -151,6 +161,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 	    if (currentStory.Corrupted)
 	    {
 		    OnError("Story #{0} is corrupted and can not be recovered.", sessionId);
+			currentStory.Dispose();
 		    wneStories.Remove(sessionId);
 		    return false;
 	    }
@@ -169,7 +180,8 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 
     private void DeliverFile(WneStory story)
     {
-		Handler.OnWneStoryComplete(new WneStoryProgress(story), story.GetStream());
+		Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream());
+		_logger.InfoFormat("Extracted file: {0}", story.DestinationFileName);
 
 		/*
 	    //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId);
@@ -351,7 +363,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 				}
 				
 				wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16));
-				Handler.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
+				Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
 				return true;
 			}
 		}
@@ -520,11 +532,11 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
 		    loggedErrors = new HashSet<string>();
 
 	    string unpacked = String.Format(message, args);
-	    unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
 	    if (!loggedErrors.Contains(unpacked))
 	    {
-		    loggedErrors.Add(unpacked);
+		    loggedErrors.Add(unpacked);
-		    _logger.Warn(unpacked);
+            unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
+            _logger.Warn(unpacked);
 	    }
     }
 

skyscraper8/ReutersWne/WneStory.cs

@@ -1,135 +1,152 @@
-namespace skyscraper8.ReutersWne;
+namespace skyscraper8.ReutersWne;
-
+
-internal class WneStory : IDisposable
+internal class WneStory : IDisposable
-{
+{
-    public uint SessionId { get; }
+    public uint SessionId { get; }
-
+
-    public WneStory(uint sessionId)
+    public WneStory(uint sessionId)
-    {
+    {
-        SessionId = sessionId;
+        SessionId = sessionId;
-    }
+    }
-    
+    
-    public byte EccGroup;
+    public byte EccGroup;
-    public byte ExpectedEccGroup;
+    public byte ExpectedEccGroup;
-    public int timesSucessfullySynced;
+    public int timesSucessfullySynced;
-    public int FileDeliveryType;
+    public int FileDeliveryType;
-    public int ExpectedContinuityCounter;
+    public int ExpectedContinuityCounter;
-    public string SourceFileName;
+    public string SourceFileName;
-    public string DestinationFileName;
+    public string DestinationFileName;
-    public uint ExpectedPayloadBlock;
+    public uint ExpectedPayloadBlock;
-
+
-    private List<byte[]> payloadBlocks;
+    private List<byte[]> payloadBlocks;
-    public bool Corrupted;
+    public bool Corrupted;
-    public bool Delivered;
+    public bool Delivered;
-
+
-    private MemoryStream preallocated;
+    private MemoryStream preallocated;
-    private uint _fileSize;
+    private uint _fileSize;
-    public uint FileSize
+    public uint FileSize
-    {
+    {
-        get 
+        get 
-        { 
+        { 
-            return _fileSize; 
+            return _fileSize; 
-        }
+        }
-        set
+        set
-        {
+        {
-            if (payloadBlocks != null)
+            if (payloadBlocks != null)
-            {
+            {
-                throw new NotImplementedException("Can't set the file size when a block was already delivered");
+                throw new NotImplementedException("Can't set the file size when a block was already delivered");
-            }
+            }
-            preallocated = new MemoryStream(new byte[_fileSize]);
+            if (preallocated == null)
-            _fileSize = value;
+            {
-        }
+                preallocated = new MemoryStream(new byte[value]);
-    }
+            }
-
+            else if (preallocated.Length != value)
-    public long CaughtPayloadBytes;
+            {
-
+                preallocated.SetLength(value);
-    public void AppendPayloadBlock(Span<byte> slice)
+            }
-    {
+            _fileSize = value;
-        if (preallocated != null)
+        }
-        {
+    }
-            preallocated.Write(slice);
+
-            ExpectedPayloadBlock++;
+    public long CaughtPayloadBytes;
-            CaughtPayloadBytes += slice.Length;
+
-        }
+    public void AppendPayloadBlock(Span<byte> slice)
-        else
+    {
-        {
+        if (preallocated != null)
-            if (payloadBlocks == null)
+        {
-            {
+            int writeBlockLen = (int)Math.Min(slice.Length, preallocated.Length - preallocated.Position);
-                payloadBlocks = new List<byte[]>();
+            slice = slice.Slice(0, writeBlockLen);
-            }
+            preallocated.Write(slice);
-            payloadBlocks.Add(slice.ToArray());
+            ExpectedPayloadBlock++;
-            ExpectedPayloadBlock++;
+            CaughtPayloadBytes += slice.Length;
-            CaughtPayloadBytes += slice.Length;
+        }
-        }
+        else
-    }
+        {
-
+            if (payloadBlocks == null)
-    public bool IsEmpty()
+            {
-    {
+                payloadBlocks = new List<byte[]>();
-        if (payloadBlocks == null)
+            }
-        {
+            payloadBlocks.Add(slice.ToArray());
-            return true;
+            ExpectedPayloadBlock++;
-        }
+            CaughtPayloadBytes += slice.Length;
-
+        }
-        if (payloadBlocks.Count == 0)
+    }
-        {
+
-            return true;
+    public bool IsEmpty()
-        }
+    {
-        
+        if (preallocated != null)
-        for (int i = 0; i < payloadBlocks.Count; i++)
+        {
-        {
+            return preallocated.Position != preallocated.Length;
-            if (payloadBlocks[i].Length > 0)
+        }
-            {
+        if (payloadBlocks == null)
-                return false;
+        {
-            }
+            return true;
-        }
+        }
-        
+
-        return true;
+        if (payloadBlocks.Count == 0)
-    }
+        {
-
+            return true;
-    public Stream ToStream()
+        }
-    {
+        
-        if (Delivered)
+        for (int i = 0; i < payloadBlocks.Count; i++)
-        {
+        {
-            throw new InvalidOperationException("Cannot convert a delivered story to a stream.");
+            if (payloadBlocks[i].Length > 0)
-        }
+            {
-
+                return false;
-		if (preallocated != null)
+            }
-		{
+        }
-			preallocated.Position = 0;
+        
-			return preallocated;
+        return true;
-		}
+    }
-		else
+
-		{
+    public Stream ToStream()
-			return new ListByteArrayStream(payloadBlocks);
+    {
-		}
+        if (Delivered)
-    }
+        {
-
+            throw new InvalidOperationException("Cannot convert a delivered story to a stream.");
-    public void Dispose()
+        }
-    {
+
-        payloadBlocks?.Clear();
+		if (preallocated != null)
-        payloadBlocks = null;
+		{
-        Delivered = true;
+			preallocated.Position = 0;
-    }
+			return preallocated;
-
+		}
-    public WneStoryProgress GetProgress()
+		else
-    {
+		{
-        return new WneStoryProgress(this);
+			return new ListByteArrayStream(payloadBlocks);
-    }
+		}
-
+    }
-}
+
-
+    public void Dispose()
-struct WneStoryProgress
+    {
-{
+        preallocated?.Dispose();
-    internal WneStoryProgress(WneStory source)
+        preallocated = null;
-    {
+        payloadBlocks?.Clear();
-        this.Filename = source.SourceFileName;
+        payloadBlocks = null;
-        this.SessionId = source.SessionId;
+        Delivered = true;
-		this.FileSize = source.FileSize;
+    }
-        this.FileCaught = source.CaughtPayloadBytes;
+
-    }
+    public WneStoryProgress GetProgress()
-
+    {
-	public string Filename { get; }
+        return new WneStoryProgress(this);
-	public uint SessionId { get; }
+    }
-	public uint FileSize { get; }
+
-	public long FileCaught { get; }
+}
+
+struct WneStoryProgress
+{
+    internal WneStoryProgress(WneStory source)
+    {
+        this.Filename = source.DestinationFileName;
+        this.SessionId = source.SessionId;
+		this.FileSize = source.FileSize;
+        this.FileCaught = source.CaughtPayloadBytes;
+        if (this.Filename.StartsWith("\\"))
+            this.Filename = this.Filename.Substring(1);
+    }
+
+	public string Filename { get; }
+	public uint SessionId { get; }
+	public uint FileSize { get; }
+	public long FileCaught { get; }
 }

skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs

@@ -245,6 +245,10 @@ namespace skyscraper5.Skyscraper.Scraper
 		void OnRcs2Tct(ushort interactiveNetworkId, Tct tct);
 		void OnRcs2Tbtp(ushort interactiveNetworkId, Tbtp tbtp);
 		void OnRcs2Sct(ushort interactiveNetworkId, Sct sct);
-		void OnRcs2Spt(ushort interactiveNetworkId, Spt spt);
+		void OnRcs2Spt(ushort interactiveNetworkId, Spt spt);
-	}
+        void WneStoryDetect(uint embeddedSessionId);
+        void WneStoryError(uint sessionId);
+        void WneStoryProgress(uint sessionId, string filename, long fileCaught, uint fileSize);
+        void WneStoryComplete(uint sessionId, string filename);
+    }
 }

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -72,6 +72,7 @@ using skyscraper8.Ietf.Rfc4236_ULE;
 using skyscraper8.InteractionChannel.Model;
 using skyscraper8.InteractionChannel.Model2;
 using skyscraper8.InteractionChannel.Model2.Descriptors;
+using skyscraper8.ReutersWne;
 using skyscraper8.Ses;
 using skyscraper8.Skyscraper.Net;
 using skyscraper8.Skyscraper.Scraper;
@@ -99,7 +100,7 @@ namespace skyscraper5.Skyscraper.Scraper
         UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
         IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
 		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
-        InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler
+        InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
         public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@@ -3495,5 +3496,29 @@ namespace skyscraper5.Skyscraper.Scraper
 				LogEvent(SkyscraperContextEvent.Tim, String.Format(" -> {0}", ts.ToString()));
 			}
 		}
-	}
+
+        void IWneHandler.OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value)
+        {
+            UiJunction?.WneStoryComplete(wneStoryProgress.SessionId, wneStoryProgress.Filename);
+            if (!ObjectStorage.TestForWneStory(wneStoryProgress.SessionId,wneStoryProgress.Filename))
+            {
+                ObjectStorage.StoreWneStory(wneStoryProgress.SessionId, wneStoryProgress.Filename, value);
+            }
+        }
+
+        public void OnWneStoryDetect(uint embeddedSessionId)
+        {
+            UiJunction?.WneStoryDetect(embeddedSessionId);
+        }
+
+        public void OnWneStoryFail(uint sessionId)
+        {
+            UiJunction?.WneStoryError(sessionId);
+        }
+
+        void IWneHandler.OnWneStoryProgress(WneStoryProgress wneStoryProgress)
+        {
+            UiJunction?.WneStoryProgress(wneStoryProgress.SessionId, wneStoryProgress.Filename, wneStoryProgress.FileCaught, wneStoryProgress.FileSize);
+        }
+    }
 }

skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs

@@ -1788,5 +1788,29 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
             RfSpectrumData result = RfSpectrumData.LoadFromStream(fileStream);
             return result;
 		}
-	}
+
+        public bool TestForWneStory(uint sessionId, string filename)
+        {
+            string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
+            FileInfo fi = new FileInfo(outFilename);
+            return fi.Exists;
+        }
+
+        public void StoreWneStory(uint sessionId, string filename, Stream value)
+        {
+            string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
+            FileInfo fi = new FileInfo(outFilename);
+            fi.Directory.EnsureExists();
+
+            if (value.CanSeek)
+                value.Position = 0;
+
+            FileStream fileStream = fi.OpenWrite();
+            value.CopyTo(fileStream);
+            fileStream.Flush();
+            fileStream.Close();
+
+            value.Dispose();
+        }
+    }
 }

skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs

@@ -168,5 +168,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 		{
 			throw new NotImplementedException();
 		}
-	}
+
+        public bool TestForWneStory(uint sessionId, string filename)
+        {
+			return true;
+        }
+
+        public void StoreWneStory(uint sessionId, string filename, Stream value)
+        {
+            throw new NotImplementedException();
+        }
+    }
 }

skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs

@@ -45,5 +45,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
         bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber);
         void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci);
 		byte[] DvbNipGetFile(string path);
-	}
+        bool TestForWneStory(uint sessionId, string filename);
+        void StoreWneStory(uint sessionId, string filename, Stream value);
+    }
 }

skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs

@@ -235,6 +235,16 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
 			byte[] buffer = tarArchive.ReadEntry(filename);
 			RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer));
 			return rfSpectrumData;
-		}
+		}
-	}
+
+        public bool TestForWneStory(uint sessionId, string filename)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void StoreWneStory(uint sessionId, string filename, Stream value)
+        {
+            throw new NotImplementedException();
+        }
+    }
 }