diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index 2fed178..9b632d8 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -106,7 +106,9 @@ namespace skyscraper5 if (args[0].ToLowerInvariant().EndsWith(".m3u8")) { M3U8Stream m3U8Stream = new M3U8Stream(args[0]); - SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext()); + DataStorage dataStorage = new InMemoryScraperStorage(); + ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo(".")); + SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), dataStorage, objectStorage); skyscraperContext.InitalizeFilterChain(); skyscraperContext.IngestFromStream(m3U8Stream); return; diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index 76da5f0..ddf05f4 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"", + "commandLineArgs": "F:\\alpha\\2026-03\\reuters-telstar-session3\\reuters3.m3u8", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/ReutersWne/IWneHandler.cs b/skyscraper8/ReutersWne/IWneHandler.cs index 006a9cc..4735190 100644 --- a/skyscraper8/ReutersWne/IWneHandler.cs +++ b/skyscraper8/ReutersWne/IWneHandler.cs @@ -11,6 +11,6 @@ namespace skyscraper8.ReutersWne void OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value); void OnWneStoryDetect(uint embeddedSessionId); void OnWneStoryFail(uint sessionId); - void OnWneStoryProgress(WneStoryProgress wneStoryProgress) + void OnWneStoryProgress(WneStoryProgress wneStoryProgress); } } diff --git a/skyscraper8/ReutersWne/ReutersWneExtractor.cs b/skyscraper8/ReutersWne/ReutersWneExtractor.cs index 6c9a61c..038578c 100644 --- a/skyscraper8/ReutersWne/ReutersWneExtractor.cs +++ b/skyscraper8/ReutersWne/ReutersWneExtractor.cs @@ -20,6 +20,16 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin //TODO: remember current time and skyscraper context if (wneStories == null) wneStories = new Dictionary(); + + IWneHandler contextableWneHandler = skyscraperContext as IWneHandler; + if (contextableWneHandler != null) + { + this.Handler = contextableWneHandler; + } + else + { + _logger.ErrorFormat("The provided SkyscraperContext does not support handling WNE stories."); + } } public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) @@ -151,6 +161,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin if (currentStory.Corrupted) { OnError("Story #{0} is corrupted and can not be recovered.", sessionId); + currentStory.Dispose(); wneStories.Remove(sessionId); return false; } @@ -169,7 +180,8 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin private void DeliverFile(WneStory story) { - Handler.OnWneStoryComplete(new WneStoryProgress(story), story.GetStream()); + Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream()); + _logger.InfoFormat("Extracted file: {0}", story.DestinationFileName); /* //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId); @@ -351,7 +363,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin } wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); - Handler.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId])); + Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId])); return true; } } @@ -520,11 +532,11 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin loggedErrors = new HashSet(); string unpacked = String.Format(message, args); - unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked); if (!loggedErrors.Contains(unpacked)) { - loggedErrors.Add(unpacked); - _logger.Warn(unpacked); + loggedErrors.Add(unpacked); + unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked); + _logger.Warn(unpacked); } } diff --git a/skyscraper8/ReutersWne/WneStory.cs b/skyscraper8/ReutersWne/WneStory.cs index 59c2d9c..7128f4c 100644 --- a/skyscraper8/ReutersWne/WneStory.cs +++ b/skyscraper8/ReutersWne/WneStory.cs @@ -1,135 +1,152 @@ -namespace skyscraper8.ReutersWne; - -internal class WneStory : IDisposable -{ - public uint SessionId { get; } - - public WneStory(uint sessionId) - { - SessionId = sessionId; - } - - public byte EccGroup; - public byte ExpectedEccGroup; - public int timesSucessfullySynced; - public int FileDeliveryType; - public int ExpectedContinuityCounter; - public string SourceFileName; - public string DestinationFileName; - public uint ExpectedPayloadBlock; - - private List payloadBlocks; - public bool Corrupted; - public bool Delivered; - - private MemoryStream preallocated; - private uint _fileSize; - public uint FileSize - { - get - { - return _fileSize; - } - set - { - if (payloadBlocks != null) - { - throw new NotImplementedException("Can't set the file size when a block was already delivered"); - } - preallocated = new MemoryStream(new byte[_fileSize]); - _fileSize = value; - } - } - - public long CaughtPayloadBytes; - - public void AppendPayloadBlock(Span slice) - { - if (preallocated != null) - { - preallocated.Write(slice); - ExpectedPayloadBlock++; - CaughtPayloadBytes += slice.Length; - } - else - { - if (payloadBlocks == null) - { - payloadBlocks = new List(); - } - payloadBlocks.Add(slice.ToArray()); - ExpectedPayloadBlock++; - CaughtPayloadBytes += slice.Length; - } - } - - public bool IsEmpty() - { - if (payloadBlocks == null) - { - return true; - } - - if (payloadBlocks.Count == 0) - { - return true; - } - - for (int i = 0; i < payloadBlocks.Count; i++) - { - if (payloadBlocks[i].Length > 0) - { - return false; - } - } - - return true; - } - - public Stream ToStream() - { - if (Delivered) - { - throw new InvalidOperationException("Cannot convert a delivered story to a stream."); - } - - if (preallocated != null) - { - preallocated.Position = 0; - return preallocated; - } - else - { - return new ListByteArrayStream(payloadBlocks); - } - } - - public void Dispose() - { - payloadBlocks?.Clear(); - payloadBlocks = null; - Delivered = true; - } - - public WneStoryProgress GetProgress() - { - return new WneStoryProgress(this); - } - -} - -struct WneStoryProgress -{ - internal WneStoryProgress(WneStory source) - { - this.Filename = source.SourceFileName; - this.SessionId = source.SessionId; - this.FileSize = source.FileSize; - this.FileCaught = source.CaughtPayloadBytes; - } - - public string Filename { get; } - public uint SessionId { get; } - public uint FileSize { get; } - public long FileCaught { get; } +namespace skyscraper8.ReutersWne; + +internal class WneStory : IDisposable +{ + public uint SessionId { get; } + + public WneStory(uint sessionId) + { + SessionId = sessionId; + } + + public byte EccGroup; + public byte ExpectedEccGroup; + public int timesSucessfullySynced; + public int FileDeliveryType; + public int ExpectedContinuityCounter; + public string SourceFileName; + public string DestinationFileName; + public uint ExpectedPayloadBlock; + + private List payloadBlocks; + public bool Corrupted; + public bool Delivered; + + private MemoryStream preallocated; + private uint _fileSize; + public uint FileSize + { + get + { + return _fileSize; + } + set + { + if (payloadBlocks != null) + { + throw new NotImplementedException("Can't set the file size when a block was already delivered"); + } + if (preallocated == null) + { + preallocated = new MemoryStream(new byte[value]); + } + else if (preallocated.Length != value) + { + preallocated.SetLength(value); + } + _fileSize = value; + } + } + + public long CaughtPayloadBytes; + + public void AppendPayloadBlock(Span slice) + { + if (preallocated != null) + { + int writeBlockLen = (int)Math.Min(slice.Length, preallocated.Length - preallocated.Position); + slice = slice.Slice(0, writeBlockLen); + preallocated.Write(slice); + ExpectedPayloadBlock++; + CaughtPayloadBytes += slice.Length; + } + else + { + if (payloadBlocks == null) + { + payloadBlocks = new List(); + } + payloadBlocks.Add(slice.ToArray()); + ExpectedPayloadBlock++; + CaughtPayloadBytes += slice.Length; + } + } + + public bool IsEmpty() + { + if (preallocated != null) + { + return preallocated.Position != preallocated.Length; + } + if (payloadBlocks == null) + { + return true; + } + + if (payloadBlocks.Count == 0) + { + return true; + } + + for (int i = 0; i < payloadBlocks.Count; i++) + { + if (payloadBlocks[i].Length > 0) + { + return false; + } + } + + return true; + } + + public Stream ToStream() + { + if (Delivered) + { + throw new InvalidOperationException("Cannot convert a delivered story to a stream."); + } + + if (preallocated != null) + { + preallocated.Position = 0; + return preallocated; + } + else + { + return new ListByteArrayStream(payloadBlocks); + } + } + + public void Dispose() + { + preallocated?.Dispose(); + preallocated = null; + payloadBlocks?.Clear(); + payloadBlocks = null; + Delivered = true; + } + + public WneStoryProgress GetProgress() + { + return new WneStoryProgress(this); + } + +} + +struct WneStoryProgress +{ + internal WneStoryProgress(WneStory source) + { + this.Filename = source.DestinationFileName; + this.SessionId = source.SessionId; + this.FileSize = source.FileSize; + this.FileCaught = source.CaughtPayloadBytes; + if (this.Filename.StartsWith("\\")) + this.Filename = this.Filename.Substring(1); + } + + public string Filename { get; } + public uint SessionId { get; } + public uint FileSize { get; } + public long FileCaught { get; } } \ No newline at end of file diff --git a/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs b/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs index 82377e0..d07519a 100644 --- a/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs +++ b/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs @@ -245,6 +245,10 @@ namespace skyscraper5.Skyscraper.Scraper void OnRcs2Tct(ushort interactiveNetworkId, Tct tct); void OnRcs2Tbtp(ushort interactiveNetworkId, Tbtp tbtp); void OnRcs2Sct(ushort interactiveNetworkId, Sct sct); - void OnRcs2Spt(ushort interactiveNetworkId, Spt spt); - } + void OnRcs2Spt(ushort interactiveNetworkId, Spt spt); + void WneStoryDetect(uint embeddedSessionId); + void WneStoryError(uint sessionId); + void WneStoryProgress(uint sessionId, string filename, long fileCaught, uint fileSize); + void WneStoryComplete(uint sessionId, string filename); + } } diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 90b5adc..fe13eb7 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -72,6 +72,7 @@ using skyscraper8.Ietf.Rfc4236_ULE; using skyscraper8.InteractionChannel.Model; using skyscraper8.InteractionChannel.Model2; using skyscraper8.InteractionChannel.Model2.Descriptors; +using skyscraper8.ReutersWne; using skyscraper8.Ses; using skyscraper8.Skyscraper.Net; using skyscraper8.Skyscraper.Scraper; @@ -99,7 +100,7 @@ namespace skyscraper5.Skyscraper.Scraper UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler, IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler, IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler, - InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler + InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler { public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true; public const bool ALLOW_FFMPEG_FRAMEGRABBER = true; @@ -3495,5 +3496,29 @@ namespace skyscraper5.Skyscraper.Scraper LogEvent(SkyscraperContextEvent.Tim, String.Format(" -> {0}", ts.ToString())); } } - } + + void IWneHandler.OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value) + { + UiJunction?.WneStoryComplete(wneStoryProgress.SessionId, wneStoryProgress.Filename); + if (!ObjectStorage.TestForWneStory(wneStoryProgress.SessionId,wneStoryProgress.Filename)) + { + ObjectStorage.StoreWneStory(wneStoryProgress.SessionId, wneStoryProgress.Filename, value); + } + } + + public void OnWneStoryDetect(uint embeddedSessionId) + { + UiJunction?.WneStoryDetect(embeddedSessionId); + } + + public void OnWneStoryFail(uint sessionId) + { + UiJunction?.WneStoryError(sessionId); + } + + void IWneHandler.OnWneStoryProgress(WneStoryProgress wneStoryProgress) + { + UiJunction?.WneStoryProgress(wneStoryProgress.SessionId, wneStoryProgress.Filename, wneStoryProgress.FileCaught, wneStoryProgress.FileSize); + } + } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs index 739e2c5..50dafb0 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs @@ -1788,5 +1788,29 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem RfSpectrumData result = RfSpectrumData.LoadFromStream(fileStream); return result; } - } + + public bool TestForWneStory(uint sessionId, string filename) + { + string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename); + FileInfo fi = new FileInfo(outFilename); + return fi.Exists; + } + + public void StoreWneStory(uint sessionId, string filename, Stream value) + { + string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename); + FileInfo fi = new FileInfo(outFilename); + fi.Directory.EnsureExists(); + + if (value.CanSeek) + value.Position = 0; + + FileStream fileStream = fi.OpenWrite(); + value.CopyTo(fileStream); + fileStream.Flush(); + fileStream.Close(); + + value.Dispose(); + } + } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs index c366a02..efbe5a9 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs @@ -168,5 +168,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage { throw new NotImplementedException(); } - } + + public bool TestForWneStory(uint sessionId, string filename) + { + return true; + } + + public void StoreWneStory(uint sessionId, string filename, Stream value) + { + throw new NotImplementedException(); + } + } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs index 4535af9..2e40f51 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs @@ -45,5 +45,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber); void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci); byte[] DvbNipGetFile(string path); - } + bool TestForWneStory(uint sessionId, string filename); + void StoreWneStory(uint sessionId, string filename, Stream value); + } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs index f6f9435..3f32ca4 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs @@ -235,6 +235,16 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar byte[] buffer = tarArchive.ReadEntry(filename); RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer)); return rfSpectrumData; - } - } + } + + public bool TestForWneStory(uint sessionId, string filename) + { + throw new NotImplementedException(); + } + + public void StoreWneStory(uint sessionId, string filename, Stream value) + { + throw new NotImplementedException(); + } + } }