ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Refactored WNE Extraction.
Some checks failed
🚀 Pack skyscraper8 / make-zip (push) Failing after 25s
Details

Browse Source
This commit is contained in:
Fey 2026-05-05 22:58:51 +02:00
parent 5715e047a3
commit fa91364b3b
11 changed files with 257 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
skyscraper8/Program.cs
View File

@ -106,7 +106,9 @@ namespace skyscraper5
if (args[0].ToLowerInvariant().EndsWith(".m3u8"))
{
M3U8Stream m3U8Stream = new M3U8Stream(args[0]);
SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext());
DataStorage dataStorage = new InMemoryScraperStorage();
ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("."));
SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
skyscraperContext.InitalizeFilterChain();
skyscraperContext.IngestFromStream(m3U8Stream);
return;

2
skyscraper8/Properties/launchSettings.json
View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"",
"commandLineArgs": "F:\\alpha\\2026-03\\reuters-telstar-session3\\reuters3.m3u8",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

2
skyscraper8/ReutersWne/IWneHandler.cs
View File

@ -11,6 +11,6 @@ namespace skyscraper8.ReutersWne
void OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value);
void OnWneStoryDetect(uint embeddedSessionId);
void OnWneStoryFail(uint sessionId);
void OnWneStoryProgress(WneStoryProgress wneStoryProgress)
void OnWneStoryProgress(WneStoryProgress wneStoryProgress);
}
}

20
skyscraper8/ReutersWne/ReutersWneExtractor.cs
View File

@ -20,6 +20,16 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
//TODO: remember current time and skyscraper context
if (wneStories == null)
wneStories = new Dictionary<uint, WneStory>();
IWneHandler contextableWneHandler = skyscraperContext as IWneHandler;
if (contextableWneHandler != null)
{
this.Handler = contextableWneHandler;
}
else
{
_logger.ErrorFormat("The provided SkyscraperContext does not support handling WNE stories.");
}
}
public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet)
@ -151,6 +161,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
if (currentStory.Corrupted)
{
OnError("Story #{0} is corrupted and can not be recovered.", sessionId);
currentStory.Dispose();
wneStories.Remove(sessionId);
return false;
}
@ -169,7 +180,8 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
private void DeliverFile(WneStory story)
{
Handler.OnWneStoryComplete(new WneStoryProgress(story), story.GetStream());
Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream());
_logger.InfoFormat("Extracted file: {0}", story.DestinationFileName);
/*
//_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId);
@ -351,7 +363,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
}
wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16));
Handler.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
return true;
}
}
@ -520,11 +532,11 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
loggedErrors = new HashSet<string>();
string unpacked = String.Format(message, args);
unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
if (!loggedErrors.Contains(unpacked))
{
loggedErrors.Add(unpacked);
_logger.Warn(unpacked);
unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
_logger.Warn(unpacked);
}
}

21
skyscraper8/ReutersWne/WneStory.cs
View File

@ -36,7 +36,14 @@ internal class WneStory : IDisposable
{
throw new NotImplementedException("Can't set the file size when a block was already delivered");
}
preallocated = new MemoryStream(new byte[_fileSize]);
if (preallocated == null)
{
preallocated = new MemoryStream(new byte[value]);
}
else if (preallocated.Length != value)
{
preallocated.SetLength(value);
}
_fileSize = value;
}
}
@ -47,6 +54,8 @@ internal class WneStory : IDisposable
{
if (preallocated != null)
{
int writeBlockLen = (int)Math.Min(slice.Length, preallocated.Length - preallocated.Position);
slice = slice.Slice(0, writeBlockLen);
preallocated.Write(slice);
ExpectedPayloadBlock++;
CaughtPayloadBytes += slice.Length;
@ -65,6 +74,10 @@ internal class WneStory : IDisposable
public bool IsEmpty()
{
if (preallocated != null)
{
return preallocated.Position != preallocated.Length;
}
if (payloadBlocks == null)
{
return true;
@ -106,6 +119,8 @@ internal class WneStory : IDisposable
public void Dispose()
{
preallocated?.Dispose();
preallocated = null;
payloadBlocks?.Clear();
payloadBlocks = null;
Delivered = true;
@ -122,10 +137,12 @@ struct WneStoryProgress
{
internal WneStoryProgress(WneStory source)
{
this.Filename = source.SourceFileName;
this.Filename = source.DestinationFileName;
this.SessionId = source.SessionId;
this.FileSize = source.FileSize;
this.FileCaught = source.CaughtPayloadBytes;
if (this.Filename.StartsWith("\\"))
this.Filename = this.Filename.Substring(1);
}
public string Filename { get; }

6
skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs
View File

@ -246,5 +246,9 @@ namespace skyscraper5.Skyscraper.Scraper
void OnRcs2Tbtp(ushort interactiveNetworkId, Tbtp tbtp);
void OnRcs2Sct(ushort interactiveNetworkId, Sct sct);
void OnRcs2Spt(ushort interactiveNetworkId, Spt spt);
}
void WneStoryDetect(uint embeddedSessionId);
void WneStoryError(uint sessionId);
void WneStoryProgress(uint sessionId, string filename, long fileCaught, uint fileSize);
void WneStoryComplete(uint sessionId, string filename);
}
}

29
skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs
View File

@ -72,6 +72,7 @@ using skyscraper8.Ietf.Rfc4236_ULE;
using skyscraper8.InteractionChannel.Model;
using skyscraper8.InteractionChannel.Model2;
using skyscraper8.InteractionChannel.Model2.Descriptors;
using skyscraper8.ReutersWne;
using skyscraper8.Ses;
using skyscraper8.Skyscraper.Net;
using skyscraper8.Skyscraper.Scraper;
@ -99,7 +100,7 @@ namespace skyscraper5.Skyscraper.Scraper
UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler
{
public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@ -3495,5 +3496,29 @@ namespace skyscraper5.Skyscraper.Scraper
LogEvent(SkyscraperContextEvent.Tim, String.Format(" -> {0}", ts.ToString()));
}
}
}
void IWneHandler.OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value)
{
UiJunction?.WneStoryComplete(wneStoryProgress.SessionId, wneStoryProgress.Filename);
if (!ObjectStorage.TestForWneStory(wneStoryProgress.SessionId,wneStoryProgress.Filename))
{
ObjectStorage.StoreWneStory(wneStoryProgress.SessionId, wneStoryProgress.Filename, value);
}
}
public void OnWneStoryDetect(uint embeddedSessionId)
{
UiJunction?.WneStoryDetect(embeddedSessionId);
}
public void OnWneStoryFail(uint sessionId)
{
UiJunction?.WneStoryError(sessionId);
}
void IWneHandler.OnWneStoryProgress(WneStoryProgress wneStoryProgress)
{
UiJunction?.WneStoryProgress(wneStoryProgress.SessionId, wneStoryProgress.Filename, wneStoryProgress.FileCaught, wneStoryProgress.FileSize);
}
}
}

26
skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs
View File

@ -1788,5 +1788,29 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
RfSpectrumData result = RfSpectrumData.LoadFromStream(fileStream);
return result;
}
}
public bool TestForWneStory(uint sessionId, string filename)
{
string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
FileInfo fi = new FileInfo(outFilename);
return fi.Exists;
}
public void StoreWneStory(uint sessionId, string filename, Stream value)
{
string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
FileInfo fi = new FileInfo(outFilename);
fi.Directory.EnsureExists();
if (value.CanSeek)
value.Position = 0;
FileStream fileStream = fi.OpenWrite();
value.CopyTo(fileStream);
fileStream.Flush();
fileStream.Close();
value.Dispose();
}
}
}

12
skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs
View File

@ -168,5 +168,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
{
throw new NotImplementedException();
}
}
public bool TestForWneStory(uint sessionId, string filename)
{
return true;
}
public void StoreWneStory(uint sessionId, string filename, Stream value)
{
throw new NotImplementedException();
}
}
}

4
skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs
View File

@ -45,5 +45,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber);
void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci);
byte[] DvbNipGetFile(string path);
}
bool TestForWneStory(uint sessionId, string filename);
void StoreWneStory(uint sessionId, string filename, Stream value);
}
}

12
skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs
View File

@ -236,5 +236,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer));
return rfSpectrumData;
}
}
public bool TestForWneStory(uint sessionId, string filename)
{
throw new NotImplementedException();
}
public void StoreWneStory(uint sessionId, string filename, Stream value)
{
throw new NotImplementedException();
}
}
}