ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Refactored WNE Extraction.
Some checks failed
🚀 Pack skyscraper8 / make-zip (push) Failing after 25s
Details

Browse Source
This commit is contained in:
Fey 2026-05-05 22:58:51 +02:00
parent 5715e047a3
commit fa91364b3b
11 changed files with 257 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
skyscraper8/Program.cs
View File

@ -106,7 +106,9 @@ namespace skyscraper5
if (args[0].ToLowerInvariant().EndsWith(".m3u8"))
{
M3U8Stream m3U8Stream = new M3U8Stream(args[0]);
SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext());
DataStorage dataStorage = new InMemoryScraperStorage();
ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("."));
SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
skyscraperContext.InitalizeFilterChain();
skyscraperContext.IngestFromStream(m3U8Stream);
return;

2
skyscraper8/Properties/launchSettings.json
View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"",
"commandLineArgs": "F:\\alpha\\2026-03\\reuters-telstar-session3\\reuters3.m3u8",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

2
skyscraper8/ReutersWne/IWneHandler.cs
View File

@ -11,6 +11,6 @@ namespace skyscraper8.ReutersWne
void OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value);
void OnWneStoryDetect(uint embeddedSessionId);
void OnWneStoryFail(uint sessionId);
void OnWneStoryProgress(WneStoryProgress wneStoryProgress)
void OnWneStoryProgress(WneStoryProgress wneStoryProgress);
}
}

22
skyscraper8/ReutersWne/ReutersWneExtractor.cs
View File

@ -20,6 +20,16 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
//TODO: remember current time and skyscraper context
if (wneStories == null)
wneStories = new Dictionary<uint, WneStory>();
IWneHandler contextableWneHandler = skyscraperContext as IWneHandler;
if (contextableWneHandler != null)
{
this.Handler = contextableWneHandler;
}
else
{
_logger.ErrorFormat("The provided SkyscraperContext does not support handling WNE stories.");
}
}
public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet)
@ -151,6 +161,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
if (currentStory.Corrupted)
{
OnError("Story #{0} is corrupted and can not be recovered.", sessionId);
currentStory.Dispose();
wneStories.Remove(sessionId);
return false;
}
@ -169,7 +180,8 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
private void DeliverFile(WneStory story)
{
Handler.OnWneStoryComplete(new WneStoryProgress(story), story.GetStream());
Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream());
_logger.InfoFormat("Extracted file: {0}", story.DestinationFileName);
/*
//_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId);
@ -351,7 +363,7 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
}
wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16));
Handler.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId]));
return true;
}
}
@ -520,11 +532,11 @@ internal class ReutersWneExtractor : ISkyscraperMpePlugin
loggedErrors = new HashSet<string>();
string unpacked = String.Format(message, args);
unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
if (!loggedErrors.Contains(unpacked))
{
loggedErrors.Add(unpacked);
_logger.Warn(unpacked);
loggedErrors.Add(unpacked);
unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
_logger.Warn(unpacked);
}
}

285
skyscraper8/ReutersWne/WneStory.cs
View File

@ -1,135 +1,152 @@
namespace skyscraper8.ReutersWne;
internal class WneStory : IDisposable
{
public uint SessionId { get; }
public WneStory(uint sessionId)
{
SessionId = sessionId;
}
public byte EccGroup;
public byte ExpectedEccGroup;
public int timesSucessfullySynced;
public int FileDeliveryType;
public int ExpectedContinuityCounter;
public string SourceFileName;
public string DestinationFileName;
public uint ExpectedPayloadBlock;
private List<byte[]> payloadBlocks;
public bool Corrupted;
public bool Delivered;
private MemoryStream preallocated;
private uint _fileSize;
public uint FileSize
{
get
{
return _fileSize;
}
set
{
if (payloadBlocks != null)
{
throw new NotImplementedException("Can't set the file size when a block was already delivered");
}
preallocated = new MemoryStream(new byte[_fileSize]);
_fileSize = value;
}
}
public long CaughtPayloadBytes;
public void AppendPayloadBlock(Span<byte> slice)
{
if (preallocated != null)
{
preallocated.Write(slice);
ExpectedPayloadBlock++;
CaughtPayloadBytes += slice.Length;
}
else
{
if (payloadBlocks == null)
{
payloadBlocks = new List<byte[]>();
}
payloadBlocks.Add(slice.ToArray());
ExpectedPayloadBlock++;
CaughtPayloadBytes += slice.Length;
}
}
public bool IsEmpty()
{
if (payloadBlocks == null)
{
return true;
}
if (payloadBlocks.Count == 0)
{
return true;
}
for (int i = 0; i < payloadBlocks.Count; i++)
{
if (payloadBlocks[i].Length > 0)
{
return false;
}
}
return true;
}
public Stream ToStream()
{
if (Delivered)
{
throw new InvalidOperationException("Cannot convert a delivered story to a stream.");
}
if (preallocated != null)
{
preallocated.Position = 0;
return preallocated;
}
else
{
return new ListByteArrayStream(payloadBlocks);
}
}
public void Dispose()
{
payloadBlocks?.Clear();
payloadBlocks = null;
Delivered = true;
}
public WneStoryProgress GetProgress()
{
return new WneStoryProgress(this);
}
}
struct WneStoryProgress
{
internal WneStoryProgress(WneStory source)
{
this.Filename = source.SourceFileName;
this.SessionId = source.SessionId;
this.FileSize = source.FileSize;
this.FileCaught = source.CaughtPayloadBytes;
}
public string Filename { get; }
public uint SessionId { get; }
public uint FileSize { get; }
public long FileCaught { get; }
namespace skyscraper8.ReutersWne;
internal class WneStory : IDisposable
{
public uint SessionId { get; }
public WneStory(uint sessionId)
{
SessionId = sessionId;
}
public byte EccGroup;
public byte ExpectedEccGroup;
public int timesSucessfullySynced;
public int FileDeliveryType;
public int ExpectedContinuityCounter;
public string SourceFileName;
public string DestinationFileName;
public uint ExpectedPayloadBlock;
private List<byte[]> payloadBlocks;
public bool Corrupted;
public bool Delivered;
private MemoryStream preallocated;
private uint _fileSize;
public uint FileSize
{
get
{
return _fileSize;
}
set
{
if (payloadBlocks != null)
{
throw new NotImplementedException("Can't set the file size when a block was already delivered");
}
if (preallocated == null)
{
preallocated = new MemoryStream(new byte[value]);
}
else if (preallocated.Length != value)
{
preallocated.SetLength(value);
}
_fileSize = value;
}
}
public long CaughtPayloadBytes;
public void AppendPayloadBlock(Span<byte> slice)
{
if (preallocated != null)
{
int writeBlockLen = (int)Math.Min(slice.Length, preallocated.Length - preallocated.Position);
slice = slice.Slice(0, writeBlockLen);
preallocated.Write(slice);
ExpectedPayloadBlock++;
CaughtPayloadBytes += slice.Length;
}
else
{
if (payloadBlocks == null)
{
payloadBlocks = new List<byte[]>();
}
payloadBlocks.Add(slice.ToArray());
ExpectedPayloadBlock++;
CaughtPayloadBytes += slice.Length;
}
}
public bool IsEmpty()
{
if (preallocated != null)
{
return preallocated.Position != preallocated.Length;
}
if (payloadBlocks == null)
{
return true;
}
if (payloadBlocks.Count == 0)
{
return true;
}
for (int i = 0; i < payloadBlocks.Count; i++)
{
if (payloadBlocks[i].Length > 0)
{
return false;
}
}
return true;
}
public Stream ToStream()
{
if (Delivered)
{
throw new InvalidOperationException("Cannot convert a delivered story to a stream.");
}
if (preallocated != null)
{
preallocated.Position = 0;
return preallocated;
}
else
{
return new ListByteArrayStream(payloadBlocks);
}
}
public void Dispose()
{
preallocated?.Dispose();
preallocated = null;
payloadBlocks?.Clear();
payloadBlocks = null;
Delivered = true;
}
public WneStoryProgress GetProgress()
{
return new WneStoryProgress(this);
}
}
struct WneStoryProgress
{
internal WneStoryProgress(WneStory source)
{
this.Filename = source.DestinationFileName;
this.SessionId = source.SessionId;
this.FileSize = source.FileSize;
this.FileCaught = source.CaughtPayloadBytes;
if (this.Filename.StartsWith("\\"))
this.Filename = this.Filename.Substring(1);
}
public string Filename { get; }
public uint SessionId { get; }
public uint FileSize { get; }
public long FileCaught { get; }
}

8
skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs
View File

@ -245,6 +245,10 @@ namespace skyscraper5.Skyscraper.Scraper
void OnRcs2Tct(ushort interactiveNetworkId, Tct tct);
void OnRcs2Tbtp(ushort interactiveNetworkId, Tbtp tbtp);
void OnRcs2Sct(ushort interactiveNetworkId, Sct sct);
void OnRcs2Spt(ushort interactiveNetworkId, Spt spt);
}
void OnRcs2Spt(ushort interactiveNetworkId, Spt spt);
void WneStoryDetect(uint embeddedSessionId);
void WneStoryError(uint sessionId);
void WneStoryProgress(uint sessionId, string filename, long fileCaught, uint fileSize);
void WneStoryComplete(uint sessionId, string filename);
}
}

29
skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs
View File

@ -72,6 +72,7 @@ using skyscraper8.Ietf.Rfc4236_ULE;
using skyscraper8.InteractionChannel.Model;
using skyscraper8.InteractionChannel.Model2;
using skyscraper8.InteractionChannel.Model2.Descriptors;
using skyscraper8.ReutersWne;
using skyscraper8.Ses;
using skyscraper8.Skyscraper.Net;
using skyscraper8.Skyscraper.Scraper;
@ -99,7 +100,7 @@ namespace skyscraper5.Skyscraper.Scraper
UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler
{
public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@ -3495,5 +3496,29 @@ namespace skyscraper5.Skyscraper.Scraper
LogEvent(SkyscraperContextEvent.Tim, String.Format(" -> {0}", ts.ToString()));
}
}
}
void IWneHandler.OnWneStoryComplete(WneStoryProgress wneStoryProgress, Stream value)
{
UiJunction?.WneStoryComplete(wneStoryProgress.SessionId, wneStoryProgress.Filename);
if (!ObjectStorage.TestForWneStory(wneStoryProgress.SessionId,wneStoryProgress.Filename))
{
ObjectStorage.StoreWneStory(wneStoryProgress.SessionId, wneStoryProgress.Filename, value);
}
}
public void OnWneStoryDetect(uint embeddedSessionId)
{
UiJunction?.WneStoryDetect(embeddedSessionId);
}
public void OnWneStoryFail(uint sessionId)
{
UiJunction?.WneStoryError(sessionId);
}
void IWneHandler.OnWneStoryProgress(WneStoryProgress wneStoryProgress)
{
UiJunction?.WneStoryProgress(wneStoryProgress.SessionId, wneStoryProgress.Filename, wneStoryProgress.FileCaught, wneStoryProgress.FileSize);
}
}
}

26
skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs
View File

@ -1788,5 +1788,29 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
RfSpectrumData result = RfSpectrumData.LoadFromStream(fileStream);
return result;
}
}
public bool TestForWneStory(uint sessionId, string filename)
{
string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
FileInfo fi = new FileInfo(outFilename);
return fi.Exists;
}
public void StoreWneStory(uint sessionId, string filename, Stream value)
{
string outFilename = Path.Combine(rootDirectory.FullName, "wne", filename);
FileInfo fi = new FileInfo(outFilename);
fi.Directory.EnsureExists();
if (value.CanSeek)
value.Position = 0;
FileStream fileStream = fi.OpenWrite();
value.CopyTo(fileStream);
fileStream.Flush();
fileStream.Close();
value.Dispose();
}
}
}

12
skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs
View File

@ -168,5 +168,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
{
throw new NotImplementedException();
}
}
public bool TestForWneStory(uint sessionId, string filename)
{
return true;
}
public void StoreWneStory(uint sessionId, string filename, Stream value)
{
throw new NotImplementedException();
}
}
}

4
skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs
View File

@ -45,5 +45,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber);
void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci);
byte[] DvbNipGetFile(string path);
}
bool TestForWneStory(uint sessionId, string filename);
void StoreWneStory(uint sessionId, string filename, Stream value);
}
}

14
skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs
View File

@ -235,6 +235,16 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
byte[] buffer = tarArchive.ReadEntry(filename);
RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer));
return rfSpectrumData;
}
}
}
public bool TestForWneStory(uint sessionId, string filename)
{
throw new NotImplementedException();
}
public void StoreWneStory(uint sessionId, string filename, Stream value)
{
throw new NotImplementedException();
}
}
}