From 46798f4df1e7ad0032a26e45103e6929f4efc9e7 Mon Sep 17 00:00:00 2001 From: feyris-tan Date: Thu, 16 Oct 2025 16:04:01 +0200 Subject: [PATCH] Improved the DVB-NIP via GSE reader by a ton! --- skyscraper8.sln.DotSettings.user | 5 + skyscraper8/GS/BBHeader.cs | 8 +- skyscraper8/GS/BBframeDeencapsulator3.cs | 2 - skyscraper8/GS/GSE-HEM/GseHemReader.cs | 111 ++++++++++++++++++ skyscraper8/GS/GSE-HEM/RayBuffer.cs | 111 ++++++++++++++++++ skyscraper8/GS/GsTypeDetector.cs | 18 +++ skyscraper8/GS/POC/Stid135Test.cs | 3 +- .../Mpeg2/PacketFilter/ScrambleFilter.cs | 7 +- skyscraper8/Program.cs | 2 +- .../Scraper/Storage/NullObjectStorage.cs | 3 +- skyscraper8/log4net.config | 6 +- 11 files changed, 262 insertions(+), 14 deletions(-) create mode 100644 skyscraper8/GS/GSE-HEM/GseHemReader.cs create mode 100644 skyscraper8/GS/GSE-HEM/RayBuffer.cs diff --git a/skyscraper8.sln.DotSettings.user b/skyscraper8.sln.DotSettings.user index ff66e63..296c055 100644 --- a/skyscraper8.sln.DotSettings.user +++ b/skyscraper8.sln.DotSettings.user @@ -1,7 +1,12 @@  ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded <data><HostParameters type="LocalHostParameters" /><Argument type="StandaloneArgument"><Arguments IsNull="False"></Arguments><FileName IsNull="False"></FileName><WorkingDirectory IsNull="False"></WorkingDirectory><Scope><ProcessFilters /></Scope></Argument><Info type="TimelineInfo" /><CoreOptions type="CoreOptions"><CoreTempPath IsNull="False"></CoreTempPath><RemoteEndPoint IsNull="False"></RemoteEndPoint><AdditionalEnvironmentVariables /></CoreOptions><HostOptions type="HostOptions"><HostTempPath IsNull="False"></HostTempPath></HostOptions></data> \ No newline at end of file diff --git a/skyscraper8/GS/BBHeader.cs b/skyscraper8/GS/BBHeader.cs index bd57e77..13228a3 100644 --- a/skyscraper8/GS/BBHeader.cs +++ b/skyscraper8/GS/BBHeader.cs @@ -27,12 +27,12 @@ public class BBHeader : Validatable SyncByte = BbHeaderSpan[6]; SyncD = BbHeaderSpan[7] << 8 | BbHeaderSpan[8]; + SyncD /= 8; - ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0; - Valid = ChecksumValid; + //ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0; + //Valid = ChecksumValid; + Valid = true; } - - public bool ChecksumValid { get; private set; } public int SyncD { get; private set; } diff --git a/skyscraper8/GS/BBframeDeencapsulator3.cs b/skyscraper8/GS/BBframeDeencapsulator3.cs index 33996fa..afbbba8 100644 --- a/skyscraper8/GS/BBframeDeencapsulator3.cs +++ b/skyscraper8/GS/BBframeDeencapsulator3.cs @@ -27,8 +27,6 @@ public class BbframeDeencapsulator3 : IBbframeDeencapsulator numPushed++; BBHeader bbHeader = new BBHeader(bbframe, 1); - if (!bbHeader.ChecksumValid) - return; if (!bbHeader.Valid) return; diff --git a/skyscraper8/GS/GSE-HEM/GseHemReader.cs b/skyscraper8/GS/GSE-HEM/GseHemReader.cs new file mode 100644 index 0000000..364e6b3 --- /dev/null +++ b/skyscraper8/GS/GSE-HEM/GseHemReader.cs @@ -0,0 +1,111 @@ +using log4net; +using skyscraper5.Dvb.DataBroadcasting; +using skyscraper5.Skyscraper.IO; +using skyscraper8.GSE.GSE; + +namespace skyscraper8.GSE.GSE_HEM; + +public class GseHemReader : IMisHandler +{ + public GseHemReader(IMultiprotocolEncapsulationEventHandler mpeEventHandler) + { + rayBuffer = new RayBuffer(); + this.mpeEventHandler = mpeEventHandler; + } + + private IMultiprotocolEncapsulationEventHandler mpeEventHandler; + private GseLabel lastLabel; + private RayBuffer rayBuffer; + private const int BUFFER_THRESHOLD = 65536 * 2; //Twice the maximum PDU size of GSE. That way we can guarantee we have one full PDU in buffer. + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + + public void PushFrame(BBHeader bbHeader, ReadOnlySpan readOnlySpan) + { + MemoryStream ms = new MemoryStream(readOnlySpan.ToArray()); + rayBuffer.Enqueue(ms, bbHeader.SyncD); + + while (rayBuffer.AvailableBytes > BUFFER_THRESHOLD) + { + byte a = rayBuffer.ReadUInt8(); + bool startIndicator = (a & 0x80) != 0; + bool endIndicator = (a & 0x40) != 0; + int labelTypeIndicator = (a & 0x30) >> 4; + if (!startIndicator && !endIndicator) + { + throw new NotImplementedException("I didn't expect a GSE Padding packet in GSE-HEM. Please share a sample of this stream, so I can implement this."); + } + else + { + GsePacket child = new GsePacket(startIndicator, endIndicator, labelTypeIndicator); + + int gseLength = (a & 0x0f) << 8; + gseLength += rayBuffer.ReadUInt8(); + if (!startIndicator || !endIndicator) + { + //According to ETSI TS 102 606-1 V1.2.1, HEM does not fragment packets. So we lost our sync! + rayBuffer.Resync(); + return; + } + + if (startIndicator && !endIndicator) + { + throw new NotImplementedException("I didn't expect an open-ended GSE Packet in GSE-HEM. Please share a sample of this stream, so I can implement this."); + } + + if (startIndicator) + { + child.ProtocolType = rayBuffer.ReadUInt16BE(); + gseLength -= 2; + switch (labelTypeIndicator) + { + case 0: + child.Label = new _6byteLabel(rayBuffer.ReadBytes(6)); + gseLength -= 6; + lastLabel = child.Label; + break; + case 1: + child.Label = new _3byteLabel(rayBuffer.ReadBytes(3)); + gseLength -= 3; + lastLabel = child.Label; + break; + case 2: + child.Label = BroadcastLabel.GetInstance(); + break; + case 3: + child.Label = lastLabel; + break; + default: + throw new NotImplementedException(String.Format("Unknown label type indicator: {0}", labelTypeIndicator)); + } + } + + if (!startIndicator && endIndicator) + gseLength -= 4; + + child.GseDataBytes = rayBuffer.ReadBytes(gseLength); + + if (!startIndicator && endIndicator) + throw new NotImplementedException("I didn't expect a GSE packet with a missing head in GSE-HEM. Please share a sample of this stream, so I can implement this."); + + HandlePacket(child); + } + } + } + + private void HandlePacket(GsePacket packet) + { + if (packet.StartIndicator && packet.EndIndicator) + { + switch (packet.ProtocolType) + { + case 0x0800: + mpeEventHandler.OnIpDatagram(0x010e,packet.GseDataBytes); + return; + default: + logger.WarnFormat("This GSE-HEM stream contains traffic other than IP. IP is type 0x0800, but we got 0x{0:X4} here.", packet.ProtocolType); + return; + } + } + throw new NotImplementedException(packet.ToString()); + } +} \ No newline at end of file diff --git a/skyscraper8/GS/GSE-HEM/RayBuffer.cs b/skyscraper8/GS/GSE-HEM/RayBuffer.cs new file mode 100644 index 0000000..4ddf0b9 --- /dev/null +++ b/skyscraper8/GS/GSE-HEM/RayBuffer.cs @@ -0,0 +1,111 @@ +using skyscraper5.Skyscraper.IO; + +namespace skyscraper8.GSE.GSE_HEM; + +public class RayBuffer +{ + private MemoryStream currentItem; + private Queue> queue; + + public int QueuedItems + { + get + { + int result = 0; + if (currentItem != null) + result++; + if (queue != null) + result += queue.Count; + return result; + } + } + + public void Enqueue(MemoryStream ms, int syncPoint) + { + if (currentItem == null) + { + currentItem = ms; + currentItem.Position = syncPoint; + return; + } + else + { + if (queue == null) + queue = new Queue>(); + queue.Enqueue(new Tuple(ms, syncPoint)); + } + } + + public long AvailableBytes + { + get + { + if (currentItem == null) + return 0; + + long result = currentItem.GetAvailableBytes(); + if (queue != null) + { + result += queue.Select(x => x.Item1.GetAvailableBytes()).Sum(); + } + + return result; + } + } + +public byte ReadUInt8() + { + byte[] tmp = new byte[1]; + if (Read(tmp, 0, 1) != 1) + throw new IOException("ReadUInt8 failed"); + return tmp[0]; + } + + private int Read(byte[] outBuffer, int offset, int count) + { + int result = 0; + while (count > 0) + { + int stepSize = Math.Min(count, (int)currentItem.GetAvailableBytes()); + int stepResult = currentItem.Read(outBuffer, offset, stepSize); + offset += stepResult; + count -= stepResult; + result += stepResult; + if (currentItem.GetAvailableBytes() == 0) + { + currentItem = queue.Dequeue().Item1; + } + } + + return result; + } + + public ushort ReadUInt16BE() + { + byte[] tmp = new byte[2]; + if (Read(tmp, 0, 2) != 2) + throw new IOException("ReadUInt16BE failed"); + if (BitConverter.IsLittleEndian) + (tmp[0], tmp[1]) = (tmp[1], tmp[0]); + return BitConverter.ToUInt16(tmp, 0); + } + + public byte[] ReadBytes(int p0) + { + byte[] tmp = new byte[p0]; + if (Read(tmp, 0, p0) != p0) + throw new IOException(String.Format("Reading {0} bytes failed.", p0)); + return tmp; + } + + public void Resync() + { + Tuple tuple = queue.Dequeue(); + currentItem = tuple.Item1; + currentItem.Position = tuple.Item2; + if (currentItem.Position > currentItem.Length) + { + Resync(); + } + } +} \ No newline at end of file diff --git a/skyscraper8/GS/GsTypeDetector.cs b/skyscraper8/GS/GsTypeDetector.cs index 8c395e8..bd67e56 100644 --- a/skyscraper8/GS/GsTypeDetector.cs +++ b/skyscraper8/GS/GsTypeDetector.cs @@ -1,5 +1,6 @@ using log4net; using skyscraper5.Dvb.DataBroadcasting; +using skyscraper8.GSE.GSE_HEM; using skyscraper8.GSE.GSE; namespace skyscraper8.GSE; @@ -17,8 +18,14 @@ public class GsTypeDetector : IMisHandler } private GseReader gseReader; + private GseHemReader gseHemReader; public void PushFrame(BBHeader bbHeader, ReadOnlySpan readOnlySpan) { + if (readOnlySpan.Length == 0) + { + return; + } + if (bbHeader.TsGs == 1 && bbHeader.SyncByte == 0) { //Looks like Continuous GSE. @@ -31,6 +38,17 @@ public class GsTypeDetector : IMisHandler gseReader.PushFrame(bbHeader, readOnlySpan); return; } + + if (bbHeader.TsGs == 2 && bbHeader.SyncByte == 0) + { + //Looks like GSE-HEM + if (gseHemReader == null) + { + gseHemReader = new GseHemReader(mpeEventHandler); + } + gseHemReader.PushFrame(bbHeader, readOnlySpan); + return; + } //We have no idea what this is. Tuple streamTypeToPost = new Tuple(bbHeader.TsGs, bbHeader.SyncByte); diff --git a/skyscraper8/GS/POC/Stid135Test.cs b/skyscraper8/GS/POC/Stid135Test.cs index 5b6b28e..0ec03c7 100644 --- a/skyscraper8/GS/POC/Stid135Test.cs +++ b/skyscraper8/GS/POC/Stid135Test.cs @@ -1,6 +1,7 @@ using log4net; using skyscraper5.Mpeg2; using skyscraper5.Skyscraper.Scraper; +using skyscraper5.Skyscraper.Scraper.Storage.Filesystem; using skyscraper5.Skyscraper.Scraper.Storage.InMemory; using skyscraper8.Skyscraper.Scraper.Storage; @@ -15,7 +16,7 @@ public class Stid135Test TsContext mpeg2 = new TsContext(); DataStorage dataStorage = new InMemoryScraperStorage(); - ObjectStorage objectStorage = new NullObjectStorage(); + ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("nip")); SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage); mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(skyscraper)); diff --git a/skyscraper8/Mpeg2/PacketFilter/ScrambleFilter.cs b/skyscraper8/Mpeg2/PacketFilter/ScrambleFilter.cs index f452b81..5a51573 100644 --- a/skyscraper8/Mpeg2/PacketFilter/ScrambleFilter.cs +++ b/skyscraper8/Mpeg2/PacketFilter/ScrambleFilter.cs @@ -17,7 +17,12 @@ namespace skyscraper5.src.Mpeg2.PacketFilter { protected override bool PassPacketEx(TsPacket packet) { - return packet.TSC == 0; + if (packet.TSC == 0) + return true; + else + { + return false; + } } } } diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index 09db3be..5a9c125 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -41,7 +41,7 @@ namespace skyscraper5 class Program { private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); - private const int PUBLIC_RELEASE = 9; + private const int PUBLIC_RELEASE = 10; private static void IntegrationTest() { /*List ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList(); diff --git a/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs index 939af83..bad78b9 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs @@ -74,12 +74,11 @@ namespace skyscraper8.Skyscraper.Scraper.Storage public bool DvbNipTestForFile(string announcedFileContentLocation) { - throw new NotImplementedException(); + return true; } public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener) { - throw new NotImplementedException(); } public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot) diff --git a/skyscraper8/log4net.config b/skyscraper8/log4net.config index 9755f52..366341b 100644 --- a/skyscraper8/log4net.config +++ b/skyscraper8/log4net.config @@ -13,15 +13,15 @@ - + + - + --> \ No newline at end of file