Improved the DVB-NIP via GSE reader by a ton!

skyscraper8.sln.DotSettings.user

@@ -1,7 +1,12 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AArray_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F6e_003Fd247db11_003FArray_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F5a_003Fcf76af61_003FFileInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AList_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F6b_003Fa410ee2c_003FList_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMemoryStream_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F0d_003F068af3a6_003FMemoryStream_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ANullable_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F0d_003F6549c49b_003FNullable_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AObject_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F55_003F6efc7017_003FObject_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Fb6_003F498e7c75_003FQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Feb_003F3c476997_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F4f_003F7bfc5050_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATuple_00602_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F9f_003F0d16f921_003FTuple_00602_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/Profiling/Configurations/=1/@EntryIndexedValue">&lt;data&gt;&lt;HostParameters type="LocalHostParameters" /&gt;&lt;Argument type="StandaloneArgument"&gt;&lt;Arguments IsNull="False"&gt;&lt;/Arguments&gt;&lt;FileName IsNull="False"&gt;&lt;/FileName&gt;&lt;WorkingDirectory IsNull="False"&gt;&lt;/WorkingDirectory&gt;&lt;Scope&gt;&lt;ProcessFilters /&gt;&lt;/Scope&gt;&lt;/Argument&gt;&lt;Info type="TimelineInfo" /&gt;&lt;CoreOptions type="CoreOptions"&gt;&lt;CoreTempPath IsNull="False"&gt;&lt;/CoreTempPath&gt;&lt;RemoteEndPoint IsNull="False"&gt;&lt;/RemoteEndPoint&gt;&lt;AdditionalEnvironmentVariables /&gt;&lt;/CoreOptions&gt;&lt;HostOptions type="HostOptions"&gt;&lt;HostTempPath IsNull="False"&gt;&lt;/HostTempPath&gt;&lt;/HostOptions&gt;&lt;/data&gt;</s:String></wpf:ResourceDictionary>

skyscraper8/GS/BBHeader.cs

@@ -27,13 +27,13 @@ public class BBHeader : Validatable
         SyncByte = BbHeaderSpan[6];
         
         SyncD = BbHeaderSpan[7] << 8  | BbHeaderSpan[8];
+        SyncD /= 8;
 
-        ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
-        Valid = ChecksumValid;
+        //ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
+        //Valid = ChecksumValid;
+        Valid = true;
     }
     
-    public bool ChecksumValid { get; private set; }
-    
     public int SyncD { get; private set; }
 
     public byte SyncByte { get; private set; }

skyscraper8/GS/BBframeDeencapsulator3.cs

@@ -27,8 +27,6 @@ public class BbframeDeencapsulator3 : IBbframeDeencapsulator
         numPushed++;
 
         BBHeader bbHeader = new BBHeader(bbframe, 1);
-        if (!bbHeader.ChecksumValid)
-            return;
         if (!bbHeader.Valid)
             return;
 

skyscraper8/GS/GSE-HEM/GseHemReader.cs

@@ -0,0 +1,111 @@
+using log4net;
+using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper5.Skyscraper.IO;
+using skyscraper8.GSE.GSE;
+
+namespace skyscraper8.GSE.GSE_HEM;
+
+public class GseHemReader : IMisHandler
+{
+    public GseHemReader(IMultiprotocolEncapsulationEventHandler mpeEventHandler)
+    {
+        rayBuffer = new RayBuffer();
+        this.mpeEventHandler = mpeEventHandler;
+    }
+
+    private IMultiprotocolEncapsulationEventHandler mpeEventHandler;
+    private GseLabel lastLabel;
+    private RayBuffer rayBuffer;
+    private const int BUFFER_THRESHOLD = 65536 * 2; //Twice the maximum PDU size of GSE. That way we can guarantee we have one full PDU in buffer.
+    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+    
+    public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
+    {
+        MemoryStream ms = new MemoryStream(readOnlySpan.ToArray());
+        rayBuffer.Enqueue(ms, bbHeader.SyncD);
+
+        while (rayBuffer.AvailableBytes > BUFFER_THRESHOLD)
+        {
+            byte a = rayBuffer.ReadUInt8();
+            bool startIndicator = (a & 0x80) != 0;
+            bool endIndicator = (a & 0x40) != 0;
+            int labelTypeIndicator = (a & 0x30) >> 4;
+            if (!startIndicator && !endIndicator)
+            {
+                throw new NotImplementedException("I didn't expect a GSE Padding packet in GSE-HEM. Please share a sample of this stream, so I can implement this.");
+            }
+            else
+            {
+                GsePacket child = new GsePacket(startIndicator, endIndicator, labelTypeIndicator);
+                
+                int gseLength = (a & 0x0f) << 8;
+                gseLength += rayBuffer.ReadUInt8();
+                if (!startIndicator || !endIndicator)
+                {
+                    //According to ETSI TS 102 606-1 V1.2.1, HEM does not fragment packets. So we lost our sync!
+                    rayBuffer.Resync();
+                    return;
+                }
+
+                if (startIndicator && !endIndicator)
+                {
+                    throw new NotImplementedException("I didn't expect an open-ended GSE Packet in GSE-HEM. Please share a sample of this stream, so I can implement this.");
+                }
+
+                if (startIndicator)
+                {
+                    child.ProtocolType = rayBuffer.ReadUInt16BE();
+                    gseLength -= 2;
+                    switch (labelTypeIndicator)
+                    {
+                        case 0:
+                            child.Label = new _6byteLabel(rayBuffer.ReadBytes(6));
+                            gseLength -= 6;
+                            lastLabel = child.Label;
+                            break;
+                        case 1:
+                            child.Label = new _3byteLabel(rayBuffer.ReadBytes(3));
+                            gseLength -= 3;
+                            lastLabel = child.Label;
+                            break;
+                        case 2:
+                            child.Label = BroadcastLabel.GetInstance();
+                            break;
+                        case 3:
+                            child.Label = lastLabel;
+                            break;
+                        default:
+                            throw new NotImplementedException(String.Format("Unknown label type indicator: {0}", labelTypeIndicator));
+                    }
+                }
+
+                if (!startIndicator && endIndicator)
+                    gseLength -= 4;
+                
+                child.GseDataBytes = rayBuffer.ReadBytes(gseLength);
+                
+                if (!startIndicator && endIndicator)
+                    throw new NotImplementedException("I didn't expect a GSE packet with a missing head in GSE-HEM.  Please share a sample of this stream, so I can implement this.");
+
+                HandlePacket(child);
+            }
+        }
+    }
+
+    private void HandlePacket(GsePacket packet)
+    {
+        if (packet.StartIndicator && packet.EndIndicator)
+        {
+            switch (packet.ProtocolType)
+            {
+                case 0x0800:
+                    mpeEventHandler.OnIpDatagram(0x010e,packet.GseDataBytes);
+                    return;
+                default:
+                    logger.WarnFormat("This GSE-HEM stream contains traffic other than IP. IP is type 0x0800, but we got 0x{0:X4} here.", packet.ProtocolType);
+                    return;
+            }
+        }
+        throw new NotImplementedException(packet.ToString());
+    }
+}

skyscraper8/GS/GSE-HEM/RayBuffer.cs

@@ -0,0 +1,111 @@
+using skyscraper5.Skyscraper.IO;
+
+namespace skyscraper8.GSE.GSE_HEM;
+
+public class RayBuffer
+{
+    private MemoryStream currentItem;
+    private Queue<Tuple<MemoryStream, int>> queue;
+
+    public int QueuedItems
+    {
+        get
+        {
+            int result = 0;
+            if (currentItem != null)
+                result++;
+            if (queue != null)
+                result += queue.Count;
+            return result;
+        }
+    }
+
+    public void Enqueue(MemoryStream ms, int syncPoint)
+    {
+        if (currentItem == null)
+        {
+            currentItem = ms;
+            currentItem.Position = syncPoint;
+            return;
+        }
+        else
+        {
+            if (queue == null)
+                queue = new Queue<Tuple<MemoryStream, int>>();
+            queue.Enqueue(new Tuple<MemoryStream, int>(ms, syncPoint));
+        }
+    }
+
+    public long AvailableBytes 
+    { 
+        get
+        {
+            if (currentItem == null)
+                return 0;
+
+            long result = currentItem.GetAvailableBytes();
+            if (queue != null)
+            {
+                result += queue.Select(x => x.Item1.GetAvailableBytes()).Sum();
+            }
+
+            return result;
+        }
+    }
+
+public byte ReadUInt8()
+    {
+        byte[] tmp = new byte[1];
+        if (Read(tmp, 0, 1) != 1)
+            throw new IOException("ReadUInt8 failed");
+        return tmp[0];
+    }
+
+    private int Read(byte[] outBuffer, int offset, int count)
+    {
+        int result = 0;
+        while (count > 0)
+        {
+            int stepSize = Math.Min(count, (int)currentItem.GetAvailableBytes());
+            int stepResult = currentItem.Read(outBuffer, offset, stepSize);
+            offset += stepResult;
+            count -= stepResult;
+            result += stepResult;
+            if (currentItem.GetAvailableBytes() == 0)
+            {
+                currentItem = queue.Dequeue().Item1;
+            }
+        }
+
+        return result;
+    }
+
+    public ushort ReadUInt16BE()
+    {
+        byte[] tmp = new byte[2];
+        if (Read(tmp, 0, 2) != 2)
+            throw new IOException("ReadUInt16BE failed");
+        if (BitConverter.IsLittleEndian)
+            (tmp[0], tmp[1]) = (tmp[1], tmp[0]);
+        return BitConverter.ToUInt16(tmp, 0);
+    }
+
+    public byte[] ReadBytes(int p0)
+    {
+        byte[] tmp = new byte[p0];
+        if (Read(tmp, 0, p0) != p0)
+            throw new IOException(String.Format("Reading {0} bytes failed.", p0));
+        return tmp;
+    }
+
+    public void Resync()
+    {
+        Tuple<MemoryStream, int> tuple = queue.Dequeue();
+        currentItem = tuple.Item1;
+        currentItem.Position = tuple.Item2;
+        if (currentItem.Position > currentItem.Length)
+        {
+            Resync();
+        }
+    }
+}

skyscraper8/GS/GsTypeDetector.cs

@@ -1,5 +1,6 @@
 using log4net;
 using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper8.GSE.GSE_HEM;
 using skyscraper8.GSE.GSE;
 
 namespace skyscraper8.GSE;
@@ -17,8 +18,14 @@ public class GsTypeDetector : IMisHandler
     }
 
     private GseReader gseReader;
+    private GseHemReader gseHemReader;
     public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
     {
+        if (readOnlySpan.Length == 0)
+        {
+            return;
+        }
+        
         if (bbHeader.TsGs == 1 && bbHeader.SyncByte == 0)
         {
             //Looks like Continuous GSE.
@@ -32,6 +39,17 @@ public class GsTypeDetector : IMisHandler
             return;
         }
 
+        if (bbHeader.TsGs == 2 && bbHeader.SyncByte == 0)
+        {
+            //Looks like GSE-HEM
+            if (gseHemReader == null)
+            {
+                gseHemReader = new GseHemReader(mpeEventHandler);
+            }
+            gseHemReader.PushFrame(bbHeader, readOnlySpan);
+            return;
+        }
+        
         //We have no idea what this is.
         Tuple<int, byte> streamTypeToPost = new Tuple<int, byte>(bbHeader.TsGs, bbHeader.SyncByte);
         if (_postedStreamTypes == null)

skyscraper8/GS/POC/Stid135Test.cs

@@ -1,6 +1,7 @@
 using log4net;
 using skyscraper5.Mpeg2;
 using skyscraper5.Skyscraper.Scraper;
+using skyscraper5.Skyscraper.Scraper.Storage.Filesystem;
 using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
 using skyscraper8.Skyscraper.Scraper.Storage;
 
@@ -15,7 +16,7 @@ public class Stid135Test
 
         TsContext mpeg2 = new TsContext();
         DataStorage dataStorage = new InMemoryScraperStorage();
-        ObjectStorage objectStorage = new NullObjectStorage();
+        ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("nip"));
         SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
 
         mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(skyscraper));

skyscraper8/Mpeg2/PacketFilter/ScrambleFilter.cs

@@ -17,7 +17,12 @@ namespace skyscraper5.src.Mpeg2.PacketFilter
 	{
 		protected override bool PassPacketEx(TsPacket packet)
 		{
-			return packet.TSC == 0;
+			if (packet.TSC == 0)
+				return true;
+			else
+			{
+				return false;
+			}
 		}
 	}
 }

skyscraper8/Program.cs

@@ -41,7 +41,7 @@ namespace skyscraper5
     class Program
     {
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-		private const int PUBLIC_RELEASE = 9;
+		private const int PUBLIC_RELEASE = 10;
 		private static void IntegrationTest()
 		{
             /*List<SsdpDevice> ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList();

skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs

@@ -74,12 +74,11 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 
 		public bool DvbNipTestForFile(string announcedFileContentLocation)
 		{
-			throw new NotImplementedException();
+			return true;
 		}
 
 		public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener)
 		{
-			throw new NotImplementedException();
 		}
 
 		public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot)

skyscraper8/log4net.config

@@ -15,13 +15,13 @@
 	</appender>
 	
 	<!--a file appender for all logs-->
+	<!--
 	<appender name="all_logs_file" type="log4net.Appender.FileAppender">
-		<!--specifying the file-->
 		<file value="c:\\logs\\all.log" />
-		<!--specifying the displayed layout-->
 		<layout type="log4net.Layout.PatternLayout">
 			<conversionPattern value="%date %level %logger - %message%newline" />
 		</layout>
 	</appender>
+	-->
 	
 </log4net>