Merge pull request 'origin/master' (#3) from origin/master into master

Reviewed-on: #3

skyscraper8.Manual/skyscraper8.Manual.tex

@@ -177,6 +177,15 @@ This document was typeset in \LaTeX{}, using the TeX Live distribution, and whil
 
 \subsection{Personal remarks and some useless bonus information}
 
+\subsubsection{How and why skyscraper8 began: A young man's dream}
+Ever since I was a child, I was fascinated by TV. But not actually watching it, rather understanding how it works. I grew up in the 90s, in an area where Cable TV was not really available, and terristial reception (yeah, those TVs with bunny ears!) only worked when it felt like it. So satellite was the best way to get my childish fix of cartoons! We can say the dish on my roof has always been a companion. Far more interesting than actually watching TV was scanning through the printed TV guides. I was curious. How did they make these? And how would I make one myself? I wondered. Of course my parents wouldn't know - they're not techies. \\
+
+Of course, as I got older, I eventually moved onto other things. Like programming, trading cards, and video games. But just like with the TV, I eventually got more interested in how these games work, instead of actually playing them. \\
+
+I got internet access much later in my life than my peers did, and when I found out that there are forums in which people discuss the technicalities about video games, it blew my mind. I used to be a frequent reader (and occasional contributor) of the XeNTaX\footnote{Unfortunately, that forum is long gone from the internet, but can still be experienced thanks to the amazing work of the people over at the Internet Archive: \url{https://web.archive.org/web/20230925120533/https://forum.xentax.com/ }} forum, from where I learned a lot! \\
+
+At the time, my circle of friends was crazy about Yu-Gi-Oh! Online 3, which was an excellent simulation of the trading card game by the same name.
+
 \subsubsection{Music!}
 Like a lot of programmers, I do enjoy listening to music while working. Some programmers even put song references in their software. Like how MKVToolnix' version names are actually song names, or how BSD developer fk even put \href{https://www.fabiankeil.de/nutzloseinfos.html}{a list of albums on his website} listing what albums he listened to while making it. Although this is absolutely useless, I'd liketo do this as well. Therefore, here follows a list of musical albums I listened to while developing skyscraper8 - no claim to completeness.
 

skyscraper8.sln.DotSettings.user

@@ -1,7 +1,12 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AArray_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F6e_003Fd247db11_003FArray_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F5a_003Fcf76af61_003FFileInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AList_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F6b_003Fa410ee2c_003FList_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMemoryStream_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F0d_003F068af3a6_003FMemoryStream_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ANullable_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F0d_003F6549c49b_003FNullable_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AObject_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F55_003F6efc7017_003FObject_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Fb6_003F498e7c75_003FQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Feb_003F3c476997_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F4f_003F7bfc5050_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATuple_00602_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F9f_003F0d16f921_003FTuple_00602_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/Profiling/Configurations/=1/@EntryIndexedValue">&lt;data&gt;&lt;HostParameters type="LocalHostParameters" /&gt;&lt;Argument type="StandaloneArgument"&gt;&lt;Arguments IsNull="False"&gt;&lt;/Arguments&gt;&lt;FileName IsNull="False"&gt;&lt;/FileName&gt;&lt;WorkingDirectory IsNull="False"&gt;&lt;/WorkingDirectory&gt;&lt;Scope&gt;&lt;ProcessFilters /&gt;&lt;/Scope&gt;&lt;/Argument&gt;&lt;Info type="TimelineInfo" /&gt;&lt;CoreOptions type="CoreOptions"&gt;&lt;CoreTempPath IsNull="False"&gt;&lt;/CoreTempPath&gt;&lt;RemoteEndPoint IsNull="False"&gt;&lt;/RemoteEndPoint&gt;&lt;AdditionalEnvironmentVariables /&gt;&lt;/CoreOptions&gt;&lt;HostOptions type="HostOptions"&gt;&lt;HostTempPath IsNull="False"&gt;&lt;/HostTempPath&gt;&lt;/HostOptions&gt;&lt;/data&gt;</s:String></wpf:ResourceDictionary>

skyscraper8/GS/BBHeader.cs

@@ -27,12 +27,12 @@ public class BBHeader : Validatable
         SyncByte = BbHeaderSpan[6];
         
         SyncD = BbHeaderSpan[7] << 8  | BbHeaderSpan[8];
+        SyncD /= 8;
 
-        ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
-        Valid = ChecksumValid;
+        //ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
+        //Valid = ChecksumValid;
+        Valid = true;
     }
-
-    public bool ChecksumValid { get; private set; }
     
     public int SyncD { get; private set; }
 

skyscraper8/GS/BBframeDeencapsulator3.cs

@@ -27,8 +27,6 @@ public class BbframeDeencapsulator3 : IBbframeDeencapsulator
         numPushed++;
 
         BBHeader bbHeader = new BBHeader(bbframe, 1);
-        if (!bbHeader.ChecksumValid)
-            return;
         if (!bbHeader.Valid)
             return;
 

skyscraper8/GS/GSE-HEM/GseHemReader.cs

@@ -0,0 +1,111 @@
+using log4net;
+using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper5.Skyscraper.IO;
+using skyscraper8.GSE.GSE;
+
+namespace skyscraper8.GSE.GSE_HEM;
+
+public class GseHemReader : IMisHandler
+{
+    public GseHemReader(IMultiprotocolEncapsulationEventHandler mpeEventHandler)
+    {
+        rayBuffer = new RayBuffer();
+        this.mpeEventHandler = mpeEventHandler;
+    }
+
+    private IMultiprotocolEncapsulationEventHandler mpeEventHandler;
+    private GseLabel lastLabel;
+    private RayBuffer rayBuffer;
+    private const int BUFFER_THRESHOLD = 65536 * 2; //Twice the maximum PDU size of GSE. That way we can guarantee we have one full PDU in buffer.
+    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+    
+    public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
+    {
+        MemoryStream ms = new MemoryStream(readOnlySpan.ToArray());
+        rayBuffer.Enqueue(ms, bbHeader.SyncD);
+
+        while (rayBuffer.AvailableBytes > BUFFER_THRESHOLD)
+        {
+            byte a = rayBuffer.ReadUInt8();
+            bool startIndicator = (a & 0x80) != 0;
+            bool endIndicator = (a & 0x40) != 0;
+            int labelTypeIndicator = (a & 0x30) >> 4;
+            if (!startIndicator && !endIndicator)
+            {
+                throw new NotImplementedException("I didn't expect a GSE Padding packet in GSE-HEM. Please share a sample of this stream, so I can implement this.");
+            }
+            else
+            {
+                GsePacket child = new GsePacket(startIndicator, endIndicator, labelTypeIndicator);
+                
+                int gseLength = (a & 0x0f) << 8;
+                gseLength += rayBuffer.ReadUInt8();
+                if (!startIndicator || !endIndicator)
+                {
+                    //According to ETSI TS 102 606-1 V1.2.1, HEM does not fragment packets. So we lost our sync!
+                    rayBuffer.Resync();
+                    return;
+                }
+
+                if (startIndicator && !endIndicator)
+                {
+                    throw new NotImplementedException("I didn't expect an open-ended GSE Packet in GSE-HEM. Please share a sample of this stream, so I can implement this.");
+                }
+
+                if (startIndicator)
+                {
+                    child.ProtocolType = rayBuffer.ReadUInt16BE();
+                    gseLength -= 2;
+                    switch (labelTypeIndicator)
+                    {
+                        case 0:
+                            child.Label = new _6byteLabel(rayBuffer.ReadBytes(6));
+                            gseLength -= 6;
+                            lastLabel = child.Label;
+                            break;
+                        case 1:
+                            child.Label = new _3byteLabel(rayBuffer.ReadBytes(3));
+                            gseLength -= 3;
+                            lastLabel = child.Label;
+                            break;
+                        case 2:
+                            child.Label = BroadcastLabel.GetInstance();
+                            break;
+                        case 3:
+                            child.Label = lastLabel;
+                            break;
+                        default:
+                            throw new NotImplementedException(String.Format("Unknown label type indicator: {0}", labelTypeIndicator));
+                    }
+                }
+
+                if (!startIndicator && endIndicator)
+                    gseLength -= 4;
+                
+                child.GseDataBytes = rayBuffer.ReadBytes(gseLength);
+                
+                if (!startIndicator && endIndicator)
+                    throw new NotImplementedException("I didn't expect a GSE packet with a missing head in GSE-HEM.  Please share a sample of this stream, so I can implement this.");
+
+                HandlePacket(child);
+            }
+        }
+    }
+
+    private void HandlePacket(GsePacket packet)
+    {
+        if (packet.StartIndicator && packet.EndIndicator)
+        {
+            switch (packet.ProtocolType)
+            {
+                case 0x0800:
+                    mpeEventHandler.OnIpDatagram(0x010e,packet.GseDataBytes);
+                    return;
+                default:
+                    logger.WarnFormat("This GSE-HEM stream contains traffic other than IP. IP is type 0x0800, but we got 0x{0:X4} here.", packet.ProtocolType);
+                    return;
+            }
+        }
+        throw new NotImplementedException(packet.ToString());
+    }
+}

skyscraper8/GS/GSE-HEM/RayBuffer.cs

@@ -0,0 +1,111 @@
+using skyscraper5.Skyscraper.IO;
+
+namespace skyscraper8.GSE.GSE_HEM;
+
+public class RayBuffer
+{
+    private MemoryStream currentItem;
+    private Queue<Tuple<MemoryStream, int>> queue;
+
+    public int QueuedItems
+    {
+        get
+        {
+            int result = 0;
+            if (currentItem != null)
+                result++;
+            if (queue != null)
+                result += queue.Count;
+            return result;
+        }
+    }
+
+    public void Enqueue(MemoryStream ms, int syncPoint)
+    {
+        if (currentItem == null)
+        {
+            currentItem = ms;
+            currentItem.Position = syncPoint;
+            return;
+        }
+        else
+        {
+            if (queue == null)
+                queue = new Queue<Tuple<MemoryStream, int>>();
+            queue.Enqueue(new Tuple<MemoryStream, int>(ms, syncPoint));
+        }
+    }
+
+    public long AvailableBytes 
+    { 
+        get
+        {
+            if (currentItem == null)
+                return 0;
+
+            long result = currentItem.GetAvailableBytes();
+            if (queue != null)
+            {
+                result += queue.Select(x => x.Item1.GetAvailableBytes()).Sum();
+            }
+
+            return result;
+        }
+    }
+
+public byte ReadUInt8()
+    {
+        byte[] tmp = new byte[1];
+        if (Read(tmp, 0, 1) != 1)
+            throw new IOException("ReadUInt8 failed");
+        return tmp[0];
+    }
+
+    private int Read(byte[] outBuffer, int offset, int count)
+    {
+        int result = 0;
+        while (count > 0)
+        {
+            int stepSize = Math.Min(count, (int)currentItem.GetAvailableBytes());
+            int stepResult = currentItem.Read(outBuffer, offset, stepSize);
+            offset += stepResult;
+            count -= stepResult;
+            result += stepResult;
+            if (currentItem.GetAvailableBytes() == 0)
+            {
+                currentItem = queue.Dequeue().Item1;
+            }
+        }
+
+        return result;
+    }
+
+    public ushort ReadUInt16BE()
+    {
+        byte[] tmp = new byte[2];
+        if (Read(tmp, 0, 2) != 2)
+            throw new IOException("ReadUInt16BE failed");
+        if (BitConverter.IsLittleEndian)
+            (tmp[0], tmp[1]) = (tmp[1], tmp[0]);
+        return BitConverter.ToUInt16(tmp, 0);
+    }
+
+    public byte[] ReadBytes(int p0)
+    {
+        byte[] tmp = new byte[p0];
+        if (Read(tmp, 0, p0) != p0)
+            throw new IOException(String.Format("Reading {0} bytes failed.", p0));
+        return tmp;
+    }
+
+    public void Resync()
+    {
+        Tuple<MemoryStream, int> tuple = queue.Dequeue();
+        currentItem = tuple.Item1;
+        currentItem.Position = tuple.Item2;
+        if (currentItem.Position > currentItem.Length)
+        {
+            Resync();
+        }
+    }
+}

skyscraper8/GS/GsTypeDetector.cs

@@ -1,5 +1,6 @@
 using log4net;
 using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper8.GSE.GSE_HEM;
 using skyscraper8.GSE.GSE;
 
 namespace skyscraper8.GSE;
@@ -17,8 +18,14 @@ public class GsTypeDetector : IMisHandler
     }
 
     private GseReader gseReader;
+    private GseHemReader gseHemReader;
     public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
     {
+        if (readOnlySpan.Length == 0)
+        {
+            return;
+        }
+        
         if (bbHeader.TsGs == 1 && bbHeader.SyncByte == 0)
         {
             //Looks like Continuous GSE.
@@ -31,6 +38,17 @@ public class GsTypeDetector : IMisHandler
             gseReader.PushFrame(bbHeader, readOnlySpan);
             return;
         }
+
+        if (bbHeader.TsGs == 2 && bbHeader.SyncByte == 0)
+        {
+            //Looks like GSE-HEM
+            if (gseHemReader == null)
+            {
+                gseHemReader = new GseHemReader(mpeEventHandler);
+            }
+            gseHemReader.PushFrame(bbHeader, readOnlySpan);
+            return;
+        }
         
         //We have no idea what this is.
         Tuple<int, byte> streamTypeToPost = new Tuple<int, byte>(bbHeader.TsGs, bbHeader.SyncByte);

skyscraper8/GS/POC/Stid135Test.cs

@@ -1,6 +1,7 @@
 using log4net;
 using skyscraper5.Mpeg2;
 using skyscraper5.Skyscraper.Scraper;
+using skyscraper5.Skyscraper.Scraper.Storage.Filesystem;
 using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
 using skyscraper8.Skyscraper.Scraper.Storage;
 
@@ -15,7 +16,7 @@ public class Stid135Test
 
         TsContext mpeg2 = new TsContext();
         DataStorage dataStorage = new InMemoryScraperStorage();
-        ObjectStorage objectStorage = new NullObjectStorage();
+        ObjectStorage objectStorage = new FilesystemStorage(new DirectoryInfo("nip"));
         SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
 
         mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(skyscraper));

skyscraper8/Mpeg2/PacketFilter/ScrambleFilter.cs

@@ -17,7 +17,12 @@ namespace skyscraper5.src.Mpeg2.PacketFilter
 	{
 		protected override bool PassPacketEx(TsPacket packet)
 		{
-			return packet.TSC == 0;
+			if (packet.TSC == 0)
+				return true;
+			else
+			{
+				return false;
+			}
 		}
 	}
 }

skyscraper8/Program.cs

@@ -41,7 +41,7 @@ namespace skyscraper5
     class Program
     {
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-		private const int PUBLIC_RELEASE = 9;
+		private const int PUBLIC_RELEASE = 10;
 		private static void IntegrationTest()
 		{
             /*List<SsdpDevice> ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList();

skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs

@@ -74,12 +74,11 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 
 		public bool DvbNipTestForFile(string announcedFileContentLocation)
 		{
-			throw new NotImplementedException();
+			return true;
 		}
 
 		public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener)
 		{
-			throw new NotImplementedException();
 		}
 
 		public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot)

skyscraper8/log4net.config

@@ -13,15 +13,15 @@
 			<conversionPattern value="%date %level %logger - %message%newline" />
 		</layout>
 	</appender>
-
+	
 	<!--a file appender for all logs-->
+	<!--
 	<appender name="all_logs_file" type="log4net.Appender.FileAppender">
-		<!--specifying the file-->
 		<file value="c:\\logs\\all.log" />
-		<!--specifying the displayed layout-->
 		<layout type="log4net.Layout.PatternLayout">
 			<conversionPattern value="%date %level %logger - %message%newline" />
 		</layout>
 	</appender>
+	-->
 	
 </log4net>