BBframeDeencapsulator version 4 will hopefully save the day, and not break CI again.

.gitea/workflows/demo.yaml

@@ -33,7 +33,8 @@ jobs:
         uses: akkuman/gitea-release-action@v1
         with:
           name: ${{ gitea.sha }}
-          md5sum: true
-          sha256sum: true
+          md5sum: false
+          sha256sum: false
+          tag_name: release_tag
           files: |-
             ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip

skyscraper8.sln.DotSettings.user

@@ -1,4 +1,7 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AArray_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F6e_003Fd247db11_003FArray_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F5a_003Fcf76af61_003FFileInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMemoryStream_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F0d_003F068af3a6_003FMemoryStream_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Feb_003F3c476997_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F4f_003F7bfc5050_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/Profiling/Configurations/=1/@EntryIndexedValue">&lt;data&gt;&lt;HostParameters type="LocalHostParameters" /&gt;&lt;Argument type="StandaloneArgument"&gt;&lt;Arguments IsNull="False"&gt;&lt;/Arguments&gt;&lt;FileName IsNull="False"&gt;&lt;/FileName&gt;&lt;WorkingDirectory IsNull="False"&gt;&lt;/WorkingDirectory&gt;&lt;Scope&gt;&lt;ProcessFilters /&gt;&lt;/Scope&gt;&lt;/Argument&gt;&lt;Info type="TimelineInfo" /&gt;&lt;CoreOptions type="CoreOptions"&gt;&lt;CoreTempPath IsNull="False"&gt;&lt;/CoreTempPath&gt;&lt;RemoteEndPoint IsNull="False"&gt;&lt;/RemoteEndPoint&gt;&lt;AdditionalEnvironmentVariables /&gt;&lt;/CoreOptions&gt;&lt;HostOptions type="HostOptions"&gt;&lt;HostTempPath IsNull="False"&gt;&lt;/HostTempPath&gt;&lt;/HostOptions&gt;&lt;/data&gt;</s:String></wpf:ResourceDictionary>

skyscraper8/Dvb/DataBroadcasting/MultiprotocolEncapsulationEventHandler.cs

@@ -7,7 +7,7 @@ using skyscraper5.Ietf.Rfc971;
 
 namespace skyscraper5.Dvb.DataBroadcasting
 {
-    interface IMultiprotocolEncapsulationEventHandler
+    public interface IMultiprotocolEncapsulationEventHandler
     {
         void OnIpv4PacketArrival(InternetHeader internetHeader, byte[] ipv4Packet);
         void OnIpDatagram(int sourcePid, byte[] payload);

skyscraper8/GS/BBframeDeencapsulator3.cs

@@ -1,9 +1,16 @@
 using log4net;
+using skyscraper5.Dvb.DataBroadcasting;
 
 namespace skyscraper8.GSE;
 
 public class BbframeDeencapsulator3 : IBbframeDeencapsulator
 {
+    public BbframeDeencapsulator3(IMultiprotocolEncapsulationEventHandler mpeEventHandler)
+    {
+        _mpeEventHandler = mpeEventHandler;
+    }
+
+    private readonly IMultiprotocolEncapsulationEventHandler _mpeEventHandler;
     private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
     private long numPushed;
     public void PushPacket(byte[] bbframe)
@@ -30,7 +37,10 @@ public class BbframeDeencapsulator3 : IBbframeDeencapsulator
         if (mis[bbHeader.Matype2] == null)
         {
             logger.InfoFormat("Found a stream on MIS {0}",bbHeader.Matype2);
-            mis[bbHeader.Matype2] = new GsTypeDetector();
+            mis[bbHeader.Matype2] = new GsTypeDetector(bbHeader.Matype2)
+            {
+                mpeEventHandler = this._mpeEventHandler
+            };
         }
 
         mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11));

skyscraper8/GS/GSE/GseFragmentation.cs

@@ -0,0 +1,44 @@
+namespace skyscraper8.GSE.GSE;
+
+public class GseFragmentation
+{
+    public GseFragmentation(GsePacket child)
+    {
+        this.ProtocolType = child.ProtocolType.Value;
+        this.Label = child.Label;
+        AddFragement(child);
+    }
+
+    private List<byte[]> fragments;
+
+    public void AddFragement(GsePacket packet)
+    {
+        if (fragments == null)
+            fragments = new List<byte[]>();
+        
+        fragments.Add(packet.GseDataBytes);
+    }
+    
+    public GseLabel Label { get; private set; }
+
+    public ushort ProtocolType { get; private set; }
+    
+    public bool Validate()
+    {
+        //To be implemented...
+        return true;
+    }
+
+    public byte[] GetGseDataBytes()
+    {
+        int totalLength = fragments.Select(x => x.Length).Sum();
+        byte[] buffer = new byte[totalLength];
+        int offset = 0;
+        for (int i = 0; i < fragments.Count; i++)
+        {
+            Array.Copy(fragments[i],0,buffer,offset,fragments[i].Length);
+            offset += fragments[i].Length;
+        }
+        return buffer;
+    }
+}

skyscraper8/GS/GSE/GseLabel.cs

@@ -0,0 +1,65 @@
+using System.Net.NetworkInformation;
+
+namespace skyscraper8.GSE.GSE;
+
+public abstract class GseLabel
+{
+    public abstract int Length { get; }
+    public abstract string ToString();
+}
+
+public class _6byteLabel : GseLabel
+{
+    public _6byteLabel(byte[] buffer)
+    {
+        MAC = new PhysicalAddress(buffer);
+    }
+    
+    public PhysicalAddress MAC { get; private set; }
+
+    override public string ToString()
+    {
+        return MAC.ToString();
+    }
+
+    public override int Length => 6;
+}
+
+public class _3byteLabel : GseLabel
+{
+    public _3byteLabel(byte[] buffer)
+    {
+        Label = buffer;
+    }
+    
+    public byte[] Label { get; private set; }
+
+    override public string ToString()
+    {
+        return BitConverter.ToString(Label);
+    }
+    
+    override public int Length => 3;
+}
+
+public class BroadcastLabel : GseLabel
+{
+    private BroadcastLabel() {}
+
+    override public string ToString()
+    {
+        return "<Broadcast>";
+    }
+    
+    override public int Length => 0;
+
+    private static BroadcastLabel _instance;
+    public static BroadcastLabel GetInstance()
+    {
+        if (_instance == null)
+        {
+            _instance = new BroadcastLabel();
+        }
+        return _instance;
+    }
+}

skyscraper8/GS/GSE/GsePacket.cs

@@ -0,0 +1,27 @@
+namespace skyscraper8.GSE.GSE;
+
+public class GsePacket
+{
+    public bool StartIndicator { get; }
+    public bool EndIndicator { get; }
+    public int LabelTypeIndicator { get; }
+    public byte? FragmentId { get; set; }
+    public ushort? TotalLength { get; set; }
+    public ushort? ProtocolType { get; set; }
+    public GseLabel Label { get; set; }
+    public byte[] GseDataBytes { get; set; }
+    public uint Crc32 { get; set; }
+
+    public GsePacket(bool startIndicator, bool endIndicator, int labelTypeIndicator)
+    {
+        StartIndicator = startIndicator;
+        EndIndicator = endIndicator;
+        LabelTypeIndicator = labelTypeIndicator;
+    }
+
+    public override string ToString()
+    {
+        return
+            $"{nameof(StartIndicator)}: {StartIndicator}, {nameof(EndIndicator)}: {EndIndicator}, {nameof(LabelTypeIndicator)}: {LabelTypeIndicator}, {nameof(FragmentId)}: {FragmentId}, {nameof(TotalLength)}: {TotalLength}, {nameof(ProtocolType)}: {ProtocolType}, {nameof(Label)}: {Label}";
+    }
+}

skyscraper8/GS/GSE/GseReader.cs

@@ -0,0 +1,145 @@
+using log4net;
+using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper5.Skyscraper.IO;
+
+namespace skyscraper8.GSE.GSE;
+
+internal class GseReader : IMisHandler
+{
+    private GseLabel lastLabel;
+    public IMultiprotocolEncapsulationEventHandler mpeEventHandler { get; set; }
+    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+    public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
+    {
+        MemoryStream ms = new MemoryStream(readOnlySpan.ToArray());
+
+        while (ms.Position < ms.Length)
+        {
+            byte a = ms.ReadUInt8();
+            bool startIndicator = (a & 0x80) != 0;
+            bool endIndicator = (a & 0x40) != 0;
+            int labelTypeIndicator = (a & 0x30) >> 4;
+            if (!startIndicator && !endIndicator && labelTypeIndicator == 0)
+            {
+                //end of BBFrame
+                return;
+            }
+            else
+            {
+                int gseLength = (a & 0x0f);
+                gseLength <<= 8;
+                gseLength += ms.ReadUInt8();
+
+                GsePacket child = new GsePacket(startIndicator, endIndicator, labelTypeIndicator);
+                if (!startIndicator || !endIndicator)
+                {
+                    child.FragmentId = ms.ReadUInt8();
+                    gseLength--;
+                }
+
+                if (startIndicator && !endIndicator)
+                {
+                    child.TotalLength = ms.ReadUInt16BE();
+                    gseLength -= 2;
+                }
+
+                if (startIndicator)
+                {
+                    child.ProtocolType = ms.ReadUInt16BE();
+                    gseLength -= 2;
+                    if (!endIndicator) 
+                        child.TotalLength -= 2;
+                    switch (labelTypeIndicator)
+                    {
+                        case 0:
+                            child.Label = new _6byteLabel(ms.ReadBytes(6));
+                            gseLength -= 6;
+                            if (!endIndicator) 
+                                child.TotalLength -= 6;
+                            lastLabel = child.Label;
+                            break;
+                        case 1:
+                            child.Label = new _3byteLabel(ms.ReadBytes(3));
+                            gseLength -= 3;
+                            if (!endIndicator) 
+                                child.TotalLength -= 3;
+                            lastLabel = child.Label;
+                            break;
+                        case 2:
+                            child.Label = BroadcastLabel.GetInstance();
+                            break;
+                        case 3:
+                            child.Label = this.lastLabel;
+                            break;
+                        default:
+                            throw new NotImplementedException(String.Format("Unknown label type: {0}", labelTypeIndicator));
+                    }
+                }
+
+                if (!startIndicator && endIndicator)
+                    gseLength -= 4; //for crc32
+
+                if (gseLength > ms.GetAvailableBytes())
+                    return;
+                child.GseDataBytes = ms.ReadBytes(gseLength);
+
+                if (!startIndicator && endIndicator)
+                    child.Crc32 = ms.ReadUInt32BE();
+                
+                ProcessPacket(child);
+            }
+        }
+    }
+
+    private bool[] warnedEthertypes;
+    private GseFragmentation[] fragmentations;
+    private void ProcessPacket(GsePacket child)
+    {
+        if (!child.StartIndicator && child.EndIndicator)
+        {
+            if (fragmentations == null)
+                return;
+            GseFragmentation currentFragmentation = fragmentations[child.FragmentId.Value];
+            if (currentFragmentation == null)
+                return;
+            currentFragmentation.AddFragement(child);
+            if (currentFragmentation.Validate())
+            {
+                mpeEventHandler.OnIpDatagram(0x010e,currentFragmentation.GetGseDataBytes());
+            }
+
+            fragmentations[child.FragmentId.Value] = null;
+            return;
+        }
+
+        if (child.StartIndicator && !child.EndIndicator)
+        {
+            if (fragmentations == null)
+                fragmentations = new GseFragmentation[256];
+            fragmentations[child.FragmentId.Value] = new GseFragmentation(child);
+            return;
+        }
+
+        if (child.StartIndicator && child.EndIndicator)
+        {
+            switch (child.ProtocolType)
+            {
+                case 0x0800:
+                    mpeEventHandler.OnIpDatagram(0x010e, child.GseDataBytes);
+                    break;
+                default:
+                    if (warnedEthertypes == null)
+                        warnedEthertypes = new bool[0xffff];
+                    if (!warnedEthertypes[child.ProtocolType.Value])
+                    {
+                        logger.WarnFormat("This GSE contains other traffic types (type {0:X4}) besides IP (type 0x0800). If it's not too much trouble, please consider submitting a sample.",child.ProtocolType.Value);
+                        warnedEthertypes[child.ProtocolType.Value] = true;
+                    }
+                    break;
+            }
+
+            return;
+        }
+        throw new NotImplementedException(child.ToString());
+    }
+} 

skyscraper8/GS/GsTypeDetector.cs

@@ -1,9 +1,45 @@
+using log4net;
+using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper8.GSE.GSE;
+
 namespace skyscraper8.GSE;
 
 public class GsTypeDetector : IMisHandler
 {
+    public IMultiprotocolEncapsulationEventHandler mpeEventHandler { get; set; }
+    private readonly byte _misId;
+    private readonly ILog logger;
+    private HashSet<Tuple<int, byte>> _postedStreamTypes;
+    public GsTypeDetector(byte misId)
+    {
+        _misId = misId;
+        logger = LogManager.GetLogger(String.Format("{0} MIS {1}",System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name,misId));
+    }
+
+    private GseReader gseReader;
     public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
     {
-        throw new NotImplementedException();
+        if (bbHeader.TsGs == 1 && bbHeader.SyncByte == 0)
+        {
+            //Looks like Continuous GSE.
+            if (gseReader == null)
+            {
+                gseReader = new GseReader();
+                gseReader.mpeEventHandler = mpeEventHandler;
+            }
+
+            gseReader.PushFrame(bbHeader, readOnlySpan);
+            return;
+        }
+        
+        //We have no idea what this is.
+        Tuple<int, byte> streamTypeToPost = new Tuple<int, byte>(bbHeader.TsGs, bbHeader.SyncByte);
+        if (_postedStreamTypes == null)
+            _postedStreamTypes = new HashSet<Tuple<int, byte>>();
+        if (!_postedStreamTypes.Contains(streamTypeToPost))
+        {
+            logger.WarnFormat("This GS contains packets of type {0} ({2}) with a sync byte of {1}. This is not supported yet. If it isn't too much trouble, please consider sharing a sample of this stream.",streamTypeToPost.Item1,streamTypeToPost.Item2, (TsGsType)streamTypeToPost.Item1);
+            _postedStreamTypes.Add(streamTypeToPost);
+        }
     }
 }

skyscraper8/GS/POC/Pts2Bbf.cs

@@ -44,7 +44,7 @@ public class Pts2Bbf
         FileStream fileStream = file.OpenRead();
         
         TsContext mpeg2 = new TsContext();
-        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(dumper));
+        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(null, dumper));
         DataStorage dataStorage = new InMemoryScraperStorage();
         ObjectStorage objectStorage = new NullObjectStorage();
         SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);

skyscraper8/GS/POC/Stid135Test.cs

@@ -13,13 +13,13 @@ public class Stid135Test
     {
         FileStream fileStream = file.OpenRead();
 
-        BbframeDeencapsulator3 decap = new BbframeDeencapsulator3();
-        
         TsContext mpeg2 = new TsContext();
-        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(decap));
         DataStorage dataStorage = new InMemoryScraperStorage();
         ObjectStorage objectStorage = new NullObjectStorage();
         SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
+
+        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(skyscraper));
+        
         skyscraper.InitalizeFilterChain();
         skyscraper.IngestFromStream(fileStream);
         

skyscraper8/GS/Stid135BbFrameReader.cs

@@ -6,6 +6,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper5.Dvb.DataBroadcasting;
 
 namespace skyscraper8.GSE
 {
@@ -13,10 +14,10 @@ namespace skyscraper8.GSE
 	{
 		private IBbframeDeencapsulator deencapsulator;
 
-		public Stid135BbFrameReader(IBbframeDeencapsulator deencapsulator = null)
+		public Stid135BbFrameReader(IMultiprotocolEncapsulationEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null)
 		{
 			if (deencapsulator == null)
-				deencapsulator = new BbframeDeencapsulator3();
+				deencapsulator = new BbframeDeencapsulator3(mpeEventHandler);
 			
 			this.deencapsulator = deencapsulator;
 		}

skyscraper8/GS/TsGsType.cs

@@ -0,0 +1,9 @@
+namespace skyscraper8.GSE;
+
+public enum TsGsType
+{
+    GenericPacketized,
+    GenericContinuous,
+    GseHem,
+    Transport
+}

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -301,7 +301,7 @@ namespace skyscraper5.Skyscraper.Scraper
                 {
                     if (!DvbContext.IsPidProcessorPresent(0x010e))
                     {
-                        DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
+                        DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
                         UiJunction?.SetGseMode();
                         LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                         SpecialTsType = 3;
@@ -356,7 +356,7 @@ namespace skyscraper5.Skyscraper.Scraper
                         {
                             if (!DvbContext.IsPidProcessorPresent(0x010e))
                             {
-                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
+                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
                                 UiJunction?.SetGseMode();
                                 LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                                 SpecialTsType = 3;