Merge remote-tracking branch 'origin/bbframes-revisited'

# Conflicts:
#	skyscraper8/GS/BbframeDeencapsulator2.cs
#	skyscraper8/GS/MisProcessor.cs
#	skyscraper8/GS/MisProcessors/0x00_Gse.cs
#	skyscraper8/GS/MisProcessors/UnknownStreamType.cs
#	skyscraper8/GS/Stid135TestingProgram.cs
#	skyscraper8/Program.cs

skyscraper8/GS/BBHeader.cs

@@ -0,0 +1,82 @@
+using skyscraper5.Skyscraper;
+
+namespace skyscraper8.GSE;
+
+public class BBHeader : Validatable
+{
+    public BBHeader(byte[] bbframe, int offset)
+    {
+        ReadOnlySpan<byte> BbHeaderSpan = new ReadOnlySpan<byte>(bbframe, offset, 10);
+        
+        //byte 1, MATYPE-1
+        TsGs = (BbHeaderSpan[0] & 0xc0) >> 6;
+        SisMis = (BbHeaderSpan[0] & 0x20) != 0; //1 single, 0 multi
+        CcmAcm = (BbHeaderSpan[0] & 0x10) != 0; //1 ccm, 0 acm
+        Issyi = (BbHeaderSpan[0] & 0x08) != 0;  //
+        Npd = (BbHeaderSpan[0] & 0x04) != 0;
+        Ro = (BbHeaderSpan[0] & 0x03);
+
+        Matype2 = (BbHeaderSpan[1]);
+        
+        UserPacketLength = BbHeaderSpan[2] << 8 | BbHeaderSpan[3];
+        UserPacketLength /= 8;
+        
+        DataFieldLength = BbHeaderSpan[4] << 8 | BbHeaderSpan[5];
+        DataFieldLength /= 8;
+        
+        SyncByte = BbHeaderSpan[6];
+        
+        SyncD = BbHeaderSpan[7] << 8  | BbHeaderSpan[8];
+
+        ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
+        Valid = ChecksumValid;
+    }
+
+    public bool ChecksumValid { get; private set; }
+    
+    public int SyncD { get; private set; }
+
+    public byte SyncByte { get; private set; }
+
+    public int DataFieldLength { get; private set; }
+
+    public int UserPacketLength { get; private set; }
+
+    /// <summary>
+    /// If MIS, this is the Input Stream Identifier
+    /// </summary>
+    public byte Matype2 { get; private set; }
+
+    /// <summary>
+    /// Transmission Roll-off factor
+    /// </summary>
+    public int Ro { get; private set; }
+
+    /// <summary>
+    /// Null Packet Deletion
+    /// </summary>
+    public bool Npd { get; private set; }
+
+    /// <summary>
+    /// If true, the ISSY field is inserted after UP
+    /// </summary>
+    public bool Issyi { get; set; }
+
+    /// <summary>
+    /// True if CCM, False if ACM
+    /// </summary>
+    public bool CcmAcm { get; private set; }
+
+    /// <summary>
+    /// True if SIS, False if MIS
+    /// </summary>
+    public bool SisMis { get; private set; }
+
+    /// <summary>
+    /// 0 = GS, packetized
+    /// 1 = GS, continuous
+    /// 2 = GSE-HEM
+    /// 3 = Transport
+    /// </summary>
+    public int TsGs { get; private set; }
+}

skyscraper8/GS/BBframeDeencapsulator3.cs

@@ -0,0 +1,40 @@
+using log4net;
+
+namespace skyscraper8.GSE;
+
+public class BbframeDeencapsulator3 : IBbframeDeencapsulator
+{
+    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+    private long numPushed;
+    public void PushPacket(byte[] bbframe)
+    {
+        //byte 0, sync byte
+        if (bbframe[0] != 0xb8)
+        {
+            if (numPushed == 0)
+            {
+                logger.InfoFormat("The stream started in the middle of a BBFrame, let's skip to the start of the next one.");
+            }
+            return;
+        }
+        numPushed++;
+
+        BBHeader bbHeader = new BBHeader(bbframe, 1);
+        if (!bbHeader.ChecksumValid)
+            return;
+        if (!bbHeader.Valid)
+            return;
+
+        if (mis == null)
+            mis = new IMisHandler[256];
+        if (mis[bbHeader.Matype2] == null)
+        {
+            logger.InfoFormat("Found a stream on MIS {0}",bbHeader.Matype2);
+            mis[bbHeader.Matype2] = new GsTypeDetector();
+        }
+
+        mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11));
+    }
+
+    private IMisHandler[] mis;
+}

skyscraper8/GS/Crc8.cs

@@ -0,0 +1,26 @@
+namespace skyscraper8.GSE;
+
+public static class DvbCrc8
+{
+    private const byte Polynomial = 0xD5; // x^8 + x^7 + x^6 + x^4 + x^2 + 1
+
+    public static byte Compute(ReadOnlySpan<byte> data)
+    {
+        byte crc = 0x00;
+
+        foreach (byte b in data)
+        {
+            crc ^= b; // XOR byte into top of crc register
+
+            for (int i = 0; i < 8; i++)
+            {
+                if ((crc & 0x80) != 0)
+                    crc = (byte)((crc << 1) ^ Polynomial);
+                else
+                    crc <<= 1;
+            }
+        }
+
+        return crc;
+    }
+}

skyscraper8/GS/GsTypeDetector.cs

@@ -0,0 +1,9 @@
+namespace skyscraper8.GSE;
+
+public class GsTypeDetector : IMisHandler
+{
+    public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
+    {
+        throw new NotImplementedException();
+    }
+}

skyscraper8/GS/IMisHandler.cs

@@ -0,0 +1,6 @@
+namespace skyscraper8.GSE;
+
+public interface IMisHandler
+{
+    void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan);
+}

skyscraper8/GS/POC/Pts2Bbf.cs

@@ -11,7 +11,7 @@ public class Pts2Bbf
 {
     private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
     
-    public static void Run(FileInfo file, bool useUdpDecap = false)
+    public static void Run(FileInfo file, bool useUdpDecap = false, IPEndPoint ipEndPoint = null)
     {
         if (!file.Exists)
         {
@@ -24,10 +24,14 @@ public class Pts2Bbf
         {
             string changeExtension = Path.ChangeExtension(file.FullName, ".sdecap");
             BbfUdpDecap.UdpDecapFileOutput udpDecapSink = new BbfUdpDecap.UdpDecapFileOutput(new FileInfo(changeExtension));
-            
+
+            if (ipEndPoint == null)
+            {
+                ipEndPoint = new IPEndPoint(IPAddress.Parse("239.199.2.1"), 1234);
+            }
             BbfUdpDecap bbfUdpDecap = new BbfUdpDecap();
-            bbfUdpDecap.SetTargetPort(1234);
-            bbfUdpDecap.SetTargetIp(IPAddress.Parse("239.199.2.1"));
+            bbfUdpDecap.SetTargetPort(ipEndPoint.Port);
+            bbfUdpDecap.SetTargetIp(ipEndPoint.Address);
             bbfUdpDecap.Sink = udpDecapSink;
             dumper = bbfUdpDecap;
         }
@@ -40,7 +44,7 @@ public class Pts2Bbf
         FileStream fileStream = file.OpenRead();
         
         TsContext mpeg2 = new TsContext();
-        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),dumper));
+        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(dumper));
         DataStorage dataStorage = new InMemoryScraperStorage();
         ObjectStorage objectStorage = new NullObjectStorage();
         SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);

skyscraper8/GS/POC/Stid135Test.cs

@@ -0,0 +1,29 @@
+using log4net;
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper.Scraper;
+using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
+using skyscraper8.Skyscraper.Scraper.Storage;
+
+namespace skyscraper8.GSE;
+
+public class Stid135Test
+{
+    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+    public static void Run(FileInfo file)
+    {
+        FileStream fileStream = file.OpenRead();
+
+        BbframeDeencapsulator3 decap = new BbframeDeencapsulator3();
+        
+        TsContext mpeg2 = new TsContext();
+        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(decap));
+        DataStorage dataStorage = new InMemoryScraperStorage();
+        ObjectStorage objectStorage = new NullObjectStorage();
+        SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
+        skyscraper.InitalizeFilterChain();
+        skyscraper.IngestFromStream(fileStream);
+        
+        fileStream.Close();
+        logger.Info("STiD135 Tester exiting");
+    }
+}

skyscraper8/GS/Stid135BbFrameReader.cs

@@ -11,15 +11,13 @@ namespace skyscraper8.GSE
 {
 	internal class Stid135BbFrameReader : ITsPacketProcessor
 	{
-		private IGsEventHandler mpeEventHandler;
 		private IBbframeDeencapsulator deencapsulator;
 
-		public Stid135BbFrameReader(IGsEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null)
+		public Stid135BbFrameReader(IBbframeDeencapsulator deencapsulator = null)
 		{
 			if (deencapsulator == null)
-				deencapsulator = new BbframeDeencapsulator();
+				deencapsulator = new BbframeDeencapsulator3();
 			
-			this.mpeEventHandler = mpeEventHandler;
 			this.deencapsulator = deencapsulator;
 		}
 
@@ -56,21 +54,6 @@ namespace skyscraper8.GSE
 			{
 				outbuf.Write(packets, 9, packets[7] - 1);
 			}
-			/*if ((packets[8] & 0xff) == 0xb8)
-			{
-				if (outbuf != null)
-				{
-					byte[] chi = outbuf.ToArray();
-					deencapsulator.PushPacket(chi);
-				}
-				outbuf = new MemoryStream();
-				outbuf.Write(packets,9, packets[7] - 1);
-			}
-			else
-			{
-				if (outbuf != null)
-					outbuf.Write(packets, 9, packets[7] - 1);
-			}*/
 		}
 	}
 }

skyscraper8/GSE/BbframeDeencapsulator.cs

@@ -1,446 +0,0 @@
-using skyscraper5.Mpeg2;
-using skyscraper5.Skyscraper;
-using skyscraper5.Skyscraper.IO;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.NetworkInformation;
-using System.Net.Sockets;
-using System.Runtime.Serialization;
-using System.Text;
-using System.Threading.Tasks;
-using log4net;
-
-namespace skyscraper8.GSE
-{
-	internal class BbframeDeencapsulator : IBbframeDeencapsulator
-	{
-		private bool interruptedGseHem;
-		private MemoryStream interruptedGseHemBuffer;
-		private bool interruptedGseHemWantsCrc32;
-		public IGsEventHandler MpeEventHandler { get; set; }
-
-        private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-
-        private bool shownNonGseWarning;
-
-        public void PushPacket(byte[] bbframe)
-		{
-			MemoryStream ms = new MemoryStream(bbframe, false);
-			//BBHeader
-			byte matype1 = ms.ReadUInt8();
-			int tsGsField = (matype1 & 0xc0) >> 6;
-			bool sisMisField = (matype1 & 0x20) != 0;
-			bool ccmAcmField = (matype1 & 0x10) != 0;
-			bool issyi = (matype1 & 0x08) != 0;
-			bool npd = (matype1 & 0x04) != 0;
-			int ro = matype1 & 0x03;
-
-			byte matype2 = ms.ReadUInt8();
-
-			ushort userPacketLength = ms.ReadUInt16BE();
-			ushort dataFieldLength = ms.ReadUInt16BE();
-			byte sync = ms.ReadUInt8();
-			ushort syncd = ms.ReadUInt16BE();
-			byte crc8 = ms.ReadUInt8();
-
-			if (userPacketLength == 0 && dataFieldLength == 0)
-				return;
-
-			switch (tsGsField)
-			{
-				case 1:
-					if (sync != 0)
-					{
-						if (!shownNonGseWarning)
-						{
-							logger.WarnFormat("This stream is a valid GS, but also contains packets which are not GSE (type 0), but of type {0} . Please share a sample of this stream!", sync);
-							shownNonGseWarning = true;
-						}
-						return;
-					}
-					int bytes = dataFieldLength / 8;
-					if (ms.GetAvailableBytes() < bytes)
-						return;
-					HandleContinous(ms);
-					break;
-				case 2:
-					int hemBytes = dataFieldLength / 8;
-					if (ms.GetAvailableBytes() < hemBytes)
-						return;
-					HandleGseHem(ms);
-					break;
-				default:    //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts
-					logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}, please share a sample of this stream.", tsGsField));
-					break;
-			}
-		}
-
-        private GseFragmentation[] gseFragmentations;
-        private void HandleContinous(MemoryStream ms)
-        {
-			ms.Position = 10;
-			
-
-			while (ms.GetAvailableBytes() > 0)
-			{
-				GsePacket packet = GsePacket.Read(ms);
-				if (packet.IsPadding)
-				{
-					break;
-				}
-
-				if (!packet.Valid)
-				{
-					break;
-				}
-
-				if (packet.IsCompletePacket)
-				{
-					if (ValidateEthertype(packet.ProtocolType.Value))
-					{
-						MpeEventHandler.OnIpDatagram(PID, packet.GseData);
-					}
-					else
-					{
-						logger.WarnFormat("Unknown EtherType: 0x{0:X4}", packet.ProtocolType.Value);
-					}
-					continue;
-				}
-
-				if (packet.StartIndicator && !packet.EndIndicator)
-				{
-					if (gseFragmentations == null)
-						gseFragmentations = new GseFragmentation[256];
-
-					gseFragmentations[packet.FragmentId.Value] = new GseFragmentation();
-					gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
-					continue;
-				}
-
-				if (!packet.StartIndicator && packet.EndIndicator)
-				{
-					if (gseFragmentations == null)
-						continue;
-					if (gseFragmentations[packet.FragmentId.Value] == null)
-						continue;
-
-					gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
-					continue;
-				}
-
-				if (!packet.StartIndicator && !packet.EndIndicator)
-				{
-					if (gseFragmentations == null)
-						continue;
-
-					throw new NotImplementedException();
-				}
-
-				throw new NotImplementedException();
-			}
-		}
-
-        private bool gseNeedMore;
-		private MemoryStream gseAssembler;
-		public int PID;
-
-		private void HandleGseHem(MemoryStream ms)
-		{
-			ms.Position = 10;
-			byte syncByte = ms.ReadUInt8();
-			ms.Position = 10;
-
-			long availableBytes = ms.GetAvailableBytes();
-			byte[] readBytes = ms.ReadBytes(availableBytes);
-
-			GseAssemblyState assemblyState = ValidateGse(gseAssembler, true);
-			if (assemblyState == GseAssemblyState.NOT_YET_BEGUN)
-			{
-				if ((syncByte & 0xc0) == 0xc0)
-				{
-					//Aktuell ist der GSE Assembler nicht beschäftigt, aber wir haben ein Sync-Byte, also kann er loslegen.
-					gseAssembler = new MemoryStream();
-					gseAssembler.Write(readBytes, 0, readBytes.Length);
-					return;
-				}
-				else
-				{
-					//Aktuell ist der GSE Assembler nicht beschäftigt, und wir haben kein gültiges Sync-Byte, also weg damit.
-					//Console.WriteLine("gse not in sync yet. sync byte = {0}", syncByte);
-					return;
-				}
-			}
-			else if (assemblyState == GseAssemblyState.NEED_MORE_DATA)
-			{
-				gseAssembler.Write(readBytes, 0, readBytes.Length);
-				return;
-			}
-			else if (assemblyState == GseAssemblyState.VALID)
-			{
-				gseAssembler.Position = 0;
-				AssembleGse(gseAssembler);
-				//Console.WriteLine("assembled {0} next sync byte is {1}", sucessfulAssembles++,syncByte);
-				if (sucessfulAssembles == 181)
-				{
-
-				}
-				gseAssembler = null;
-				HandleGseHem(ms);
-				return;
-			}
-			else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE)
-			{
-				gseAssembler = null;
-				HandleGseHem(ms);
-				return;
-			}
-			else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH)
-			{
-				gseAssembler = null;
-				HandleGseHem(ms);
-				return;
-			}
-			else
-			{
-				throw new NotImplementedException(string.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
-			}
-		}
-
-		private long sucessfulAssembles;
-		enum GseAssemblyState
-		{
-			NOT_YET_BEGUN,
-			VALID,
-			BROKEN_NEGATIVE_LENGTH,
-			NEED_MORE_DATA,
-			BROKEN_SUSPECT_LENGTH,
-			BROKEN_SUSPECT_ETHERTYPE,
-			VALID_NULL_PACKET,	//somehow these only show up on TBS6903x, not DD Max SX8?
-		}
-
-		private GseAssemblyState ValidateGse(MemoryStream ms, bool tryFix = false)
-		{
-			if (ms == null)
-				return GseAssemblyState.NOT_YET_BEGUN;
-
-			long backupPosition = ms.Position;
-			ms.Position = 0;
-			while (ms.GetAvailableBytes() > 2)
-			{
-				long fixPosition = ms.Position;
-
-				//GSE-Header
-				byte byteA = ms.ReadUInt8();
-				bool startIndicator = (byteA & 0x80) != 0;
-				bool endIndicator = (byteA & 0x40) != 0;
-				int labelIndicator = (byteA & 0x30) >> 4;
-				if (!startIndicator && !endIndicator && labelIndicator == 0)
-				{
-					//end of base band frame
-					ms.Position = backupPosition;
-					return GseAssemblyState.VALID;
-				}
-				else
-				{
-					byte byteB = ms.ReadUInt8();
-					int gseLength = byteA & 0x0f;
-					gseLength <<= 8;
-					gseLength += byteB;
-					//Console.WriteLine("GSE Length: {0}", gseLength);
-					if (gseLength > 9000)
-					{
-						if (tryFix)
-						{
-							ms.Position = fixPosition;
-							ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
-							break;
-						}
-						ms.Position = backupPosition;
-						return GseAssemblyState.BROKEN_SUSPECT_LENGTH;
-					}
-
-					if (!startIndicator || !endIndicator)
-					{
-						byte fragId = ms.ReadUInt8();
-						gseLength--;
-					}
-
-					if (startIndicator && !endIndicator)
-					{
-						if (2 > ms.GetAvailableBytes())
-						{
-							ms.Position = backupPosition;
-							return GseAssemblyState.NEED_MORE_DATA;
-						}
-						ushort totalLength = ms.ReadUInt16BE();
-						gseLength -= 2;
-					}
-
-					if (startIndicator)
-					{
-						if (2 > ms.GetAvailableBytes())
-						{
-							ms.Position = backupPosition;
-							return GseAssemblyState.NEED_MORE_DATA;
-						}
-						ushort protocolType = ms.ReadUInt16BE();
-						if (!ValidateEthertype(protocolType))
-						{
-							if (tryFix)
-							{
-								ms.Position = fixPosition;
-								ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
-								break;
-							}
-							ms.Position = backupPosition;
-							return GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE;
-						}
-						gseLength -= 2;
-						if (labelIndicator == 0)
-						{
-							if (6 > ms.GetAvailableBytes())
-							{
-								ms.Position = backupPosition;
-								return GseAssemblyState.NEED_MORE_DATA;
-							}
-							PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
-							gseLength -= 6;
-						}
-						else if (labelIndicator == 1)
-						{
-							byte[] threeByteLabel = ms.ReadBytes(3);
-							gseLength -= 3;
-						}
-					}
-
-					if (!startIndicator && endIndicator)
-						gseLength -= 4;
-
-					int startCrc32 = (int)ms.Position;
-
-					if (gseLength < 0)
-					{
-						if (tryFix)
-						{
-							ms.Position = fixPosition;
-							ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
-							break;
-						}
-						ms.Position = backupPosition;
-						return GseAssemblyState.BROKEN_NEGATIVE_LENGTH;
-					}
-
-					if (gseLength > ms.GetAvailableBytes())
-					{
-						ms.Position = backupPosition;
-						return GseAssemblyState.NEED_MORE_DATA;
-					}
-
-					ms.Position += gseLength;
-					if (!startIndicator && endIndicator)
-					{
-						if (4 > ms.GetAvailableBytes())
-						{
-							ms.Position = backupPosition;
-							return GseAssemblyState.NEED_MORE_DATA;
-						}
-						uint crc32 = ms.ReadUInt32BE();
-						int endCrc32 = (int)ms.Position;
-						DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
-					}
-				}
-			}
-
-			ms.Position = backupPosition;
-			return GseAssemblyState.VALID;
-		}
-
-		private bool ValidateEthertype(ushort ethertype)
-		{
-			if (ethertype == 0x0800)
-				return true;			//IPv4
-			if (ethertype == 0x86DD)
-				return true;			//IPv6
-			else
-				return false;			//There are many more valid values, but since we don't speak anything aside from IPv4/IPv6, we discard it.
-		}
-		private void AssembleGse(MemoryStream ms)
-		{
-			ushort protocolType = 0;
-			while (ms.GetAvailableBytes() > 2)
-			{
-				//GSE-Header
-				byte byteA = ms.ReadUInt8();
-				bool startIndicator = (byteA & 0x80) != 0;
-				bool endIndicator = (byteA & 0x40) != 0;
-				int labelIndicator = (byteA & 0x30) >> 4;
-				if (!startIndicator && !endIndicator && labelIndicator == 0)
-				{
-					//end of base band frame
-					return;
-				}
-				else
-				{
-					byte byteB = ms.ReadUInt8();
-					int gseLength = byteA & 0x0f;
-					gseLength <<= 8;
-					gseLength += byteB;
-
-					if (!startIndicator || !endIndicator)
-					{
-						byte fragId = ms.ReadUInt8();
-						gseLength--;
-					}
-
-					if (startIndicator && !endIndicator)
-					{
-						ushort totalLength = ms.ReadUInt16BE();
-						gseLength -= 2;
-					}
-
-					if (startIndicator)
-					{
-						protocolType = ms.ReadUInt16BE();
-						gseLength -= 2;
-						if (labelIndicator == 0)
-						{
-							PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
-							gseLength -= 6;
-						}
-						else if (labelIndicator == 1)
-						{
-							byte[] threeByteLabel = ms.ReadBytes(3);
-							gseLength -= 3;
-						}
-					}
-
-					if (!startIndicator && endIndicator)
-						gseLength -= 4;
-
-					int startCrc32 = (int)ms.Position;
-					
-					byte[] payload = ms.ReadBytes(gseLength);
-					if (!startIndicator && endIndicator)
-					{
-						uint crc32 = ms.ReadUInt32BE();
-						int endCrc32 = (int)ms.Position;
-						DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
-					}
-
-					switch (protocolType)
-					{
-						case 0x0800:	//IPv4
-							MpeEventHandler.OnIpDatagram(PID, payload);
-							break;
-						default:
-							//throw new NotImplementedException(String.Format("Unknown GSE Protocol ID 0x{0:X4}", protocolType));
-							break;
-					}
-				}
-			}
-		}
-
-
-	}
-}

skyscraper8/GSE/BbframeDeencapsulator2.cs

@@ -1,124 +0,0 @@
-using log4net;
-using skyscraper5.Skyscraper.IO;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Text;
-using System.Threading.Tasks;
-using System.Xml.Linq;
-using skyscraper5.Skyscraper.Net.Pcap;
-using skyscraper8.GSE.MisProcessors;
-
-namespace skyscraper8.GSE
-{
-	public class BbframeDeencapsulator2 : IBbframeDeencapsulator
-	{
-		[StructLayout(LayoutKind.Sequential, Pack = 1)]
-		public struct bbheader
-		{
-			public byte MaType1;
-			public byte MaType2;
-			public byte Upl1;
-			public byte Upl2;
-			public byte Dfl1;
-			public byte Dfl2;
-			public byte Sync;
-			public byte SyncD1;
-			public byte SyncD2;
-			public byte Crc8;
-		}
-
-		[StructLayout(LayoutKind.Sequential, Pack = 1)]
-		public unsafe struct Layer3
-		{
-			public byte l3sync;
-			public bbheader header;
-			public fixed byte Data[7254 - 10];
-		}
-
-		public long NumPushed { get; private set; }
-		private byte[] inData;
-
-		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-		public unsafe void PushPacket(byte[] bbframe)
-		{
-			NumPushed++;
-
-			if (inData == null)
-			{
-				inData = new byte[8100];
-			}
-
-			
-			MemoryStream inbuf = new MemoryStream(bbframe);
-			Layer3 myLayer;
-			while (true)
-			{
-				if (inbuf.Position == inbuf.Length)
-					return;
-
-				if (inbuf.Peek() == 0xb8)
-					break;
-				inbuf.ReadByte();
-			}
-
-			while (true)
-			{
-				int fail = inbuf.Read(inData, 0, 11);
-				if (fail != 11)
-					break;
-				myLayer = MemoryMarshal.Read<Layer3>(inData);
-				if (myLayer.l3sync != 0xb8)
-				{
-					while (true)
-					{
-						if (inbuf.Position == inbuf.Length)
-							return;
-						if (inbuf.Peek() == 0xb8)
-							break;
-						fail = inbuf.Read(inData, 0, 1);
-						if (fail != 1)
-							break;
-					}
-				}
-
-				int bblength = myLayer.header.Dfl1 << 8 | myLayer.header.Dfl2;
-				bblength >>= 3;
-				fail = inbuf.Read(inData, 11, bblength);
-				if (fail != bblength)
-					break;
-				myLayer = MemoryMarshal.Read<Layer3>(inData);
-
-				byte streamType = myLayer.header.Sync;
-				byte mis = myLayer.header.MaType2;
-
-				if (misStreams == null)
-					misStreams = new MisProcessor[256];
-				if (misStreams[mis] == null)
-				{
-					logger.InfoFormat("Found MIS {0}", mis);
-					MisProcessor processor;
-					switch (streamType)
-					{
-						case 0:
-							processor = new _0x00_Gse();
-							break;
-						default:
-							processor = new UnknownStreamType(streamType,mis);
-							break;
-					}
-
-					misStreams[mis] = processor;
-				}
-
-				misStreams[mis].PushPacket(bblength, myLayer);
-
-
-			}
-		}
-
-		private MisProcessor[] misStreams;
-		
-	}
-}

skyscraper8/GSE/GsEventHandler.cs

@@ -1,14 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace skyscraper5.Skyscraper
-{
-    internal interface IGsEventHandler
-    {
-        void OnIpDatagram(int pid, byte[] payload);
-        void GsIpTrafficDetected();
-    }
-}

skyscraper8/GSE/GseFragmentation.cs

@@ -1,26 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace skyscraper8.GSE
-{
-	internal class GseFragmentation
-	{
-		public void AddPacket(GsePacket packet)
-		{
-			if (spanningGsePackets == null)
-			{
-				spanningGsePackets = new List<GsePacket>();
-				TotalLength = packet.TotalLength.Value;
-			}
-
-			spanningGsePackets.Add(packet);
-		}
-
-		public ushort TotalLength { get; private set; }
-
-		private List<GsePacket> spanningGsePackets;
-	}
-}

skyscraper8/GSE/GsePacket.cs

@@ -1,167 +0,0 @@
-using skyscraper5.Skyscraper.IO;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.NetworkInformation;
-using System.Text;
-using System.Threading.Tasks;
-using skyscraper5.Skyscraper;
-
-namespace skyscraper8.GSE
-{
-	internal class GsePacket : Validatable
-	{
-		public static GsePacket Read(MemoryStream ms)
-		{
-			GsePacket packet = new GsePacket();
-
-			byte byteA = ms.ReadUInt8();
-			packet.StartIndicator = (byteA & 0x80) != 0;
-			packet.EndIndicator = (byteA & 0x40) != 0;
-			packet.LabelIndicator = (byteA & 0x30) >> 4;
-			if (!packet.StartIndicator && !packet.EndIndicator && packet.LabelIndicator == 0)
-			{
-				//End of baseband frame
-				return packet;
-			}
-			else
-			{
-				byte byteB = ms.ReadUInt8();
-				int gseLength = byteA & 0x0f;
-				gseLength <<= 8;
-				gseLength += byteB;
-
-				if (!packet.StartIndicator || !packet.EndIndicator)
-				{
-					packet.FragmentId = ms.ReadUInt8();
-					gseLength--;
-				}
-				
-				if (packet.StartIndicator && !packet.EndIndicator)
-				{
-					packet.TotalLength = ms.ReadUInt16BE();
-					gseLength -= 2;
-				}
-
-				if (packet.StartIndicator)
-				{
-					packet.ProtocolType = ms.ReadUInt16BE();
-					gseLength -= 2;
-					if (packet.LabelIndicator == 0)
-					{
-						packet.Label = new SixByteGseLabel(ms.ReadBytes(6));
-						gseLength -= 6;
-					}
-					else if (packet.LabelIndicator == 1)
-					{
-						packet.Label = new ThreeByteGseLabel(ms.ReadBytes(3));
-						gseLength -= 3;
-					}
-				}
-
-				if (!packet.StartIndicator && packet.EndIndicator)
-					gseLength -= 4;
-
-				if (gseLength < 0)
-				{
-					packet.Valid = false;
-					return packet;
-				}
-				if (gseLength > ms.GetAvailableBytes())
-				{
-					packet.Valid = false;
-					return packet;
-				}
-				packet.GseData = ms.ReadBytes(gseLength);
-
-				if (!packet.StartIndicator && packet.EndIndicator)
-					packet.Crc32 = ms.ReadUInt32BE();
-			}
-
-			packet.Valid = true;
-			return packet;
-		}
-
-		public uint? Crc32 { get; set; }
-
-		public byte[] GseData { get; set; }
-
-		public GseLabel Label { get; set; }
-		public ushort? ProtocolType { get; set; }
-
-		public ushort? TotalLength { get; set; }
-		public byte? FragmentId { get; set; }
-		public int LabelIndicator { get; private set; }
-		public bool EndIndicator { get; private set; }
-		public bool StartIndicator { get; private set; }
-		public bool IsPadding
-		{
-			get
-			{
-				return !StartIndicator && !EndIndicator && LabelIndicator == 0;
-			}
-		}
-
-		public bool IsCompletePacket
-		{
-			get
-			{
-				return StartIndicator && EndIndicator;
-			}
-		}
-	}
-
-	public abstract class GseLabel : Validatable
-	{
-		public abstract int LabelLength { get; }
-		public abstract string ToString();
-	}
-
-	public class ThreeByteGseLabel : GseLabel
-	{
-		public ThreeByteGseLabel(byte[] buffer)
-		{
-			if (buffer.Length != 3)
-			{
-				Valid = false;
-				return;
-			}
-
-			LabelBytes = buffer;
-			Valid = true;
-		}
-
-		public byte[] LabelBytes { get; private set; }
-
-		public override string ToString()
-		{
-			return BitConverter.ToString(LabelBytes);
-		}
-
-		public override int LabelLength => 3;
-	}
-
-	public class SixByteGseLabel : GseLabel
-	{
-		public SixByteGseLabel(byte[] buffer)
-		{
-			if (buffer.Length != 6)
-			{
-				Valid = false;
-				return;
-			}
-
-			MacAddress = new PhysicalAddress(buffer);
-			Valid = true;
-		}
-
-		public PhysicalAddress MacAddress { get; private set; }
-
-		public override string ToString()
-		{
-			return MacAddress.ToString();
-		}
-
-		public override int LabelLength => 6;
-	}
-}

skyscraper8/GSE/MisProcessor.cs

@@ -1,13 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace skyscraper8.GSE
-{
-	public interface MisProcessor
-	{
-		public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer);
-	}
-}

skyscraper8/GSE/MisProcessors/0x00_Gse.cs

@@ -1,131 +0,0 @@
-using skyscraper5.Skyscraper.Net.Pcap;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace skyscraper8.GSE.MisProcessors
-{
-	internal class _0x00_Gse : MisProcessor
-	{
-			private byte[][] fragmentor;
-			private int[] fragmentorLength;
-			private int[] fragmentorPos;
-			private PcapWriter pcapWriter;
-
-			public _0x00_Gse()
-			{
-				fragmentor = new byte[256][];
-				fragmentorLength = new int[256];
-				fragmentorPos = new int[256];
-				FileStream fileStream = File.OpenWrite(string.Format("{0}.pcap", DateTime.Now.Ticks));
-				pcapWriter = new PcapWriter(fileStream, TcpdumpNetworkType.RawIp);
-			}
-
-			public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer)
-			{
-				int pos = 0;
-				while (pos < bblength - 4)
-				{
-					int gseLength = ((myLayer.Data[pos] & 0x0f) << 8) | (myLayer.Data[pos + 1]);
-					if ((myLayer.Data[pos] & 0xf0) == 0) break;
-					if (gseLength + 2 > bblength - pos) break;
-					if (!ProcessBbFrame(&myLayer.Data[pos], gseLength))
-						break;
-					pos += gseLength += 2;
-				}
-			}
-
-			private unsafe void Copy(byte* src, byte[] dst, int srcOffset, int dstOffset, int len)
-			{
-				for (int i = 0; i < len; i++)
-				{
-					dst[dstOffset + i] = src[srcOffset + i];
-				}
-			}
-
-			private unsafe bool ProcessBbFrame(byte* payload, int gseLength)
-			{
-				uint offset = 0;
-				uint fragId = 0;
-
-				if ((payload[0] & 0xc0) == 0x80)
-				{
-					fragId = payload[2];
-					int length = (payload[3] << 8) | payload[4];
-					if (fragmentor[fragId] != null)
-						fragmentor[fragId] = null;
-					fragmentor[fragId] = new byte[length + 2];
-					fragmentorLength[fragId] = length + 2;
-					fragmentor[fragId][0] = payload[0];
-					fragmentor[fragId][1] = payload[1];
-					fragmentor[fragId][0] |= 0xc0;
-					Copy(payload, fragmentor[fragId], 5, 2, gseLength - 3);
-					fragmentorPos[fragId] = gseLength - 1;
-				}
-				else if ((payload[0] & 0xc0) == 0x00)
-				{
-					fragId = payload[2];
-					if (fragmentor[fragId] == null)
-						return true;
-
-					Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 1);
-					fragmentorPos[fragId] += gseLength - 1;
-				}
-				else if ((payload[0] & 0xc0) == 0x40)
-				{
-					fragId = payload[2];
-					if (fragmentor[fragId] == null)
-						return true;
-
-					Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 5);
-					fragmentorPos[fragId] += gseLength - 1;
-					fixed (byte* p = fragmentor[fragId])
-					{
-						ProcessBbFrame(p, fragmentorLength[fragId]);
-					}
-					fragmentor[fragId] = null;
-
-				}
-				else if ((payload[0] & 0xc0) == 0xc0)
-				{
-					if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x04)
-					{
-						throw new NotImplementedException("ethertype 0x0400");
-					}
-					else if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x00)
-					{
-						throw new NotImplementedException("ethertype 0x0000");
-					}
-					else if (payload[offset + 2] == 0x08 && payload[offset + 3] == 0x00)
-					{
-						if ((payload[0] & 0x30) == 0x01)
-						{
-							offset += 3;    //3-byte label
-						}
-						else if ((payload[0] & 0x30) == 0x00)
-						{
-							offset += 6;    //MAC address
-						}
-
-						offset += 4;
-
-						//at this point, the IP header starts at "offset"
-						long sinkSize = gseLength + 2 - offset;
-						byte[] sink = new byte[sinkSize];
-						Copy(payload, sink, (int)offset, 0, (int)sinkSize);
-						pcapWriter.WritePacket(sink);
-						return true;
-					}
-
-					//throw new NotImplementedException("complete packet");
-				}
-				else if ((payload[0] & 0x0f0) == 0x00)
-				{
-					throw new NotImplementedException("padding packet");
-				}
-				return true;
-			}
-		}
-}

skyscraper8/GSE/MisProcessors/UnknownStreamType.cs

@@ -1,38 +0,0 @@
-using log4net;
-using skyscraper5.Skyscraper.Scraper;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace skyscraper8.GSE.MisProcessors
-{
-	internal class UnknownStreamType : MisProcessor
-	{
-		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-		private bool[] warningsSpoken;
-		private byte mis;
-
-		public UnknownStreamType(byte streamType, byte mis)
-		{
-			this.mis = mis;
-			this.warningsSpoken = new bool[256];
-			this.EncounterPacket(streamType);
-		}
-
-		private void EncounterPacket(byte streamType)
-		{
-			if (!warningsSpoken[streamType])
-			{
-				logger.WarnFormat("This is a valid GS, however, it not GSE (type 0), but type {0}, which is not supported yet. Please consider sharing a sample of this stream.", streamType);
-				warningsSpoken[streamType] = true;
-			}
-		}
-
-		public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer)
-		{
-			EncounterPacket(myLayer.header.Sync);
-		}
-	}
-}

skyscraper8/GSE/NullGsEventHandler.cs

@@ -1,16 +0,0 @@
-using skyscraper5.Skyscraper;
-
-namespace skyscraper8.GSE;
-
-public class NullGsEventHandler : IGsEventHandler
-{
-    public void OnIpDatagram(int pid, byte[] payload)
-    {
-        
-    }
-
-    public void GsIpTrafficDetected()
-    {
-        
-    }
-}

skyscraper8/GSE/Stid135TestingProgram.cs

@@ -1,40 +0,0 @@
-using log4net;
-using log4net.Repository.Hierarchy;
-using skyscraper5.Mpeg2;
-using skyscraper5.Skyscraper.Scraper;
-using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
-using skyscraper8.Skyscraper.Scraper.Storage;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace skyscraper8.GSE
-{
-	internal class Stid135TestingProgram
-	{
-		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-		public static void Run(FileInfo file)
-		{
-			if (!file.Exists)
-			{
-				logger.Error("File not found: " + file.FullName);
-				return;
-			}
-
-			FileStream fileStream = file.OpenRead();
-
-			TsContext mpeg2 = new TsContext();
-			mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),new BbframeDeencapsulator2()));
-			DataStorage dataStorage = new InMemoryScraperStorage();
-			ObjectStorage objectStorage = new NullObjectStorage();
-			SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
-			skyscraper.InitalizeFilterChain();
-			skyscraper.IngestFromStream(fileStream);
-
-			fileStream.Close();
-			logger.Info("Stid135TestingProgram finished");
-		}
-	}
-}

skyscraper8/Program.cs

@@ -326,9 +326,9 @@ namespace skyscraper5
 
                 if (args[0].ToLowerInvariant().Equals("stid135test"))
                 {
-	                FileInfo fi = new FileInfo(args[1]);
-	                Stid135TestingProgram.Run(fi);
-	                return;
+                    FileInfo fi = new FileInfo(args[1]);
+                    Stid135Test.Run(fi);
+                    return;
                 }
 			}
 

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -92,7 +92,7 @@ namespace skyscraper5.Skyscraper.Scraper
         IEitEventHandler, IAitEventHandler, ISubtitleEventHandler,
         UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
         IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
-		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, IGsEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
+		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
         InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
@@ -301,7 +301,7 @@ namespace skyscraper5.Skyscraper.Scraper
                 {
                     if (!DvbContext.IsPidProcessorPresent(0x010e))
                     {
-                        DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
+                        DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
                         UiJunction?.SetGseMode();
                         LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                         SpecialTsType = 3;
@@ -356,7 +356,7 @@ namespace skyscraper5.Skyscraper.Scraper
                         {
                             if (!DvbContext.IsPidProcessorPresent(0x010e))
                             {
-                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
+                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
                                 UiJunction?.SetGseMode();
                                 LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                                 SpecialTsType = 3;