skyscraper8/GS/BBHeader.cs

@@ -1,82 +0,0 @@
-using skyscraper5.Skyscraper;
-
-namespace skyscraper8.GSE;
-
-public class BBHeader : Validatable
-{
-    public BBHeader(byte[] bbframe, int offset)
-    {
-        ReadOnlySpan<byte> BbHeaderSpan = new ReadOnlySpan<byte>(bbframe, offset, 10);
-        
-        //byte 1, MATYPE-1
-        TsGs = (BbHeaderSpan[0] & 0xc0) >> 6;
-        SisMis = (BbHeaderSpan[0] & 0x20) != 0; //1 single, 0 multi
-        CcmAcm = (BbHeaderSpan[0] & 0x10) != 0; //1 ccm, 0 acm
-        Issyi = (BbHeaderSpan[0] & 0x08) != 0;  //
-        Npd = (BbHeaderSpan[0] & 0x04) != 0;
-        Ro = (BbHeaderSpan[0] & 0x03);
-
-        Matype2 = (BbHeaderSpan[1]);
-        
-        UserPacketLength = BbHeaderSpan[2] << 8 | BbHeaderSpan[3];
-        UserPacketLength /= 8;
-        
-        DataFieldLength = BbHeaderSpan[4] << 8 | BbHeaderSpan[5];
-        DataFieldLength /= 8;
-        
-        SyncByte = BbHeaderSpan[6];
-        
-        SyncD = BbHeaderSpan[7] << 8  | BbHeaderSpan[8];
-
-        ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
-        Valid = ChecksumValid;
-    }
-
-    public bool ChecksumValid { get; private set; }
-    
-    public int SyncD { get; private set; }
-
-    public byte SyncByte { get; private set; }
-
-    public int DataFieldLength { get; private set; }
-
-    public int UserPacketLength { get; private set; }
-
-    /// <summary>
-    /// If MIS, this is the Input Stream Identifier
-    /// </summary>
-    public byte Matype2 { get; private set; }
-
-    /// <summary>
-    /// Transmission Roll-off factor
-    /// </summary>
-    public int Ro { get; private set; }
-
-    /// <summary>
-    /// Null Packet Deletion
-    /// </summary>
-    public bool Npd { get; private set; }
-
-    /// <summary>
-    /// If true, the ISSY field is inserted after UP
-    /// </summary>
-    public bool Issyi { get; set; }
-
-    /// <summary>
-    /// True if CCM, False if ACM
-    /// </summary>
-    public bool CcmAcm { get; private set; }
-
-    /// <summary>
-    /// True if SIS, False if MIS
-    /// </summary>
-    public bool SisMis { get; private set; }
-
-    /// <summary>
-    /// 0 = GS, packetized
-    /// 1 = GS, continuous
-    /// 2 = GSE-HEM
-    /// 3 = Transport
-    /// </summary>
-    public int TsGs { get; private set; }
-}

skyscraper8/GS/BBframeDeencapsulator3.cs

@@ -1,40 +0,0 @@
-using log4net;
-
-namespace skyscraper8.GSE;
-
-public class BbframeDeencapsulator3 : IBbframeDeencapsulator
-{
-    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-    private long numPushed;
-    public void PushPacket(byte[] bbframe)
-    {
-        //byte 0, sync byte
-        if (bbframe[0] != 0xb8)
-        {
-            if (numPushed == 0)
-            {
-                logger.InfoFormat("The stream started in the middle of a BBFrame, let's skip to the start of the next one.");
-            }
-            return;
-        }
-        numPushed++;
-
-        BBHeader bbHeader = new BBHeader(bbframe, 1);
-        if (!bbHeader.ChecksumValid)
-            return;
-        if (!bbHeader.Valid)
-            return;
-
-        if (mis == null)
-            mis = new IMisHandler[256];
-        if (mis[bbHeader.Matype2] == null)
-        {
-            logger.InfoFormat("Found a stream on MIS {0}",bbHeader.Matype2);
-            mis[bbHeader.Matype2] = new GsTypeDetector();
-        }
-
-        mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11));
-    }
-
-    private IMisHandler[] mis;
-}

skyscraper8/GS/Crc8.cs

@@ -1,26 +0,0 @@
-namespace skyscraper8.GSE;
-
-public static class DvbCrc8
-{
-    private const byte Polynomial = 0xD5; // x^8 + x^7 + x^6 + x^4 + x^2 + 1
-
-    public static byte Compute(ReadOnlySpan<byte> data)
-    {
-        byte crc = 0x00;
-
-        foreach (byte b in data)
-        {
-            crc ^= b; // XOR byte into top of crc register
-
-            for (int i = 0; i < 8; i++)
-            {
-                if ((crc & 0x80) != 0)
-                    crc = (byte)((crc << 1) ^ Polynomial);
-                else
-                    crc <<= 1;
-            }
-        }
-
-        return crc;
-    }
-}

skyscraper8/GS/GsTypeDetector.cs

@@ -1,9 +0,0 @@
-namespace skyscraper8.GSE;
-
-public class GsTypeDetector : IMisHandler
-{
-    public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
-    {
-        throw new NotImplementedException();
-    }
-}

skyscraper8/GS/IMisHandler.cs

@@ -1,6 +0,0 @@
-namespace skyscraper8.GSE;
-
-public interface IMisHandler
-{
-    void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan);
-}

skyscraper8/GS/POC/Stid135Test.cs

@@ -1,29 +0,0 @@
-using log4net;
-using skyscraper5.Mpeg2;
-using skyscraper5.Skyscraper.Scraper;
-using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
-using skyscraper8.Skyscraper.Scraper.Storage;
-
-namespace skyscraper8.GSE;
-
-public class Stid135Test
-{
-    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-    public static void Run(FileInfo file)
-    {
-        FileStream fileStream = file.OpenRead();
-
-        BbframeDeencapsulator3 decap = new BbframeDeencapsulator3();
-        
-        TsContext mpeg2 = new TsContext();
-        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(decap));
-        DataStorage dataStorage = new InMemoryScraperStorage();
-        ObjectStorage objectStorage = new NullObjectStorage();
-        SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
-        skyscraper.InitalizeFilterChain();
-        skyscraper.IngestFromStream(fileStream);
-        
-        fileStream.Close();
-        logger.Info("STiD135 Tester exiting");
-    }
-}

skyscraper8/GSE/BbframeDeencapsulator.cs

@@ -0,0 +1,446 @@
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper;
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using System.Runtime.Serialization;
+using System.Text;
+using System.Threading.Tasks;
+using log4net;
+
+namespace skyscraper8.GSE
+{
+	internal class BbframeDeencapsulator : IBbframeDeencapsulator
+	{
+		private bool interruptedGseHem;
+		private MemoryStream interruptedGseHemBuffer;
+		private bool interruptedGseHemWantsCrc32;
+		public IGsEventHandler MpeEventHandler { get; set; }
+
+        private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+
+        private bool shownNonGseWarning;
+
+        public void PushPacket(byte[] bbframe)
+		{
+			MemoryStream ms = new MemoryStream(bbframe, false);
+			//BBHeader
+			byte matype1 = ms.ReadUInt8();
+			int tsGsField = (matype1 & 0xc0) >> 6;
+			bool sisMisField = (matype1 & 0x20) != 0;
+			bool ccmAcmField = (matype1 & 0x10) != 0;
+			bool issyi = (matype1 & 0x08) != 0;
+			bool npd = (matype1 & 0x04) != 0;
+			int ro = matype1 & 0x03;
+
+			byte matype2 = ms.ReadUInt8();
+
+			ushort userPacketLength = ms.ReadUInt16BE();
+			ushort dataFieldLength = ms.ReadUInt16BE();
+			byte sync = ms.ReadUInt8();
+			ushort syncd = ms.ReadUInt16BE();
+			byte crc8 = ms.ReadUInt8();
+
+			if (userPacketLength == 0 && dataFieldLength == 0)
+				return;
+
+			switch (tsGsField)
+			{
+				case 1:
+					if (sync != 0)
+					{
+						if (!shownNonGseWarning)
+						{
+							logger.WarnFormat("This stream is a valid GS, but also contains packets which are not GSE (type 0), but of type {0} . Please share a sample of this stream!", sync);
+							shownNonGseWarning = true;
+						}
+						return;
+					}
+					int bytes = dataFieldLength / 8;
+					if (ms.GetAvailableBytes() < bytes)
+						return;
+					HandleContinous(ms);
+					break;
+				case 2:
+					int hemBytes = dataFieldLength / 8;
+					if (ms.GetAvailableBytes() < hemBytes)
+						return;
+					HandleGseHem(ms);
+					break;
+				default:    //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts
+					logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}, please share a sample of this stream.", tsGsField));
+					break;
+			}
+		}
+
+        private GseFragmentation[] gseFragmentations;
+        private void HandleContinous(MemoryStream ms)
+        {
+			ms.Position = 10;
+			
+
+			while (ms.GetAvailableBytes() > 0)
+			{
+				GsePacket packet = GsePacket.Read(ms);
+				if (packet.IsPadding)
+				{
+					break;
+				}
+
+				if (!packet.Valid)
+				{
+					break;
+				}
+
+				if (packet.IsCompletePacket)
+				{
+					if (ValidateEthertype(packet.ProtocolType.Value))
+					{
+						MpeEventHandler.OnIpDatagram(PID, packet.GseData);
+					}
+					else
+					{
+						logger.WarnFormat("Unknown EtherType: 0x{0:X4}", packet.ProtocolType.Value);
+					}
+					continue;
+				}
+
+				if (packet.StartIndicator && !packet.EndIndicator)
+				{
+					if (gseFragmentations == null)
+						gseFragmentations = new GseFragmentation[256];
+
+					gseFragmentations[packet.FragmentId.Value] = new GseFragmentation();
+					gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
+					continue;
+				}
+
+				if (!packet.StartIndicator && packet.EndIndicator)
+				{
+					if (gseFragmentations == null)
+						continue;
+					if (gseFragmentations[packet.FragmentId.Value] == null)
+						continue;
+
+					gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
+					continue;
+				}
+
+				if (!packet.StartIndicator && !packet.EndIndicator)
+				{
+					if (gseFragmentations == null)
+						continue;
+
+					throw new NotImplementedException();
+				}
+
+				throw new NotImplementedException();
+			}
+		}
+
+        private bool gseNeedMore;
+		private MemoryStream gseAssembler;
+		public int PID;
+
+		private void HandleGseHem(MemoryStream ms)
+		{
+			ms.Position = 10;
+			byte syncByte = ms.ReadUInt8();
+			ms.Position = 10;
+
+			long availableBytes = ms.GetAvailableBytes();
+			byte[] readBytes = ms.ReadBytes(availableBytes);
+
+			GseAssemblyState assemblyState = ValidateGse(gseAssembler, true);
+			if (assemblyState == GseAssemblyState.NOT_YET_BEGUN)
+			{
+				if ((syncByte & 0xc0) == 0xc0)
+				{
+					//Aktuell ist der GSE Assembler nicht beschäftigt, aber wir haben ein Sync-Byte, also kann er loslegen.
+					gseAssembler = new MemoryStream();
+					gseAssembler.Write(readBytes, 0, readBytes.Length);
+					return;
+				}
+				else
+				{
+					//Aktuell ist der GSE Assembler nicht beschäftigt, und wir haben kein gültiges Sync-Byte, also weg damit.
+					//Console.WriteLine("gse not in sync yet. sync byte = {0}", syncByte);
+					return;
+				}
+			}
+			else if (assemblyState == GseAssemblyState.NEED_MORE_DATA)
+			{
+				gseAssembler.Write(readBytes, 0, readBytes.Length);
+				return;
+			}
+			else if (assemblyState == GseAssemblyState.VALID)
+			{
+				gseAssembler.Position = 0;
+				AssembleGse(gseAssembler);
+				//Console.WriteLine("assembled {0} next sync byte is {1}", sucessfulAssembles++,syncByte);
+				if (sucessfulAssembles == 181)
+				{
+
+				}
+				gseAssembler = null;
+				HandleGseHem(ms);
+				return;
+			}
+			else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE)
+			{
+				gseAssembler = null;
+				HandleGseHem(ms);
+				return;
+			}
+			else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH)
+			{
+				gseAssembler = null;
+				HandleGseHem(ms);
+				return;
+			}
+			else
+			{
+				throw new NotImplementedException(string.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
+			}
+		}
+
+		private long sucessfulAssembles;
+		enum GseAssemblyState
+		{
+			NOT_YET_BEGUN,
+			VALID,
+			BROKEN_NEGATIVE_LENGTH,
+			NEED_MORE_DATA,
+			BROKEN_SUSPECT_LENGTH,
+			BROKEN_SUSPECT_ETHERTYPE,
+			VALID_NULL_PACKET,	//somehow these only show up on TBS6903x, not DD Max SX8?
+		}
+
+		private GseAssemblyState ValidateGse(MemoryStream ms, bool tryFix = false)
+		{
+			if (ms == null)
+				return GseAssemblyState.NOT_YET_BEGUN;
+
+			long backupPosition = ms.Position;
+			ms.Position = 0;
+			while (ms.GetAvailableBytes() > 2)
+			{
+				long fixPosition = ms.Position;
+
+				//GSE-Header
+				byte byteA = ms.ReadUInt8();
+				bool startIndicator = (byteA & 0x80) != 0;
+				bool endIndicator = (byteA & 0x40) != 0;
+				int labelIndicator = (byteA & 0x30) >> 4;
+				if (!startIndicator && !endIndicator && labelIndicator == 0)
+				{
+					//end of base band frame
+					ms.Position = backupPosition;
+					return GseAssemblyState.VALID;
+				}
+				else
+				{
+					byte byteB = ms.ReadUInt8();
+					int gseLength = byteA & 0x0f;
+					gseLength <<= 8;
+					gseLength += byteB;
+					//Console.WriteLine("GSE Length: {0}", gseLength);
+					if (gseLength > 9000)
+					{
+						if (tryFix)
+						{
+							ms.Position = fixPosition;
+							ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
+							break;
+						}
+						ms.Position = backupPosition;
+						return GseAssemblyState.BROKEN_SUSPECT_LENGTH;
+					}
+
+					if (!startIndicator || !endIndicator)
+					{
+						byte fragId = ms.ReadUInt8();
+						gseLength--;
+					}
+
+					if (startIndicator && !endIndicator)
+					{
+						if (2 > ms.GetAvailableBytes())
+						{
+							ms.Position = backupPosition;
+							return GseAssemblyState.NEED_MORE_DATA;
+						}
+						ushort totalLength = ms.ReadUInt16BE();
+						gseLength -= 2;
+					}
+
+					if (startIndicator)
+					{
+						if (2 > ms.GetAvailableBytes())
+						{
+							ms.Position = backupPosition;
+							return GseAssemblyState.NEED_MORE_DATA;
+						}
+						ushort protocolType = ms.ReadUInt16BE();
+						if (!ValidateEthertype(protocolType))
+						{
+							if (tryFix)
+							{
+								ms.Position = fixPosition;
+								ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
+								break;
+							}
+							ms.Position = backupPosition;
+							return GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE;
+						}
+						gseLength -= 2;
+						if (labelIndicator == 0)
+						{
+							if (6 > ms.GetAvailableBytes())
+							{
+								ms.Position = backupPosition;
+								return GseAssemblyState.NEED_MORE_DATA;
+							}
+							PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
+							gseLength -= 6;
+						}
+						else if (labelIndicator == 1)
+						{
+							byte[] threeByteLabel = ms.ReadBytes(3);
+							gseLength -= 3;
+						}
+					}
+
+					if (!startIndicator && endIndicator)
+						gseLength -= 4;
+
+					int startCrc32 = (int)ms.Position;
+
+					if (gseLength < 0)
+					{
+						if (tryFix)
+						{
+							ms.Position = fixPosition;
+							ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
+							break;
+						}
+						ms.Position = backupPosition;
+						return GseAssemblyState.BROKEN_NEGATIVE_LENGTH;
+					}
+
+					if (gseLength > ms.GetAvailableBytes())
+					{
+						ms.Position = backupPosition;
+						return GseAssemblyState.NEED_MORE_DATA;
+					}
+
+					ms.Position += gseLength;
+					if (!startIndicator && endIndicator)
+					{
+						if (4 > ms.GetAvailableBytes())
+						{
+							ms.Position = backupPosition;
+							return GseAssemblyState.NEED_MORE_DATA;
+						}
+						uint crc32 = ms.ReadUInt32BE();
+						int endCrc32 = (int)ms.Position;
+						DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
+					}
+				}
+			}
+
+			ms.Position = backupPosition;
+			return GseAssemblyState.VALID;
+		}
+
+		private bool ValidateEthertype(ushort ethertype)
+		{
+			if (ethertype == 0x0800)
+				return true;			//IPv4
+			if (ethertype == 0x86DD)
+				return true;			//IPv6
+			else
+				return false;			//There are many more valid values, but since we don't speak anything aside from IPv4/IPv6, we discard it.
+		}
+		private void AssembleGse(MemoryStream ms)
+		{
+			ushort protocolType = 0;
+			while (ms.GetAvailableBytes() > 2)
+			{
+				//GSE-Header
+				byte byteA = ms.ReadUInt8();
+				bool startIndicator = (byteA & 0x80) != 0;
+				bool endIndicator = (byteA & 0x40) != 0;
+				int labelIndicator = (byteA & 0x30) >> 4;
+				if (!startIndicator && !endIndicator && labelIndicator == 0)
+				{
+					//end of base band frame
+					return;
+				}
+				else
+				{
+					byte byteB = ms.ReadUInt8();
+					int gseLength = byteA & 0x0f;
+					gseLength <<= 8;
+					gseLength += byteB;
+
+					if (!startIndicator || !endIndicator)
+					{
+						byte fragId = ms.ReadUInt8();
+						gseLength--;
+					}
+
+					if (startIndicator && !endIndicator)
+					{
+						ushort totalLength = ms.ReadUInt16BE();
+						gseLength -= 2;
+					}
+
+					if (startIndicator)
+					{
+						protocolType = ms.ReadUInt16BE();
+						gseLength -= 2;
+						if (labelIndicator == 0)
+						{
+							PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
+							gseLength -= 6;
+						}
+						else if (labelIndicator == 1)
+						{
+							byte[] threeByteLabel = ms.ReadBytes(3);
+							gseLength -= 3;
+						}
+					}
+
+					if (!startIndicator && endIndicator)
+						gseLength -= 4;
+
+					int startCrc32 = (int)ms.Position;
+					
+					byte[] payload = ms.ReadBytes(gseLength);
+					if (!startIndicator && endIndicator)
+					{
+						uint crc32 = ms.ReadUInt32BE();
+						int endCrc32 = (int)ms.Position;
+						DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
+					}
+
+					switch (protocolType)
+					{
+						case 0x0800:	//IPv4
+							MpeEventHandler.OnIpDatagram(PID, payload);
+							break;
+						default:
+							//throw new NotImplementedException(String.Format("Unknown GSE Protocol ID 0x{0:X4}", protocolType));
+							break;
+					}
+				}
+			}
+		}
+
+
+	}
+}

skyscraper8/GSE/BbframeDeencapsulator2.cs

@@ -0,0 +1,124 @@
+using log4net;
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+using System.Xml.Linq;
+using skyscraper5.Skyscraper.Net.Pcap;
+using skyscraper8.GSE.MisProcessors;
+
+namespace skyscraper8.GSE
+{
+	public class BbframeDeencapsulator2 : IBbframeDeencapsulator
+	{
+		[StructLayout(LayoutKind.Sequential, Pack = 1)]
+		public struct bbheader
+		{
+			public byte MaType1;
+			public byte MaType2;
+			public byte Upl1;
+			public byte Upl2;
+			public byte Dfl1;
+			public byte Dfl2;
+			public byte Sync;
+			public byte SyncD1;
+			public byte SyncD2;
+			public byte Crc8;
+		}
+
+		[StructLayout(LayoutKind.Sequential, Pack = 1)]
+		public unsafe struct Layer3
+		{
+			public byte l3sync;
+			public bbheader header;
+			public fixed byte Data[7254 - 10];
+		}
+
+		public long NumPushed { get; private set; }
+		private byte[] inData;
+
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		public unsafe void PushPacket(byte[] bbframe)
+		{
+			NumPushed++;
+
+			if (inData == null)
+			{
+				inData = new byte[8100];
+			}
+
+			
+			MemoryStream inbuf = new MemoryStream(bbframe);
+			Layer3 myLayer;
+			while (true)
+			{
+				if (inbuf.Position == inbuf.Length)
+					return;
+
+				if (inbuf.Peek() == 0xb8)
+					break;
+				inbuf.ReadByte();
+			}
+
+			while (true)
+			{
+				int fail = inbuf.Read(inData, 0, 11);
+				if (fail != 11)
+					break;
+				myLayer = MemoryMarshal.Read<Layer3>(inData);
+				if (myLayer.l3sync != 0xb8)
+				{
+					while (true)
+					{
+						if (inbuf.Position == inbuf.Length)
+							return;
+						if (inbuf.Peek() == 0xb8)
+							break;
+						fail = inbuf.Read(inData, 0, 1);
+						if (fail != 1)
+							break;
+					}
+				}
+
+				int bblength = myLayer.header.Dfl1 << 8 | myLayer.header.Dfl2;
+				bblength >>= 3;
+				fail = inbuf.Read(inData, 11, bblength);
+				if (fail != bblength)
+					break;
+				myLayer = MemoryMarshal.Read<Layer3>(inData);
+
+				byte streamType = myLayer.header.Sync;
+				byte mis = myLayer.header.MaType2;
+
+				if (misStreams == null)
+					misStreams = new MisProcessor[256];
+				if (misStreams[mis] == null)
+				{
+					logger.InfoFormat("Found MIS {0}", mis);
+					MisProcessor processor;
+					switch (streamType)
+					{
+						case 0:
+							processor = new _0x00_Gse();
+							break;
+						default:
+							processor = new UnknownStreamType(streamType,mis);
+							break;
+					}
+
+					misStreams[mis] = processor;
+				}
+
+				misStreams[mis].PushPacket(bblength, myLayer);
+
+
+			}
+		}
+
+		private MisProcessor[] misStreams;
+		
+	}
+}

skyscraper8/GSE/GsEventHandler.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper5.Skyscraper
+{
+    internal interface IGsEventHandler
+    {
+        void OnIpDatagram(int pid, byte[] payload);
+        void GsIpTrafficDetected();
+    }
+}

skyscraper8/GSE/GseFragmentation.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE
+{
+	internal class GseFragmentation
+	{
+		public void AddPacket(GsePacket packet)
+		{
+			if (spanningGsePackets == null)
+			{
+				spanningGsePackets = new List<GsePacket>();
+				TotalLength = packet.TotalLength.Value;
+			}
+
+			spanningGsePackets.Add(packet);
+		}
+
+		public ushort TotalLength { get; private set; }
+
+		private List<GsePacket> spanningGsePackets;
+	}
+}

skyscraper8/GSE/GsePacket.cs

@@ -0,0 +1,167 @@
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.NetworkInformation;
+using System.Text;
+using System.Threading.Tasks;
+using skyscraper5.Skyscraper;
+
+namespace skyscraper8.GSE
+{
+	internal class GsePacket : Validatable
+	{
+		public static GsePacket Read(MemoryStream ms)
+		{
+			GsePacket packet = new GsePacket();
+
+			byte byteA = ms.ReadUInt8();
+			packet.StartIndicator = (byteA & 0x80) != 0;
+			packet.EndIndicator = (byteA & 0x40) != 0;
+			packet.LabelIndicator = (byteA & 0x30) >> 4;
+			if (!packet.StartIndicator && !packet.EndIndicator && packet.LabelIndicator == 0)
+			{
+				//End of baseband frame
+				return packet;
+			}
+			else
+			{
+				byte byteB = ms.ReadUInt8();
+				int gseLength = byteA & 0x0f;
+				gseLength <<= 8;
+				gseLength += byteB;
+
+				if (!packet.StartIndicator || !packet.EndIndicator)
+				{
+					packet.FragmentId = ms.ReadUInt8();
+					gseLength--;
+				}
+				
+				if (packet.StartIndicator && !packet.EndIndicator)
+				{
+					packet.TotalLength = ms.ReadUInt16BE();
+					gseLength -= 2;
+				}
+
+				if (packet.StartIndicator)
+				{
+					packet.ProtocolType = ms.ReadUInt16BE();
+					gseLength -= 2;
+					if (packet.LabelIndicator == 0)
+					{
+						packet.Label = new SixByteGseLabel(ms.ReadBytes(6));
+						gseLength -= 6;
+					}
+					else if (packet.LabelIndicator == 1)
+					{
+						packet.Label = new ThreeByteGseLabel(ms.ReadBytes(3));
+						gseLength -= 3;
+					}
+				}
+
+				if (!packet.StartIndicator && packet.EndIndicator)
+					gseLength -= 4;
+
+				if (gseLength < 0)
+				{
+					packet.Valid = false;
+					return packet;
+				}
+				if (gseLength > ms.GetAvailableBytes())
+				{
+					packet.Valid = false;
+					return packet;
+				}
+				packet.GseData = ms.ReadBytes(gseLength);
+
+				if (!packet.StartIndicator && packet.EndIndicator)
+					packet.Crc32 = ms.ReadUInt32BE();
+			}
+
+			packet.Valid = true;
+			return packet;
+		}
+
+		public uint? Crc32 { get; set; }
+
+		public byte[] GseData { get; set; }
+
+		public GseLabel Label { get; set; }
+		public ushort? ProtocolType { get; set; }
+
+		public ushort? TotalLength { get; set; }
+		public byte? FragmentId { get; set; }
+		public int LabelIndicator { get; private set; }
+		public bool EndIndicator { get; private set; }
+		public bool StartIndicator { get; private set; }
+		public bool IsPadding
+		{
+			get
+			{
+				return !StartIndicator && !EndIndicator && LabelIndicator == 0;
+			}
+		}
+
+		public bool IsCompletePacket
+		{
+			get
+			{
+				return StartIndicator && EndIndicator;
+			}
+		}
+	}
+
+	public abstract class GseLabel : Validatable
+	{
+		public abstract int LabelLength { get; }
+		public abstract string ToString();
+	}
+
+	public class ThreeByteGseLabel : GseLabel
+	{
+		public ThreeByteGseLabel(byte[] buffer)
+		{
+			if (buffer.Length != 3)
+			{
+				Valid = false;
+				return;
+			}
+
+			LabelBytes = buffer;
+			Valid = true;
+		}
+
+		public byte[] LabelBytes { get; private set; }
+
+		public override string ToString()
+		{
+			return BitConverter.ToString(LabelBytes);
+		}
+
+		public override int LabelLength => 3;
+	}
+
+	public class SixByteGseLabel : GseLabel
+	{
+		public SixByteGseLabel(byte[] buffer)
+		{
+			if (buffer.Length != 6)
+			{
+				Valid = false;
+				return;
+			}
+
+			MacAddress = new PhysicalAddress(buffer);
+			Valid = true;
+		}
+
+		public PhysicalAddress MacAddress { get; private set; }
+
+		public override string ToString()
+		{
+			return MacAddress.ToString();
+		}
+
+		public override int LabelLength => 6;
+	}
+}

skyscraper8/GSE/MisProcessor.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE
+{
+	public interface MisProcessor
+	{
+		public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer);
+	}
+}

skyscraper8/GSE/MisProcessors/0x00_Gse.cs

@@ -0,0 +1,131 @@
+using skyscraper5.Skyscraper.Net.Pcap;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE.MisProcessors
+{
+	internal class _0x00_Gse : MisProcessor
+	{
+			private byte[][] fragmentor;
+			private int[] fragmentorLength;
+			private int[] fragmentorPos;
+			private PcapWriter pcapWriter;
+
+			public _0x00_Gse()
+			{
+				fragmentor = new byte[256][];
+				fragmentorLength = new int[256];
+				fragmentorPos = new int[256];
+				FileStream fileStream = File.OpenWrite(string.Format("{0}.pcap", DateTime.Now.Ticks));
+				pcapWriter = new PcapWriter(fileStream, TcpdumpNetworkType.RawIp);
+			}
+
+			public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer)
+			{
+				int pos = 0;
+				while (pos < bblength - 4)
+				{
+					int gseLength = ((myLayer.Data[pos] & 0x0f) << 8) | (myLayer.Data[pos + 1]);
+					if ((myLayer.Data[pos] & 0xf0) == 0) break;
+					if (gseLength + 2 > bblength - pos) break;
+					if (!ProcessBbFrame(&myLayer.Data[pos], gseLength))
+						break;
+					pos += gseLength += 2;
+				}
+			}
+
+			private unsafe void Copy(byte* src, byte[] dst, int srcOffset, int dstOffset, int len)
+			{
+				for (int i = 0; i < len; i++)
+				{
+					dst[dstOffset + i] = src[srcOffset + i];
+				}
+			}
+
+			private unsafe bool ProcessBbFrame(byte* payload, int gseLength)
+			{
+				uint offset = 0;
+				uint fragId = 0;
+
+				if ((payload[0] & 0xc0) == 0x80)
+				{
+					fragId = payload[2];
+					int length = (payload[3] << 8) | payload[4];
+					if (fragmentor[fragId] != null)
+						fragmentor[fragId] = null;
+					fragmentor[fragId] = new byte[length + 2];
+					fragmentorLength[fragId] = length + 2;
+					fragmentor[fragId][0] = payload[0];
+					fragmentor[fragId][1] = payload[1];
+					fragmentor[fragId][0] |= 0xc0;
+					Copy(payload, fragmentor[fragId], 5, 2, gseLength - 3);
+					fragmentorPos[fragId] = gseLength - 1;
+				}
+				else if ((payload[0] & 0xc0) == 0x00)
+				{
+					fragId = payload[2];
+					if (fragmentor[fragId] == null)
+						return true;
+
+					Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 1);
+					fragmentorPos[fragId] += gseLength - 1;
+				}
+				else if ((payload[0] & 0xc0) == 0x40)
+				{
+					fragId = payload[2];
+					if (fragmentor[fragId] == null)
+						return true;
+
+					Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 5);
+					fragmentorPos[fragId] += gseLength - 1;
+					fixed (byte* p = fragmentor[fragId])
+					{
+						ProcessBbFrame(p, fragmentorLength[fragId]);
+					}
+					fragmentor[fragId] = null;
+
+				}
+				else if ((payload[0] & 0xc0) == 0xc0)
+				{
+					if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x04)
+					{
+						throw new NotImplementedException("ethertype 0x0400");
+					}
+					else if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x00)
+					{
+						throw new NotImplementedException("ethertype 0x0000");
+					}
+					else if (payload[offset + 2] == 0x08 && payload[offset + 3] == 0x00)
+					{
+						if ((payload[0] & 0x30) == 0x01)
+						{
+							offset += 3;    //3-byte label
+						}
+						else if ((payload[0] & 0x30) == 0x00)
+						{
+							offset += 6;    //MAC address
+						}
+
+						offset += 4;
+
+						//at this point, the IP header starts at "offset"
+						long sinkSize = gseLength + 2 - offset;
+						byte[] sink = new byte[sinkSize];
+						Copy(payload, sink, (int)offset, 0, (int)sinkSize);
+						pcapWriter.WritePacket(sink);
+						return true;
+					}
+
+					//throw new NotImplementedException("complete packet");
+				}
+				else if ((payload[0] & 0x0f0) == 0x00)
+				{
+					throw new NotImplementedException("padding packet");
+				}
+				return true;
+			}
+		}
+}

skyscraper8/GSE/MisProcessors/UnknownStreamType.cs

@@ -0,0 +1,38 @@
+using log4net;
+using skyscraper5.Skyscraper.Scraper;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE.MisProcessors
+{
+	internal class UnknownStreamType : MisProcessor
+	{
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		private bool[] warningsSpoken;
+		private byte mis;
+
+		public UnknownStreamType(byte streamType, byte mis)
+		{
+			this.mis = mis;
+			this.warningsSpoken = new bool[256];
+			this.EncounterPacket(streamType);
+		}
+
+		private void EncounterPacket(byte streamType)
+		{
+			if (!warningsSpoken[streamType])
+			{
+				logger.WarnFormat("This is a valid GS, however, it not GSE (type 0), but type {0}, which is not supported yet. Please consider sharing a sample of this stream.", streamType);
+				warningsSpoken[streamType] = true;
+			}
+		}
+
+		public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer)
+		{
+			EncounterPacket(myLayer.header.Sync);
+		}
+	}
+}

skyscraper8/GSE/NullGsEventHandler.cs

@@ -0,0 +1,16 @@
+using skyscraper5.Skyscraper;
+
+namespace skyscraper8.GSE;
+
+public class NullGsEventHandler : IGsEventHandler
+{
+    public void OnIpDatagram(int pid, byte[] payload)
+    {
+        
+    }
+
+    public void GsIpTrafficDetected()
+    {
+        
+    }
+}

skyscraper8/GSE/Pts2Bbf.cs

@@ -11,7 +11,7 @@ public class Pts2Bbf
 {
     private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
     
-    public static void Run(FileInfo file, bool useUdpDecap = false, IPEndPoint ipEndPoint = null)
+    public static void Run(FileInfo file, bool useUdpDecap = false)
     {
         if (!file.Exists)
         {
@@ -24,14 +24,10 @@ public class Pts2Bbf
         {
             string changeExtension = Path.ChangeExtension(file.FullName, ".sdecap");
             BbfUdpDecap.UdpDecapFileOutput udpDecapSink = new BbfUdpDecap.UdpDecapFileOutput(new FileInfo(changeExtension));
-
+            
-            if (ipEndPoint == null)
-            {
-                ipEndPoint = new IPEndPoint(IPAddress.Parse("239.199.2.1"), 1234);
-            }
             BbfUdpDecap bbfUdpDecap = new BbfUdpDecap();
-            bbfUdpDecap.SetTargetPort(ipEndPoint.Port);
+            bbfUdpDecap.SetTargetPort(1234);
-            bbfUdpDecap.SetTargetIp(ipEndPoint.Address);
+            bbfUdpDecap.SetTargetIp(IPAddress.Parse("239.199.2.1"));
             bbfUdpDecap.Sink = udpDecapSink;
             dumper = bbfUdpDecap;
         }
@@ -44,7 +40,7 @@ public class Pts2Bbf
         FileStream fileStream = file.OpenRead();
         
         TsContext mpeg2 = new TsContext();
-        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(dumper));
+        mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),dumper));
         DataStorage dataStorage = new InMemoryScraperStorage();
         ObjectStorage objectStorage = new NullObjectStorage();
         SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);

skyscraper8/GSE/Stid135BbFrameReader.cs

@@ -11,13 +11,15 @@ namespace skyscraper8.GSE
 {
 	internal class Stid135BbFrameReader : ITsPacketProcessor
 	{
+		private IGsEventHandler mpeEventHandler;
 		private IBbframeDeencapsulator deencapsulator;
 
-		public Stid135BbFrameReader(IBbframeDeencapsulator deencapsulator = null)
+		public Stid135BbFrameReader(IGsEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null)
 		{
 			if (deencapsulator == null)
-				deencapsulator = new BbframeDeencapsulator3();
+				deencapsulator = new BbframeDeencapsulator();
 			
+			this.mpeEventHandler = mpeEventHandler;
 			this.deencapsulator = deencapsulator;
 		}
 
@@ -54,6 +56,21 @@ namespace skyscraper8.GSE
 			{
 				outbuf.Write(packets, 9, packets[7] - 1);
 			}
+			/*if ((packets[8] & 0xff) == 0xb8)
+			{
+				if (outbuf != null)
+				{
+					byte[] chi = outbuf.ToArray();
+					deencapsulator.PushPacket(chi);
+				}
+				outbuf = new MemoryStream();
+				outbuf.Write(packets,9, packets[7] - 1);
+			}
+			else
+			{
+				if (outbuf != null)
+					outbuf.Write(packets, 9, packets[7] - 1);
+			}*/
 		}
 	}
 }

skyscraper8/GSE/Stid135TestingProgram.cs

@@ -0,0 +1,40 @@
+using log4net;
+using log4net.Repository.Hierarchy;
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper.Scraper;
+using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
+using skyscraper8.Skyscraper.Scraper.Storage;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE
+{
+	internal class Stid135TestingProgram
+	{
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		public static void Run(FileInfo file)
+		{
+			if (!file.Exists)
+			{
+				logger.Error("File not found: " + file.FullName);
+				return;
+			}
+
+			FileStream fileStream = file.OpenRead();
+
+			TsContext mpeg2 = new TsContext();
+			mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),new BbframeDeencapsulator2()));
+			DataStorage dataStorage = new InMemoryScraperStorage();
+			ObjectStorage objectStorage = new NullObjectStorage();
+			SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
+			skyscraper.InitalizeFilterChain();
+			skyscraper.IngestFromStream(fileStream);
+
+			fileStream.Close();
+			logger.Info("Stid135TestingProgram finished");
+		}
+	}
+}

skyscraper8/Program.cs

@@ -326,9 +326,9 @@ namespace skyscraper5
 
                 if (args[0].ToLowerInvariant().Equals("stid135test"))
                 {
-                    FileInfo fi = new FileInfo(args[1]);
+	                FileInfo fi = new FileInfo(args[1]);
-                    Stid135Test.Run(fi);
+	                Stid135TestingProgram.Run(fi);
-                    return;
+	                return;
                 }
 			}
 

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -92,7 +92,7 @@ namespace skyscraper5.Skyscraper.Scraper
         IEitEventHandler, IAitEventHandler, ISubtitleEventHandler,
         UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
         IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
-		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
+		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, IGsEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
         InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
@@ -301,7 +301,7 @@ namespace skyscraper5.Skyscraper.Scraper
                 {
                     if (!DvbContext.IsPidProcessorPresent(0x010e))
                     {
-                        DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
+                        DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
                         UiJunction?.SetGseMode();
                         LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                         SpecialTsType = 3;
@@ -356,7 +356,7 @@ namespace skyscraper5.Skyscraper.Scraper
                         {
                             if (!DvbContext.IsPidProcessorPresent(0x010e))
                             {
-                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
+                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
                                 UiJunction?.SetGseMode();
                                 LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                                 SpecialTsType = 3;