Don't crash on broken FLUTE messages. Instead, discard them.

Documentation/storage_system_ids.md

@@ -12,3 +12,10 @@ These contain blob-like data, like Screenshots, DSM-CC Carousels, etc.
 |--|----------|
 | 1|MinIO     |
 | 2|Filesystem|
+
+#IP Traffic handlers
+These will have IP packets forwarded to.
+|ID|Type       |
+|--|-----------|
+| 1|PCAP Writer|
+| 2|Discard    |

GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanConfigWindow.cs

@@ -170,6 +170,8 @@ namespace SDL2Demo.Jobs
 						ImGui.Text("This assumes that both tuners are connected to the same antenna set-up, using e.g. a splitter.");
 
 						ImGui.PushID("tunerB");
+						if (settingsWindowSetFilterTunerSelection - 1 < tuners.Count)
+							settingsWindowSetFilterTunerSelection = 0;
 						if (ImGui.BeginCombo("Tuner for BLScanEx and RFScan", tuners[settingsWindowSetFilterTunerSelection].Name))
 						{
 							for (int i = 0; i < tuners.Count; i++)

skyscraper8/DvbNip/DvbNipReceiver.cs

@@ -187,7 +187,6 @@ namespace skyscraper8.DvbNip
 						rawSlepString = rawSlepString.Replace("</dvbi-types:", "</");
 						rawSlepString = rawSlepString.Replace("<dvbisdt:", "<");
 						rawSlepString = rawSlepString.Replace("</dvbisdt:", "</");
-						File.WriteAllText("slep3.xml", rawSlepString);
 						ServiceListEntryPoints serviceListEntryPoints = DvbIUtils.UnpackServiceListEntryPoints(rawSlepString);
 						EventHandler?.OnServiceListEntryPoints(CurrentCarrierInformation, serviceListEntryPoints, currentTime.Value, this);
 						return true;

skyscraper8/Ietf/FLUTE/FluteListener.cs

@@ -72,6 +72,9 @@ namespace skyscraper8.Ietf.FLUTE
 			if (blocks == null)
 				blocks = new Dictionary<Tuple<ushort, ushort>, FluteBlock>();
 
+			if (lctFrame.FecHeader == null)
+				return;
+
 			ushort sbn = lctFrame.FecHeader.SourceBlockNumber;
 			ushort esi = lctFrame.FecHeader.EncodingSymbolId;
 

skyscraper8/Ietf/FLUTE/LctFrame.cs

@@ -1,9 +1,11 @@
 using skyscraper5.Skyscraper.IO;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper5.Skyscraper;
 
 namespace skyscraper8.Ietf.FLUTE
 {
@@ -25,7 +27,10 @@ namespace skyscraper8.Ietf.FLUTE
 			this.CloseObjectFlag = (byteB & 0x01) != 0;
 
 			if (this.Version != 1)
-				throw new NotSupportedException(String.Format("FLUTE Version {0}", Version));
+			{
+				Debug.WriteLine(String.Format("FLUTE Version {0}", Version));
+				return;
+			}
 
 			int HeaderLength = ms.ReadUInt8();
 			HeaderLength *= 4;
@@ -35,6 +40,8 @@ namespace skyscraper8.Ietf.FLUTE
 
 			byte[] headerBuffer = ms.ReadBytes(HeaderLength);
 			this.LctHeader = new LctHeader(headerBuffer,CongestionControlFlag,TransportSessionIdentifierFlag,TransportObjectIdentifierFlag,HalfWordFlag);
+			if (!this.LctHeader.Valid)
+				return;
 
 			if (ms.GetAvailableBytes() < 4)
 				return;

skyscraper8/Ietf/FLUTE/LctHeader.cs

@@ -2,13 +2,15 @@
 using skyscraper8.DvbNip;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper5.Skyscraper;
 
 namespace skyscraper8.Ietf.FLUTE
 {
-	internal class LctHeader
+	internal class LctHeader : Validatable
 	{
 		/// <summary>
 		/// 
@@ -32,6 +34,11 @@ namespace skyscraper8.Ietf.FLUTE
 			CongestionControlInformation = ReadField(ms, cciLength);
 			TransportSessionIdentifier = ReadField(ms, tsiLength);
 			TransportObjectIdentifier = ReadField(ms, toiLength);
+			if (CongestionControlInformation == UInt64.MaxValue || TransportSessionIdentifier == UInt64.MaxValue || TransportObjectIdentifier == UInt64.MaxValue)
+			{
+				Valid = false;
+				return;
+			}
 
 			while (ms.GetAvailableBytes() >= 1)
 			{
@@ -49,7 +56,9 @@ namespace skyscraper8.Ietf.FLUTE
 							this.ContentEncoding = new ContentEncodingOfFdtInstance(fixedHeaderExtension);
 							break;
 						default:
-							throw new NotImplementedException(String.Format("LCT Header Extension {0}", extensionId));
+							Debug.WriteLine(String.Format("LCT Header Extension {0}", extensionId));
+							Valid = false;
+							return;
 					}
 				}
 				else
@@ -58,6 +67,11 @@ namespace skyscraper8.Ietf.FLUTE
 					headerExtensionLength *= 4;
 					if (headerExtensionLength >= 2)
 						headerExtensionLength -= 2;
+					if (headerExtensionLength > ms.GetAvailableBytes())
+					{
+						Valid = false;
+						return;
+					}
 					byte[] extensionBuffer = ms.ReadBytes(headerExtensionLength);
 					switch(extensionId)
 					{
@@ -73,11 +87,15 @@ namespace skyscraper8.Ietf.FLUTE
 							this.NipActualCarrierInformation = new NipActualCarrierInformation(extensionBuffer);
 							break;
 						default:
-							throw new NotImplementedException(String.Format("LCT Header Extension {0}", extensionId));
+							Debug.WriteLine(String.Format("LCT Header Extension {0}", extensionId));
+							Valid = false;
+							return;
 					}
 				}
 				
 			}
+
+			Valid = true;
 		}
 
 		private ulong ReadField(Stream stream, int bits)
@@ -93,7 +111,7 @@ namespace skyscraper8.Ietf.FLUTE
 				case 64:
 					return stream.ReadUInt64BE();
 				default:
-					throw new NotImplementedException(String.Format("{0} bits.", bits));
+					return UInt64.MaxValue;
 			}
 		}
 		public ulong CongestionControlInformation { get; private set; }

skyscraper8/Program.cs

@@ -32,6 +32,7 @@ using skyscraper8;
 using skyscraper8.SatIp;
 using skyscraper8.SatIp.RtspResponses;
 using skyscraper8.SimpleServiceDiscoveryProtocol;
+using skyscraper8.Skyscraper.Math;
 
 [assembly: log4net.Config.XmlConfigurator(ConfigFile = "log4net.config")]
 namespace skyscraper5
@@ -39,19 +40,9 @@ namespace skyscraper5
     class Program
     {
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
-		private const int PUBLIC_RELEASE = 7;
+		private const int PUBLIC_RELEASE = 9;
 		private static void IntegrationTest()
 		{
-			PluginManager pluginManager = PluginManager.GetInstance();
-            StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
-            DataStorageFactory dataStorageFactory = storageConnectionManager.GetDefaultDataStorageFactory();
-            ObjectStorageFactory objectStorageFactory = storageConnectionManager.GetDefaultObjectStorageFactory();
-            M3U8Stream m3U8Stream = new M3U8Stream("D:\\NIP-Research\\nip.m3u8");
-
-            SkyscraperContext context = new SkyscraperContext(new TsContext(), dataStorageFactory.CreateDataStorage(), objectStorageFactory.CreateObjectStorage());
-            context.InitalizeFilterChain();
-            context.IngestFromStream(m3U8Stream);
-
             /*List<SsdpDevice> ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList();
             foreach (SsdpDevice ssdpDevice in ssdpDevices)
             {
@@ -282,7 +273,40 @@ namespace skyscraper5
 	                qadsipc.Run();
 	                return;
                 }
-            }
+
+                if (args[0].ToLowerInvariant().Equals("shannon"))
+                {
+	                if (args.Length != 2)
+	                {
+		                Console.WriteLine("Specify ONE file, please.");
+		                return;
+	                }
+	                FileInfo fi = new FileInfo(args[1]);
+	                if (!fi.Exists)
+	                {
+		                Console.WriteLine("{0} doesn't exist.", fi.FullName);
+		                return;
+	                }
+	                FileStream fileStream = fi.OpenRead();
+	                EntropyCalculatorStream entropyCalculatorStream = new EntropyCalculatorStream();
+	                fileStream.CopyTo(entropyCalculatorStream);
+                    fileStream.Close();
+                    fileStream.Dispose();
+                    Console.WriteLine("Entropy value: {0} ({1}%)", entropyCalculatorStream.Entropy, entropyCalculatorStream.Percentage);
+                    return;
+                }
+
+                if (args[0].ToLowerInvariant().Equals("pcapon"))
+                {
+	                TogglePcapConfiguration(1);
+	                return;
+                }
+                if (args[0].ToLowerInvariant().Equals("pcapoff"))
+                {
+	                TogglePcapConfiguration(2);
+	                return;
+                }
+			}
 
             /*Passing passing = new Passing();
             if (!passing.Boot())
@@ -296,12 +320,28 @@ namespace skyscraper5
             Console.WriteLine("         or: .\\skyscraper8.exe \"C:\\path\\to\\file.ts\\");
             Console.WriteLine("         or: .\\skyscraper8.exe \"C:\\path\\to\\my\\folder\\with\\ts\\files\\\" (for batch extraction)");
             Console.WriteLine("         or: .\\skyscraper8.exe satip IP_ADDRESS DISEQC POLARITY FREQUENCY SYSTEM SYMBOL_RATE (see README file)");
+            Console.WriteLine("         or: .\\skyscraper8.exe pcapon - to write a configuration file that allows PCAP writing during scraping.");
+            Console.WriteLine("         or: .\\skyscraper8.exe pcapoff - to write a configuration file that turns off PCAP writing during scraping.");
             Console.WriteLine();
-            Console.WriteLine("Bonus feature:");
+            Console.WriteLine("default behaviour is pcap writing on.");
+			Console.WriteLine();
+            Console.WriteLine("Bonus features:");
             Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC.");
             Console.WriteLine(".\\skyscraper8.exe hlsproxy-destructive \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC and delete HLS segments afterwards. (be careful!)");
+            Console.WriteLine(".\\skyscraper8.exe shannon \"C:\\some\\file.bmp\" - calculates the Shannon entropy value for any given file.");
+		}
+
+		private static void TogglePcapConfiguration(int value)
+		{
+			PluginManager pluginManager = PluginManager.GetInstance();
+			if (!pluginManager.Ini.ContainsKey("ip_handler"))
+				pluginManager.Ini.Add("ip_handler", new IniSection());
+
+			pluginManager.Ini["ip_handler"]["type"] = value.ToString();
+			pluginManager.SaveConfiguration();
+
+            Console.WriteLine("Wrote skyscraper5.ini.");
 		}
-        
         private static void ProcessDirectory(DirectoryInfo di)
         {
             DataStorage dataStorage = new InMemoryScraperStorage();

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "cscan tcp://172.20.20.203:6969",
+      "commandLineArgs": "satip auto 1 H 11141 S2 23500",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/QuickAndDirtySatIpClient.cs

@@ -20,6 +20,7 @@ namespace skyscraper8
 	internal class QuickAndDirtySatIpClient
 	{
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		private const bool ENABLE_TS_WRITER = false;
 
 		public QuickAndDirtySatIpClient(string[] args)
 		{
@@ -105,6 +106,11 @@ namespace skyscraper8
 
 			packetQueue = new Queue<byte[]>();
 			RtspSetupResponse setup = rtspClient.GetSetup(url);
+			if (setup.RtspStatusCode == 404)
+			{
+				logger.Fatal("Your SAT>IP server doesn't have a tuner available that can talk to the requested frequency.");
+				return;
+			}
 			setup.OnRtcpPacket += Setup_OnRtcpPacket;
 			setup.OnRtpPacket += Setup_OnRtpPacket;
 
@@ -114,7 +120,7 @@ namespace skyscraper8
 				objectStorage = new FilesystemStorage(new DirectoryInfo("."));
 			context = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
 			context.EnableTimeout = true;
-			context.TimeoutSeconds = 10;
+			context.TimeoutSeconds = 60;
 			context.InitalizeFilterChain();
 
 			RtspPlayResponse play = rtspClient.GetPlay(setup);
@@ -150,6 +156,7 @@ namespace skyscraper8
 			rtspClient.Dispose();
 		}
 
+		private FileStream fs;
 		private uint stuffingBytes;
 		private Queue<byte[]> packetQueue;
 		private ObjectStorage objectStorage;
@@ -167,6 +174,8 @@ namespace skyscraper8
 					{
 						packetQueue.Enqueue(buffer);
 					}
+
+					DumpPacket(buffer);
 					i += 187;
 				}
 				else if (data[i] == 0xff)
@@ -176,6 +185,20 @@ namespace skyscraper8
 			}
 		}
 
+		private void DumpPacket(byte[] buffer)
+		{
+			if (!ENABLE_TS_WRITER)
+				return;
+
+			if (fs == null)
+			{
+				string fname = String.Format("{0}.ts", DateTime.Now.Ticks);
+				fs = File.OpenWrite(fname);
+			}
+
+			fs.Write(buffer, 0, buffer.Length);
+		}
+
 		private int rtcps;
 		private void Setup_OnRtcpPacket(byte[] data, int length)
 		{

skyscraper8/Skyscraper/BbframeDeencapsulator.cs

@@ -0,0 +1,326 @@
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper;
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using System.Runtime.Serialization;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper
+{
+	internal class BbframeDeencapsulator
+	{
+		private bool interruptedGseHem;
+		private MemoryStream interruptedGseHemBuffer;
+		private bool interruptedGseHemWantsCrc32;
+		public IGsEventHandler MpeEventHandler { get; set; }
+
+		public void PushPacket(byte[] bbframe)
+		{
+			MemoryStream ms = new MemoryStream(bbframe, false);
+			//BBHeader
+			byte matype1 = ms.ReadUInt8();
+			int tsGsField = (matype1 & 0xc0) >> 6;
+			bool sisMisField = (matype1 & 0x20) != 0;
+			bool ccmAcmField = (matype1 & 0x10) != 0;
+			bool issyi = (matype1 & 0x08) != 0;
+			bool npd = (matype1 & 0x04) != 0;
+			int ro = (matype1 & 0x03);
+
+			byte matype2 = ms.ReadUInt8();
+
+			ushort userPacketLength = ms.ReadUInt16BE();
+			ushort dataFieldLength = ms.ReadUInt16BE();
+			byte sync = ms.ReadUInt8();
+			ushort syncd = ms.ReadUInt16BE();
+			byte crc8 = ms.ReadUInt8();
+
+			switch (tsGsField)
+			{
+				case 2:
+					int bytes = dataFieldLength / 8;
+					if (ms.GetAvailableBytes() < bytes)
+						return;
+					HandleGse(ms);
+					break;
+				default:	//0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts
+					throw new NotImplementedException(String.Format("TS/GS field says 0x{0:X2}", tsGsField));
+			}
+		}
+
+		private bool gseNeedMore;
+		private MemoryStream gseAssembler;
+		public int PID;
+
+		private void HandleGse(MemoryStream ms)
+		{
+			ms.Position = 10;
+			byte syncByte = ms.ReadUInt8();
+			ms.Position = 10;
+
+			long availableBytes = ms.GetAvailableBytes();
+			byte[] readBytes = ms.ReadBytes(availableBytes);
+			
+			GseAssemblyState assemblyState = ValidateGse(gseAssembler);
+			if (assemblyState == GseAssemblyState.NOT_YET_BEGUN)
+			{
+				if ((syncByte & 0xc0) == 0xc0)
+				{
+					//Aktuell ist der GSE Assembler nicht beschäftigt, aber wir haben ein Sync-Byte, also kann er loslegen.
+					gseAssembler = new MemoryStream();
+					gseAssembler.Write(readBytes, 0, readBytes.Length);
+					return;
+				}
+				else
+				{
+					//Aktuell ist der GSE Assembler nicht beschäftigt, und wir haben kein gültiges Sync-Byte, also weg damit.
+					//Console.WriteLine("gse not in sync yet. sync byte = {0}", syncByte);
+					return;
+				}
+			}
+			else if (assemblyState == GseAssemblyState.NEED_MORE_DATA)
+			{
+				gseAssembler.Write(readBytes, 0, readBytes.Length);
+				return;
+			}
+			else if (assemblyState == GseAssemblyState.VALID)
+			{
+				gseAssembler.Position = 0;
+				AssembleGse(gseAssembler);
+				//Console.WriteLine("assembled {0} next sync byte is {1}", sucessfulAssembles++,syncByte);
+				if (sucessfulAssembles == 181)
+				{
+
+				}
+				gseAssembler = null;
+				HandleGse(ms);
+				return;
+			}
+			else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE)
+			{
+				gseAssembler = null;
+				HandleGse(ms);
+				return;
+			}
+			else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH)
+			{
+				gseAssembler = null;
+				HandleGse(ms);
+				return;
+			}
+			else
+			{
+				throw new NotImplementedException(String.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
+			}
+		}
+
+		private long sucessfulAssembles;
+		enum GseAssemblyState
+		{
+			NOT_YET_BEGUN,
+			VALID,
+			BROKEN_NEGATIVE_LENGTH,
+			NEED_MORE_DATA,
+			BROKEN_SUSPECT_LENGTH,
+			BROKEN_SUSPECT_ETHERTYPE
+		}
+
+		private GseAssemblyState ValidateGse(MemoryStream ms)
+		{
+			if (ms == null)
+				return GseAssemblyState.NOT_YET_BEGUN;
+
+			long backupPosition = ms.Position;
+			ms.Position = 0;
+			while (ms.GetAvailableBytes() > 2)
+			{
+				//GSE-Header
+				byte byteA = ms.ReadUInt8();
+				bool startIndicator = (byteA & 0x80) != 0;
+				bool endIndicator = (byteA & 0x40) != 0;
+				int labelIndicator = (byteA & 0x30) >> 4;
+				if (!startIndicator && !endIndicator && labelIndicator == 0)
+				{
+					//end of base band frame
+					ms.Position = backupPosition;
+					return GseAssemblyState.VALID;
+				}
+				else
+				{
+					byte byteB = ms.ReadUInt8();
+					int gseLength = byteA & 0x0f;
+					gseLength <<= 8;
+					gseLength += byteB;
+					//Console.WriteLine("GSE Length: {0}", gseLength);
+					if (gseLength > 9000)
+					{
+						ms.Position = backupPosition;
+						return GseAssemblyState.BROKEN_SUSPECT_LENGTH;
+					}
+
+					if (!startIndicator || !endIndicator)
+					{
+						byte fragId = ms.ReadUInt8();
+						gseLength--;
+					}
+
+					if (startIndicator && !endIndicator)
+					{
+						ushort totalLength = ms.ReadUInt16BE();
+						gseLength -= 2;
+					}
+
+					if (startIndicator)
+					{
+						if (2 > ms.GetAvailableBytes())
+						{
+							ms.Position = backupPosition;
+							return GseAssemblyState.NEED_MORE_DATA;
+						}
+						ushort protocolType = ms.ReadUInt16BE();
+						if (!ValidateEthertype(protocolType))
+						{
+							ms.Position = backupPosition;
+							return GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE;
+						}
+						gseLength -= 2;
+						if (labelIndicator == 0)
+						{
+							if (6 > ms.GetAvailableBytes())
+							{
+								ms.Position = backupPosition;
+								return GseAssemblyState.NEED_MORE_DATA;
+							}
+							PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
+							gseLength -= 6;
+						}
+						else if (labelIndicator == 1)
+						{
+							byte[] threeByteLabel = ms.ReadBytes(3);
+							gseLength -= 3;
+						}
+					}
+
+					if (!startIndicator && endIndicator)
+						gseLength -= 4;
+
+					int startCrc32 = (int)ms.Position;
+
+					if (gseLength < 0)
+					{
+						ms.Position = backupPosition;
+						return GseAssemblyState.BROKEN_NEGATIVE_LENGTH;
+					}
+
+					if (gseLength > ms.GetAvailableBytes())
+					{
+						ms.Position = backupPosition;
+						return GseAssemblyState.NEED_MORE_DATA;
+					}
+
+					ms.Position += gseLength;
+					if (!startIndicator && endIndicator)
+					{
+						uint crc32 = ms.ReadUInt32BE();
+						int endCrc32 = (int)ms.Position;
+						DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
+					}
+				}
+			}
+
+			ms.Position = backupPosition;
+			return GseAssemblyState.VALID;
+		}
+
+		private bool ValidateEthertype(ushort ethertype)
+		{
+			if (ethertype == 0x0800)
+				return true;			//IPv4
+			if (ethertype == 0x86DD)
+				return true;			//IPv6
+			else
+				return false;			//There are many more valid values, but since we don't speak anything aside from IPv4/IPv6, we discard it.
+		}
+		private void AssembleGse(MemoryStream ms)
+		{
+			ushort protocolType = 0;
+			while (ms.GetAvailableBytes() > 2)
+			{
+				//GSE-Header
+				byte byteA = ms.ReadUInt8();
+				bool startIndicator = (byteA & 0x80) != 0;
+				bool endIndicator = (byteA & 0x40) != 0;
+				int labelIndicator = (byteA & 0x30) >> 4;
+				if (!startIndicator && !endIndicator && labelIndicator == 0)
+				{
+					//end of base band frame
+					return;
+				}
+				else
+				{
+					byte byteB = ms.ReadUInt8();
+					int gseLength = byteA & 0x0f;
+					gseLength <<= 8;
+					gseLength += byteB;
+
+					if (!startIndicator || !endIndicator)
+					{
+						byte fragId = ms.ReadUInt8();
+						gseLength--;
+					}
+
+					if (startIndicator && !endIndicator)
+					{
+						ushort totalLength = ms.ReadUInt16BE();
+						gseLength -= 2;
+					}
+
+					if (startIndicator)
+					{
+						protocolType = ms.ReadUInt16BE();
+						gseLength -= 2;
+						if (labelIndicator == 0)
+						{
+							PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
+							gseLength -= 6;
+						}
+						else if (labelIndicator == 1)
+						{
+							byte[] threeByteLabel = ms.ReadBytes(3);
+							gseLength -= 3;
+						}
+					}
+
+					if (!startIndicator && endIndicator)
+						gseLength -= 4;
+
+					int startCrc32 = (int)ms.Position;
+					
+					byte[] payload = ms.ReadBytes(gseLength);
+					if (!startIndicator && endIndicator)
+					{
+						uint crc32 = ms.ReadUInt32BE();
+						int endCrc32 = (int)ms.Position;
+						DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
+					}
+
+					switch (protocolType)
+					{
+						case 0x0800:	//IPv4
+							MpeEventHandler.OnIpDatagram(PID, payload);
+							break;
+						default:
+							//throw new NotImplementedException(String.Format("Unknown GSE Protocol ID 0x{0:X4}", protocolType));
+							break;
+					}
+				}
+			}
+		}
+
+
+	}
+}

skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs

@@ -6,18 +6,22 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper8.Skyscraper;
 
 namespace skyscraper5.src.Skyscraper
 {
 	internal class DigitalDevicesBbFrameReader : ITsPacketProcessor
 	{
 		private IGsEventHandler mpeEventHandler;
+		private BbframeDeencapsulator deencapsulator;
 
 		public DigitalDevicesBbFrameReader(IGsEventHandler mpeEventHandler)
 		{
 			this.mpeEventHandler = mpeEventHandler;
 		}
 
+		private long packetsRecovered;
+		private long packetsLost;
 		private MemoryStream outbuf;
 		private bool annoncementDone;
 		public void PushPacket(TsPacket packet)
@@ -35,19 +39,17 @@ namespace skyscraper5.src.Skyscraper
 				if (outbuf != null)
 				{
 					byte[] chi = outbuf.ToArray();
-					byte[] ipPacket = IpPacketFinder.LookForIpPacket(outbuf.ToArray(), 32);
-					if (ipPacket != null)
+					if (deencapsulator == null)
 					{
-						if (!annoncementDone)
-						{
-							mpeEventHandler.GsIpTrafficDetected();
-							annoncementDone = true;
-						}
-						mpeEventHandler.OnIpDatagram(0x0118, ipPacket);
+						deencapsulator = new BbframeDeencapsulator();
+						deencapsulator.MpeEventHandler = mpeEventHandler;
+						deencapsulator.PID = pid;
 					}
+
+					deencapsulator.PushPacket(chi);
 				}
 				outbuf = new MemoryStream();
-				outbuf.Write(packets,8, packets[7]);
+				outbuf.Write(packets,9, packets[7] - 1);
 			}
 			else
 			{

skyscraper8/Skyscraper/IO/M3U8Stream.cs

@@ -131,7 +131,7 @@ namespace skyscraper8.Skyscraper.IO
 				segmentNames = newLines;
 				return true;
 			}
-			for (int i = 0; i < Math.Min(newLines.Length, segmentNames.Length); i++)
+			for (int i = 0; i < System.Math.Min(newLines.Length, segmentNames.Length); i++)
 			{
 				if (!segmentNames[i].Equals(newLines[i]))
 				{

skyscraper8/Skyscraper/IpPacketFinder.cs

@@ -13,7 +13,7 @@ namespace skyscraper5.Skyscraper
         {
             byte[] ipBuffer = new byte[20];
 
-            for (int i = 1; i < searchDepth; i++)
+            for (int i = 0; i < searchDepth; i++)
             {
                 int v4HeaderEnd = i + 20;
                 if (v4HeaderEnd > gsPacket.Length)
@@ -39,5 +39,40 @@ namespace skyscraper5.Skyscraper
 
             return null;
         }
+
+        public static List<byte[]> LookForIpPackets(byte[] gsPacket)
+        {
+	        List<byte[]> result = null;
+
+	        byte[] ipBuffer = new byte[20];
+	        for (int offset = 0; offset < gsPacket.Length; offset++)
+	        {
+		        int v4HeaderEnd = offset + 20;
+		        if (v4HeaderEnd > gsPacket.Length)
+			        break;
+
+		        if (gsPacket[offset] == 0x45)
+		        {
+			        Array.Copy(gsPacket, offset, ipBuffer, 0, ipBuffer.Length);
+			        InternetHeader ipv4 = new(ipBuffer);
+			        if (!ipv4.ChecksumValid)
+				        continue;
+			        int payloadStart = offset + ipBuffer.Length;
+			        int payloadEnd = offset + ipv4.TotalLength;
+			        if (payloadEnd > gsPacket.Length)
+				        break;
+			        int packetLength = payloadEnd - offset;
+
+			        byte[] finalPacket = new byte[packetLength];
+			        Array.Copy(gsPacket, offset, finalPacket, 0, packetLength);
+			        if (result == null)
+				        result = new List<byte[]>();
+			        result.Add(finalPacket);
+			        offset = payloadEnd;
+		        }
+	        }
+
+	        return result;
+        }
     }
 }

skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Math
+{
+	internal class EntropyCalculatorStream : Stream
+	{
+		public override void Flush()
+		{
+			//no actual writes done, so nothing to flush.
+		}
+
+		public override int Read(byte[] buffer, int offset, int count)
+		{
+			//this is a sink, so nothing to read here.
+			throw new NotSupportedException();
+		}
+
+		public override long Seek(long offset, SeekOrigin origin)
+		{
+			if (offset == 0)
+				return 0;
+
+			//this is a sink, so we can't do actual seeks
+			throw new NotSupportedException();
+		}
+
+		public override void SetLength(long value)
+		{
+			//TODO: allow this later, although I have no idea why one would actually want this.
+			throw new NotSupportedException();
+		}
+
+		private double _entropy;
+		private long[] freq;
+		private long internalPosition;
+		public override void Write(byte[] buffer, int offset, int count)
+		{
+			//Count the bytes
+			if (freq == null)
+				freq = new long[byte.MaxValue + 1];
+			for (int i = 0; i < count; i++)
+				freq[buffer[i + offset]]++;
+			internalPosition += count;
+
+
+			//Actually calculate the entropy here.
+			double newEntropy = 0.0;
+			for (int i = 0; i < freq.Length; i++)
+			{
+				double p_i = (double)freq[i] / (double)internalPosition;
+				if (p_i > 0)
+					newEntropy -= p_i * System.Math.Log2(p_i);
+			}
+
+			//Raise events if we feel like it
+			if (newEntropy != Entropy)
+			{
+				double oldEntropy = Entropy;
+				double oldPercentage = Percentage;
+				double newPercentage = (newEntropy / 8.0) * 100.0;
+				if (newEntropy > Entropy)
+					EntropyRising?.Invoke(oldEntropy, newEntropy, oldPercentage, newPercentage);
+				else if (newEntropy < Entropy)
+					EntropyFalling?.Invoke(oldEntropy, newEntropy, oldPercentage, newPercentage);
+			}
+
+			_entropy = newEntropy;
+		}
+
+		public override bool CanRead => false;
+		public override bool CanSeek => false;
+		public override bool CanWrite => true;
+		public override long Length => internalPosition;
+		public override long Position
+		{
+			get
+			{
+				return internalPosition;
+			}
+			set
+			{
+				throw new NotSupportedException();
+			}
+		}
+		public double Entropy
+		{
+			get
+			{
+				return _entropy;
+			}
+		}
+
+		public double Percentage
+		{
+			get
+			{
+				return (Entropy / 8.0) * 100.0;
+			}
+		}
+
+		public delegate void EntropyCallback(double oldEntropy, double newEntropy, double oldPercentage, double newPercentage);
+
+		public event EntropyCallback EntropyRising;
+		public event EntropyCallback EntropyFalling;
+	}
+}

skyscraper8/Skyscraper/Net/IpTrafficHandler.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Net
+{
+	internal interface IpTrafficHandler : IDisposable
+	{
+		void HandlePacket(int pid, byte[] payload);
+	}
+}

skyscraper8/Skyscraper/Net/NullIpTrafficHandler.cs

@@ -0,0 +1,27 @@
+using log4net;
+using skyscraper5.Skyscraper.Plugins;
+using skyscraper8.Skyscraper.Scraper.Storage;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Net
+{
+	[SkyscraperPlugin]
+	[StorageId(2)]
+	[StorageName("Packet Discarder")]
+	internal class NullIpTrafficHandler : IpTrafficHandler
+	{
+		public void Dispose()
+		{
+			
+		}
+
+		public void HandlePacket(int pid, byte[] payload)
+		{
+			
+		}
+	}
+}

skyscraper8/Skyscraper/Net/Pcap/PcapIpTrafficHandler.cs

@@ -0,0 +1,46 @@
+using log4net;
+using skyscraper5.Skyscraper.Net.Pcap;
+using skyscraper5.Skyscraper.Plugins;
+using skyscraper8.Skyscraper.Scraper.Storage;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Net.Pcap
+{
+	[SkyscraperPlugin]
+	[StorageId(1)]
+	[StorageName("PCAP Writer")]
+	internal class PcapIpTrafficHandler : IpTrafficHandler
+	{
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		private PcapWriter[] pcapWriters;
+		public void HandlePacket(int pid, byte[] payload)
+		{
+			if (pcapWriters == null)
+				pcapWriters = new PcapWriter[0x1fff];
+			if (pcapWriters[pid] == null)
+			{
+				string fname = String.Format("{0:X4},{1}.pcap", pid, DateTime.Now.Ticks);
+				logger.InfoFormat("Opening file for writing: {0}", fname);
+				FileStream fs = File.OpenWrite(fname);
+				pcapWriters[pid] = new PcapWriter(fs, TcpdumpNetworkType.RawIp);
+			}
+			pcapWriters[pid].WritePacket(payload);
+		}
+
+		public void Dispose()
+		{
+			if (pcapWriters != null)
+			{
+				for (int i = 0; i < pcapWriters.Length; i++)
+				{
+					if (pcapWriters[i] != null)
+						pcapWriters[i].Dispose();
+				}
+			}
+		}
+	}
+}

skyscraper8/Skyscraper/Plugins/PluginManager.cs

@@ -17,6 +17,9 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using skyscraper8.Skyscraper.Net;
+using skyscraper8.Skyscraper.Net.Pcap;
+using skyscraper8.Skyscraper.Plugins;
 using skyscraper8.Skyscraper.Scraper.Storage;
 using skyscraper8.Skyscraper.Text;
 
@@ -42,8 +45,11 @@ namespace skyscraper5.Skyscraper.Plugins
 			_mpeg2ExtensionDescriptors = new ConstructorInfo[256];
 
 			ScanAssembly(this.GetType().Assembly);
-			
-			FileInfo iniFileInfo = new FileInfo(iniFilename);
+
+			FileInfo skyscraperMainAssembly = GetSkyscraperMainAssembly();
+			DirectoryInfo skyscraperHome = skyscraperMainAssembly.Directory;
+
+			iniFileInfo = new FileInfo(Path.Combine(skyscraperHome.FullName, iniFilename));
 			if (iniFileInfo.Exists)
 			{
 				Ini = new Ini(iniFileInfo.FullName);
@@ -76,12 +82,13 @@ namespace skyscraper5.Skyscraper.Plugins
 			else
 			{
 				Debug.WriteLine(String.Format("{0} was not found. Create it using the UI!", iniFileInfo.FullName));
+				Ini = new Ini();
 			}
 
-			FileInfo fi = GetSkyscraperMainAssembly();
-			if (fi != null)
+			
+			if (skyscraperMainAssembly != null)
 			{
-				DirectoryInfo directory = fi.Directory;
+				DirectoryInfo directory = skyscraperMainAssembly.Directory;
 				logger.DebugFormat("Found skyscraper main assembly at: {0}", directory.FullName);
 				FileInfo[] fileInfos = directory.GetFiles("skyscraper5.*.dll");
 				foreach (FileInfo fileInfo in fileInfos)
@@ -173,6 +180,7 @@ namespace skyscraper5.Skyscraper.Plugins
 		}
 
 
+		private FileInfo iniFileInfo;
 		private const string iniFilename = "skyscraper5.ini";
 		private static PluginManager _instance;
 
@@ -199,6 +207,7 @@ namespace skyscraper5.Skyscraper.Plugins
 		private Type filesystemProcessorType = typeof(FilesystemProcessorPlugin);
 		private Type textDecoderType = typeof(TextDecoder);
 		private PluginPrioritySorter sorter = new PluginPrioritySorter();
+		private Type ipTrafficHandler = typeof(IpTrafficHandler);
 
 		private List<IDnsParser> _dnsParsers;
 		private List<ISkyscraperMpePlugin> _mpePlugins;
@@ -214,6 +223,7 @@ namespace skyscraper5.Skyscraper.Plugins
 		private ConstructorInfo[] _dsmCcMessages;
 		private ConstructorInfo[] _mpeg2ExtensionDescriptors;
 		private TextDecoder[] _textDecoders;
+		private Dictionary<int, Type> _ipTrafficHandles;
 
 		private void ScanAssembly(Assembly assembly)
 		{
@@ -319,13 +329,36 @@ namespace skyscraper5.Skyscraper.Plugins
 					HandleTextDecoder(type);
 					continue;
 				}
+				else if (type.IsAssignableTo(ipTrafficHandler))
+				{
+					HandleIpTrafficHandler(type);
+					continue;
+				}
 
-					throw new NotImplementedException();
+				throw new NotImplementedException();
 			}
 			
 			_mpePlugins.Sort(sorter);
 		}
 
+		private void HandleIpTrafficHandler(Type type)
+		{
+			Attribute attribute = type.GetCustomAttribute(typeof(StorageIdAttribute));
+			if (attribute == null)
+				throw new PluginsException(String.Format("The type {0} does not have a {1}", type.Name, typeof(StorageIdAttribute)));
+
+			if (_ipTrafficHandles == null)
+				_ipTrafficHandles = new Dictionary<int, Type>();
+
+			StorageIdAttribute sia = (StorageIdAttribute)attribute;
+			if (_ipTrafficHandles.ContainsKey(sia.Id))
+			{
+				throw new PluginsException(String.Format("Multiple {0} claim to have ID {1}. {2} <-> {3}", nameof(IpTrafficHandler), sia.Id, _ipTrafficHandles[sia.Id].Name,type.Name));
+			}
+
+			_ipTrafficHandles.Add(sia.Id, type);
+		}
+
 		private void HandleTextDecoder(Type type)
 		{
 			List<EncodingTypeIdAttribute> idAttributes = type.GetCustomAttributes<EncodingTypeIdAttribute>().ToList();
@@ -701,6 +734,11 @@ namespace skyscraper5.Skyscraper.Plugins
 			return _dataStorages.AsReadOnly();
 		}
 
+		public ReadOnlyDictionary<int, Type> GetIpTrafficHandlers()
+		{
+			return _ipTrafficHandles.AsReadOnly();
+		}
+
 		public void AutoconfigureObject(string categoryName, object targetObject)
 		{
 			if (Ini == null)
@@ -743,5 +781,10 @@ namespace skyscraper5.Skyscraper.Plugins
 			TextDecoder[] saneClone = (TextDecoder[])clone;
 			return saneClone;
 		}
+
+		public void SaveConfiguration()
+		{
+			Ini.Export(iniFileInfo);
+		}
 	}
 }

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -78,6 +78,7 @@ using System.Security.Policy;
 using System.Text;
 using skyscraper8.Experimentals.NdsSsu;
 using skyscraper8.Experimentals.OtvSsu;
+using skyscraper8.Skyscraper.Net;
 using skyscraper8.Skyscraper.Scraper;
 using Tsubasa.IO;
 using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
@@ -95,7 +96,7 @@ namespace skyscraper5.Skyscraper.Scraper
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
         public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
-        public const bool ENABLE_MPE_TO_PCAP = true;
+
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
 
 		public TsContext DvbContext { get; }
@@ -1658,20 +1659,17 @@ namespace skyscraper5.Skyscraper.Scraper
             return;
         }
 
-        private PcapWriter[] pcapWriters;
+
+        private IpTrafficHandler ipTrafficHandler;
         public void OnIpDatagram(int pid, byte[] payload)
         {
-            if (ENABLE_MPE_TO_PCAP)
-            {
-                if (pcapWriters == null)
-                    pcapWriters = new PcapWriter[0x1fff];
-                if (pcapWriters[pid] == null)
-                {
-                    FileStream fs = File.OpenWrite(String.Format("{0:X4},{1}.pcap", pid,DateTime.Now.Ticks));
-                    pcapWriters[pid] = new PcapWriter(fs, TcpdumpNetworkType.RawIp);
-                }
-                pcapWriters[pid].WritePacket(payload);
-            }
+	        if (ipTrafficHandler == null)
+	        {
+		        StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
+		        ipTrafficHandler = storageConnectionManager.GetDefaultIpTrafficHandler();
+	        }
+
+	        ipTrafficHandler?.HandlePacket(pid, payload);
 
             int ipVersion = (payload[0] & 0xf0) >> 4;
             if (ipVersion == 4)
@@ -2039,19 +2037,12 @@ namespace skyscraper5.Skyscraper.Scraper
             if (TcpProxyEnabled)
                 TcpProxyEnabled = false;
 
-            
 
-            if (ENABLE_MPE_TO_PCAP)
+            if (ipTrafficHandler != null)
             {
-                if (pcapWriters != null)
-                {
-                    for (int i = 0; i < pcapWriters.Length; i++)
-                    {
-                        if (pcapWriters[i] != null)
-                            pcapWriters[i].Dispose();
-                    }
-                }
+                ipTrafficHandler.Dispose();
             }
+            
 
             if (DvbContext.TcpProxyEnabled)
             {

skyscraper8/Skyscraper/Scraper/Storage/StorageConnectionManager.cs

@@ -8,7 +8,10 @@ using System.Linq;
 using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper5.Skyscraper.Net;
 using skyscraper5.Skyscraper.Scraper.Storage;
+using skyscraper8.Skyscraper.Net;
+using skyscraper8.Skyscraper.Net.Pcap;
 
 namespace skyscraper8.Skyscraper.Scraper.Storage
 {
@@ -139,5 +142,50 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 		{
 			return plugins.GetDataStorages();
 		}
+
+		private bool IsIpTafficHandlerConfigured()
+		{
+			if (ini == null)
+				return false;
+			if (!ini.ContainsKey("ip_handler"))
+				return false;
+			if (!ini["ip_handler"].ContainsKey("type"))
+				return false;
+			return true;
+		}
+		
+		
+		internal IpTrafficHandler GetDefaultIpTrafficHandler()
+		{
+			Type targetType;
+			if (IsIpTafficHandlerConfigured())
+			{
+				ReadOnlyDictionary<int, Type> handlers = plugins.GetIpTrafficHandlers();
+				int type = Int32.Parse(ini["ip_handler"]["type"]);
+				if (!handlers.ContainsKey(type))
+				{
+					logger.WarnFormat("The IP traffic handler with ID {0} was not found. I'm gonna assume you want the pcap files.", type);
+					targetType = typeof(PcapIpTrafficHandler);
+				}
+				else
+				{
+					targetType = handlers[type];
+				}
+			}
+			else
+			{
+				logger.WarnFormat("You didn't call \"pcapon\" or \"pcapoff\" before. I'm gonna assume you want the pcap files.");
+				targetType = typeof(PcapIpTrafficHandler);
+			}
+
+			ConstructorInfo constructorInfo = targetType.GetConstructor(new Type[] { });
+			if (constructorInfo == null)
+			{
+				throw new ScraperStorageException(String.Format("The type {0} doesn't contain a parameterless constructor.", targetType.Name));
+			}
+
+			object invoke = constructorInfo.Invoke(new object[] { });
+			return (IpTrafficHandler) invoke;
+		}
 	}
 }