Merge branch 'master' of http://172.20.20.19:3000/ft/skyscraper8

skyscraper8/DvbNip/DvbNipReceiver.cs

@@ -108,9 +108,9 @@ namespace skyscraper8.DvbNip
 					if (isValidXml)
 					{
 						FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream);
-						EventHandler.FluteFileAnnouncement(fluteCoordinate.Item1, fluteCoordinate.Item2, fdtAnnouncement);
 						if (fdtAnnouncement != null)
 						{
+							EventHandler.FluteFileAnnouncement(fluteCoordinate.Item1, fluteCoordinate.Item2, fdtAnnouncement);
 							SetFileAssociations(fluteListener, fdtAnnouncement);
 						}
 					}

skyscraper8/GSE/BbframeDeencapsulator.cs

@@ -11,7 +11,7 @@ using System.Text;
 using System.Threading.Tasks;
 using log4net;
 
-namespace skyscraper8.Skyscraper
+namespace skyscraper8.GSE
 {
 	internal class BbframeDeencapsulator
 	{
@@ -22,6 +22,8 @@ namespace skyscraper8.Skyscraper
 
         private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
 
+        private bool shownNonGseWarning;
+
         public void PushPacket(byte[] bbframe)
 		{
 			MemoryStream ms = new MemoryStream(bbframe, false);
@@ -32,7 +34,7 @@ namespace skyscraper8.Skyscraper
 			bool ccmAcmField = (matype1 & 0x10) != 0;
 			bool issyi = (matype1 & 0x08) != 0;
 			bool npd = (matype1 & 0x04) != 0;
-			int ro = (matype1 & 0x03);
+			int ro = matype1 & 0x03;
 
 			byte matype2 = ms.ReadUInt8();
 
@@ -42,25 +44,84 @@ namespace skyscraper8.Skyscraper
 			ushort syncd = ms.ReadUInt16BE();
 			byte crc8 = ms.ReadUInt8();
 
+			if (userPacketLength == 0 && dataFieldLength == 0)
+				return;
+
 			switch (tsGsField)
 			{
-				case 2:
+				case 1:
+					if (sync != 0)
+					{
+						if (!shownNonGseWarning)
+						{
+							logger.WarnFormat("This stream is a valid GS, but also contains packets which are not GSE (type 0), but of type {0} . Please share a sample of this stream!", sync);
+							shownNonGseWarning = true;
+						}
+						return;
+					}
 					int bytes = dataFieldLength / 8;
 					if (ms.GetAvailableBytes() < bytes)
 						return;
-					HandleGse(ms);
+					HandleContinous(ms);
+					break;
+				case 2:
+					int hemBytes = dataFieldLength / 8;
+					if (ms.GetAvailableBytes() < hemBytes)
+						return;
+					HandleGseHem(ms);
 					break;
 				default:    //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts
-					logger.Warn(String.Format("Unsupported: TS/GS field says 0x{0:X2}", tsGsField));
+					logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}, please share a sample of this stream.", tsGsField));
 					break;
 			}
 		}
 
-		private bool gseNeedMore;
+        private GseFragmentation[] gseFragmentations;
+        private void HandleContinous(MemoryStream ms)
+        {
+			ms.Position = 10;
+			
+
+			while (ms.GetAvailableBytes() > 0)
+			{
+				GsePacket packet = GsePacket.Read(ms);
+				if (packet.IsPadding)
+				{
+					break;
+				}
+
+				if (packet.IsCompletePacket)
+				{
+					if (ValidateEthertype(packet.ProtocolType.Value))
+					{
+						MpeEventHandler.OnIpDatagram(PID, packet.GseData);
+					}
+					else
+					{
+						logger.WarnFormat("Unknown EtherType: 0x{0:X4}", packet.ProtocolType.Value);
+					}
+					continue;
+				}
+
+				if (packet.StartIndicator && !packet.EndIndicator)
+				{
+					if (gseFragmentations == null)
+						gseFragmentations = new GseFragmentation[256];
+
+					gseFragmentations[packet.FragmentId.Value] = new GseFragmentation();
+					gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
+					continue;
+				}
+
+				throw new NotImplementedException();
+			}
+		}
+
+        private bool gseNeedMore;
 		private MemoryStream gseAssembler;
 		public int PID;
 
-		private void HandleGse(MemoryStream ms)
+		private void HandleGseHem(MemoryStream ms)
 		{
 			ms.Position = 10;
 			byte syncByte = ms.ReadUInt8();
@@ -101,24 +162,24 @@ namespace skyscraper8.Skyscraper
 
 				}
 				gseAssembler = null;
-				HandleGse(ms);
+				HandleGseHem(ms);
 				return;
 			}
 			else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE)
 			{
 				gseAssembler = null;
-				HandleGse(ms);
+				HandleGseHem(ms);
 				return;
 			}
 			else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH)
 			{
 				gseAssembler = null;
-				HandleGse(ms);
+				HandleGseHem(ms);
 				return;
 			}
 			else
 			{
-				throw new NotImplementedException(String.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
+				throw new NotImplementedException(string.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
 			}
 		}
 
@@ -130,7 +191,8 @@ namespace skyscraper8.Skyscraper
 			BROKEN_NEGATIVE_LENGTH,
 			NEED_MORE_DATA,
 			BROKEN_SUSPECT_LENGTH,
-			BROKEN_SUSPECT_ETHERTYPE
+			BROKEN_SUSPECT_ETHERTYPE,
+			VALID_NULL_PACKET,	//somehow these only show up on TBS6903x, not DD Max SX8?
 		}
 
 		private GseAssemblyState ValidateGse(MemoryStream ms)

skyscraper8/GSE/GseFragmentation.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE
+{
+	internal class GseFragmentation
+	{
+		public void AddPacket(GsePacket packet)
+		{
+			if (spanningGsePackets == null)
+			{
+				spanningGsePackets = new List<GsePacket>();
+				TotalLength = packet.TotalLength.Value;
+			}
+
+			spanningGsePackets.Add(packet);
+		}
+
+		public ushort TotalLength { get; private set; }
+
+		private List<GsePacket> spanningGsePackets;
+	}
+}

skyscraper8/GSE/GsePacket.cs

@@ -0,0 +1,156 @@
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.NetworkInformation;
+using System.Text;
+using System.Threading.Tasks;
+using skyscraper5.Skyscraper;
+
+namespace skyscraper8.GSE
+{
+	internal class GsePacket
+	{
+		public static GsePacket Read(MemoryStream ms)
+		{
+			GsePacket packet = new GsePacket();
+
+			byte byteA = ms.ReadUInt8();
+			packet.StartIndicator = (byteA & 0x80) != 0;
+			packet.EndIndicator = (byteA & 0x40) != 0;
+			packet.LabelIndicator = (byteA & 0x30) >> 4;
+			if (!packet.StartIndicator && !packet.EndIndicator && packet.LabelIndicator == 0)
+			{
+				//End of baseband frame
+				return packet;
+			}
+			else
+			{
+				byte byteB = ms.ReadUInt8();
+				int gseLength = byteA & 0x0f;
+				gseLength <<= 8;
+				gseLength += byteB;
+
+				if (!packet.StartIndicator || !packet.EndIndicator)
+				{
+					packet.FragmentId = ms.ReadUInt8();
+					gseLength--;
+				}
+				
+				if (packet.StartIndicator && !packet.EndIndicator)
+				{
+					packet.TotalLength = ms.ReadUInt16BE();
+					gseLength -= 2;
+				}
+
+				if (packet.StartIndicator)
+				{
+					packet.ProtocolType = ms.ReadUInt16BE();
+					gseLength -= 2;
+					if (packet.LabelIndicator == 0)
+					{
+						packet.Label = new SixByteGseLabel(ms.ReadBytes(6));
+						gseLength -= 6;
+					}
+					else if (packet.LabelIndicator == 1)
+					{
+						packet.Label = new ThreeByteGseLabel(ms.ReadBytes(3));
+						gseLength -= 3;
+					}
+				}
+
+				if (!packet.StartIndicator && packet.EndIndicator)
+					gseLength -= 4;
+
+				packet.GseData = ms.ReadBytes(gseLength);
+
+				if (!packet.StartIndicator && packet.EndIndicator)
+					packet.Crc32 = ms.ReadUInt32BE();
+			}
+
+			return packet;
+		}
+
+		public uint? Crc32 { get; set; }
+
+		public byte[] GseData { get; set; }
+
+		public GseLabel Label { get; set; }
+		public ushort? ProtocolType { get; set; }
+
+		public ushort? TotalLength { get; set; }
+		public byte? FragmentId { get; set; }
+		public int LabelIndicator { get; private set; }
+		public bool EndIndicator { get; private set; }
+		public bool StartIndicator { get; private set; }
+		public bool IsPadding
+		{
+			get
+			{
+				return !StartIndicator && !EndIndicator && LabelIndicator == 0;
+			}
+		}
+
+		public bool IsCompletePacket
+		{
+			get
+			{
+				return StartIndicator && EndIndicator;
+			}
+		}
+	}
+
+	public abstract class GseLabel : Validatable
+	{
+		public abstract int LabelLength { get; }
+		public abstract string ToString();
+	}
+
+	public class ThreeByteGseLabel : GseLabel
+	{
+		public ThreeByteGseLabel(byte[] buffer)
+		{
+			if (buffer.Length != 3)
+			{
+				Valid = false;
+				return;
+			}
+
+			LabelBytes = buffer;
+			Valid = true;
+		}
+
+		public byte[] LabelBytes { get; private set; }
+
+		public override string ToString()
+		{
+			return BitConverter.ToString(LabelBytes);
+		}
+
+		public override int LabelLength => 3;
+	}
+
+	public class SixByteGseLabel : GseLabel
+	{
+		public SixByteGseLabel(byte[] buffer)
+		{
+			if (buffer.Length != 6)
+			{
+				Valid = false;
+				return;
+			}
+
+			MacAddress = new PhysicalAddress(buffer);
+			Valid = true;
+		}
+
+		public PhysicalAddress MacAddress { get; private set; }
+
+		public override string ToString()
+		{
+			return MacAddress.ToString();
+		}
+
+		public override int LabelLength => 6;
+	}
+}

skyscraper8/GSE/Stid135BbFrameReader.cs

@@ -6,16 +6,15 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-using skyscraper8.Skyscraper;
 
-namespace skyscraper5.src.Skyscraper
+namespace skyscraper8.GSE
 {
-	internal class DigitalDevicesBbFrameReader : ITsPacketProcessor
+	internal class Stid135BbFrameReader : ITsPacketProcessor
 	{
 		private IGsEventHandler mpeEventHandler;
 		private BbframeDeencapsulator deencapsulator;
 
-		public DigitalDevicesBbFrameReader(IGsEventHandler mpeEventHandler)
+		public Stid135BbFrameReader(IGsEventHandler mpeEventHandler)
 		{
 			this.mpeEventHandler = mpeEventHandler;
 		}
@@ -27,10 +26,10 @@ namespace skyscraper5.src.Skyscraper
 		public void PushPacket(TsPacket packet)
 		{
 			byte[] packets = packet.RawPacket;
-			int pid = (packets[1]);
+			int pid = packets[1];
 			pid &= 0x01f;
 			pid <<= 8;
-			pid |= (packets[2]);
+			pid |= packets[2];
 			if (pid != 0x010e)
 				return;
 

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "satip auto 1 H 11141 S2 23500",
+      "commandLineArgs": "\"\\\\utena\\mergerfs\\Skyscraper\\ipProto253_eutelsat7_10803h-max-sx8.ts\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/OldStreamReaderDetector.cs

@@ -5,6 +5,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using log4net;
 using skyscraper5.Dvb;
 using skyscraper5.Dvb.DataBroadcasting;
 using skyscraper5.Mpeg2;
@@ -12,13 +13,12 @@ using skyscraper5.Skyscraper.IO;
 
 namespace skyscraper5.Skyscraper
 {
-    internal class Tbs6903xGsReader : ITsPacketProcessor
+    internal class OldStreamReaderDetector : ITsPacketProcessor
     {
-        private readonly IGsEventHandler mpeEventHandler;
+	    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
 
-        public Tbs6903xGsReader(IGsEventHandler mpeEventHandler)
+		public OldStreamReaderDetector()
         {
-            this.mpeEventHandler = mpeEventHandler;
         }
 
         private MemoryStream outbuf;
@@ -27,13 +27,16 @@ namespace skyscraper5.Skyscraper
 
         public void PushPacket(TsPacket packet)
         {
-                byte[] packets = packet.RawPacket;
+	        if (annoncementDone)
-                int pid = (packets[1]);
+		        return;
-                pid &= 0x01f;
+
-                pid <<= 8;
+	        byte[] packets = packet.RawPacket;
-                pid |= (packets[2]);
+	        int pid = (packets[1]);
-                if (pid != 0x0118)
+	        pid &= 0x01f;
-                    return;
+	        pid <<= 8;
+	        pid |= (packets[2]);
+	        if (pid != 0x0118)
+		        return;
 
             if ((packets[8] & 0xff) == 0xd0)
             {
@@ -45,10 +48,9 @@ namespace skyscraper5.Skyscraper
                     {
                         if (!annoncementDone)
                         {
-                            mpeEventHandler.GsIpTrafficDetected();
+                            logger.WarnFormat("It seems like you're using a StreamReader.dll version which interferes with the STiD135's BBFrame encapsulation. Try updating to 1.2.5.208, this is known to work.");
                             annoncementDone = true;
                         }
-                        mpeEventHandler.OnIpDatagram(0x0118, ipPacket);
                     }
                 }
                 outbuf = new MemoryStream();

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -78,6 +78,7 @@ using System.Security.Policy;
 using System.Text;
 using skyscraper8.Experimentals.NdsSsu;
 using skyscraper8.Experimentals.OtvSsu;
+using skyscraper8.GSE;
 using skyscraper8.Skyscraper.Net;
 using skyscraper8.Skyscraper.Scraper;
 using Tsubasa.IO;
@@ -293,16 +294,14 @@ namespace skyscraper5.Skyscraper.Scraper
                 LogEvent(SkyscraperContextEvent.StartPacketProcessing);
                 if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x18 && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00)
                 {
-                    DvbContext.RegisterPacketProcessor(0x0118, new Tbs6903xGsReader(this));
+                    DvbContext.RegisterPacketProcessor(0x0118, new OldStreamReaderDetector());
-                    UiJunction?.SetGseMode();
-                    LogEvent(SkyscraperContextEvent.SpecialTsMode, "TBS 6903-X GSE Pseudo TS detected.");
                     SpecialTsType = 2;
                 }
                 if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x0e && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00)
                 {
-					DvbContext.RegisterPacketProcessor(0x010e, new DigitalDevicesBbFrameReader(this));
+					DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
 					UiJunction?.SetGseMode();
-					LogEvent(SkyscraperContextEvent.SpecialTsMode, "Digital Devices BBFrame Pseudo TS detected.");
+					LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
 					SpecialTsType = 3;
 				}
                 firstPacketDone = true;
@@ -339,10 +338,7 @@ namespace skyscraper5.Skyscraper.Scraper
                     {
                         if (!DvbContext.IsPidProcessorPresent(0x0118))
                         {
-                            DvbContext.RegisterPacketProcessor(0x0118, new Tbs6903xGsReader(this));
+                            DvbContext.RegisterPacketProcessor(0x0118, new OldStreamReaderDetector());
-                            UiJunction?.SetGseMode();
-                            LogEvent(SkyscraperContextEvent.SpecialTsMode, "TBS 6903-X GSE Pseudo TS detected.");
-                            SpecialTsType = 2;
                             return;
                         }
                     }
@@ -357,9 +353,9 @@ namespace skyscraper5.Skyscraper.Scraper
                         {
                             if (!DvbContext.IsPidProcessorPresent(0x010e))
                             {
-                                DvbContext.RegisterPacketProcessor(0x010e, new DigitalDevicesBbFrameReader(this));
+                                DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
                                 UiJunction?.SetGseMode();
-                                LogEvent(SkyscraperContextEvent.SpecialTsMode, "Digital Devices BBFRAME data TS detected.");
+                                LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                                 SpecialTsType = 3;
                                 return;
                             }