Added an alternative implementation of the BbframeDeencapsulator for testing.

skyscraper8/GSE/BbframeDeencapsulator2.cs

@@ -0,0 +1,213 @@
+using log4net;
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+using System.Xml.Linq;
+using skyscraper5.Skyscraper.Net.Pcap;
+
+namespace skyscraper8.GSE
+{
+	internal class BbframeDeencapsulator2 : IBbframeDeencapsulator
+	{
+		[StructLayout(LayoutKind.Sequential, Pack = 1)]
+		struct bbheader
+		{
+			public byte MaType1;
+			public byte MaType2;
+			public byte Upl1;
+			public byte Upl2;
+			public byte Dfl1;
+			public byte Dfl2;
+			public byte Sync;
+			public byte SyncD1;
+			public byte SyncD2;
+			public byte Crc8;
+		}
+
+		[StructLayout(LayoutKind.Sequential, Pack = 1)]
+		unsafe struct Layer3
+		{
+			public byte l3sync;
+			public bbheader header;
+			public fixed byte Data[7254 - 10];
+		}
+
+		public long NumPushed { get; private set; }
+		private byte[] inData;
+
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		public unsafe void PushPacket(byte[] bbframe)
+		{
+			NumPushed++;
+
+			if (inData == null)
+			{
+				inData = new byte[8100];
+			}
+
+			
+			MemoryStream inbuf = new MemoryStream(bbframe);
+			Layer3 myLayer;
+			while (true)
+			{
+				if (inbuf.Peek() == 0xb8)
+					break;
+				inbuf.ReadByte();
+			}
+
+			while (true)
+			{
+				int fail = inbuf.Read(inData, 0, 11);
+				if (fail != 11)
+					break;
+				myLayer = MemoryMarshal.Read<Layer3>(inData);
+				if (myLayer.l3sync != 0xb8)
+				{
+					while (true)
+					{
+						if (inbuf.Peek() == 0xb8)
+							break;
+						fail = inbuf.Read(inData, 0, 1);
+						if (fail != 1)
+							break;
+					}
+				}
+
+				int bblength = myLayer.header.Dfl1 << 8 | myLayer.header.Dfl2;
+				bblength >>= 3;
+				fail = inbuf.Read(inData, 11, bblength);
+				if (fail != bblength)
+					break;
+				myLayer = MemoryMarshal.Read<Layer3>(inData);
+				byte mis = myLayer.header.MaType2;
+
+				if (misStreams == null)
+					misStreams = new MisProcessor[256];
+				if (misStreams[mis] == null)
+				{
+					logger.InfoFormat("Found MIS {0}", mis);
+					misStreams[mis] = new MisProcessor();
+				}
+
+				int pos = 0;
+				while (pos < bblength - 4)
+				{
+					int gseLength = ((myLayer.Data[pos] & 0x0f) << 8) | (myLayer.Data[pos + 1]);
+					if ((myLayer.Data[pos] & 0xf0) == 0) break;
+					if (gseLength + 2 > bblength - pos) break;
+					if (!misStreams[mis].ProcessBbFrame(&myLayer.Data[pos], gseLength))
+						break;
+					pos += gseLength += 2;
+				}
+			}
+		}
+
+		private MisProcessor[] misStreams;
+		private class MisProcessor
+		{
+			private byte[][] fragmentor;
+			private int[] fragmentorLength;
+			private int[] fragmentorPos;
+			private PcapWriter pcapWriter;
+
+			public MisProcessor()
+			{
+				fragmentor = new byte[256][];
+				fragmentorLength = new int[256];
+				fragmentorPos = new int[256];
+				FileStream fileStream = File.OpenWrite(string.Format("{0}.pcap", DateTime.Now.Ticks));
+				pcapWriter = new PcapWriter(fileStream, TcpdumpNetworkType.RawIp);
+			}
+
+			private unsafe void Copy(byte* src, byte[] dst, int srcOffset, int dstOffset, int len)
+			{
+				for (int i = 0; i < len; i++)
+				{
+					dst[dstOffset + i] = src[srcOffset + i];
+				}
+			}
+
+			public unsafe bool ProcessBbFrame(byte* payload, int gseLength)
+			{
+				uint offset = 0;
+				uint fragId = 0;
+
+				if ((payload[0] & 0xc0) == 0x80)
+				{
+					fragId = payload[2];
+					int length = (payload[3] << 8) | payload[4];
+					if (fragmentor[fragId] != null)
+						fragmentor[fragId] = null;
+					fragmentor[fragId] = new byte[length + 2];
+					fragmentorLength[fragId] = length + 2;
+					fragmentor[fragId][0] = payload[0];
+					fragmentor[fragId][1] = payload[1];
+					fragmentor[fragId][0] |= 0xc0;
+					Copy(payload, fragmentor[fragId], 5, 2, gseLength - 3);
+					fragmentorPos[fragId] = gseLength - 1;
+				}
+				else if ((payload[0] & 0xc0) == 0x00)
+				{
+					throw new NotImplementedException("inbetween packet");
+				}
+				else if ((payload[0] & 0xc0) == 0x40)
+				{
+					fragId = payload[2];
+					if (fragmentor[fragId] == null)
+						return true;
+
+					Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 5);
+					fragmentorPos[fragId] += gseLength - 1;
+					fixed (byte* p = fragmentor[fragId])
+					{
+						ProcessBbFrame(p, fragmentorLength[fragId]);
+					}
+					fragmentor[fragId] = null;
+
+				}
+				else if ((payload[0] & 0xc0) == 0xc0)
+				{
+					if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x04)
+					{
+						throw new NotImplementedException("ethertype 0x0400");
+					}
+					else if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x00)
+					{
+						throw new NotImplementedException("ethertype 0x0000");
+					}
+					else if (payload[offset + 2] == 0x08 && payload[offset + 3] == 0x00)
+					{
+						if ((payload[0] & 0x30) == 0x01)
+						{
+							offset += 3;    //3-byte label
+						}
+						else if ((payload[0] & 0x30) == 0x00)
+						{
+							offset += 6;    //MAC address
+						}
+
+						offset += 4;
+
+						//at this point, the IP header starts at "offset"
+						long sinkSize = gseLength + 2 - offset;
+						byte[] sink = new byte[sinkSize];
+						Copy(payload, sink, (int)offset, 0, (int)sinkSize);
+						pcapWriter.WritePacket(sink);
+						return true;
+					}
+
+					//throw new NotImplementedException("complete packet");
+				}
+				else if ((payload[0] & 0x0f0) == 0x00)
+				{
+					throw new NotImplementedException("padding packet");
+				}
+				return true;
+			}
+		}
+	}
+}

skyscraper8/GSE/Stid135TestingProgram.cs

@@ -0,0 +1,40 @@
+using log4net;
+using log4net.Repository.Hierarchy;
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper.Scraper;
+using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
+using skyscraper8.Skyscraper.Scraper.Storage;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.GSE
+{
+	internal class Stid135TestingProgram
+	{
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		public static void Run(FileInfo file)
+		{
+			if (!file.Exists)
+			{
+				logger.Error("File not found: " + file.FullName);
+				return;
+			}
+
+			FileStream fileStream = file.OpenRead();
+
+			TsContext mpeg2 = new TsContext();
+			mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),new BbframeDeencapsulator2()));
+			DataStorage dataStorage = new InMemoryScraperStorage();
+			ObjectStorage objectStorage = new NullObjectStorage();
+			SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
+			skyscraper.InitalizeFilterChain();
+			skyscraper.IngestFromStream(fileStream);
+
+			fileStream.Close();
+			logger.Info("Stid135TestingProgram finished");
+		}
+	}
+}

skyscraper8/Program.cs

@@ -323,6 +323,13 @@ namespace skyscraper5
                         return;
                     }
                 }
+
+                if (args[0].ToLowerInvariant().Equals("stid135test"))
+                {
+	                FileInfo fi = new FileInfo(args[1]);
+	                Stid135TestingProgram.Run(fi);
+	                return;
+                }
 			}
 
             /*Passing passing = new Passing();

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "cscan tcp://172.20.20.203:6969",
+      "commandLineArgs": "stid135test \"\\\\utena\\mergerfs\\Skyscraper\\dab_thor_10717_v_max-sx8.ts\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {