Built a "sub-TS" handling system.

skyscraper8/Abertis/AbertisSubSkyscraperKey.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Abertis
+{
+	internal class AbertisSubSkyscraperKey
+	{
+		public int Pid { get; }
+
+		public AbertisSubSkyscraperKey(int pid)
+		{
+			Pid = pid;
+		}
+
+		public override string ToString()
+		{
+			return String.Format("{0:X4}", Pid);
+		}
+	}
+}

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\638963394655023111.ts",
+      "commandLineArgs": "\"C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\11049v.ts\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -77,12 +77,14 @@ using System.Net.NetworkInformation;
 using System.Security.Policy;
 using System.Text;
 using skyscraper5.src.InteractionChannel.Model2;
+using skyscraper8.Abertis;
 using skyscraper8.Experimentals.NdsSsu;
 using skyscraper8.Experimentals.OtvSsu;
 using skyscraper8.GSE;
 using skyscraper8.InteractionChannel.Model2;
 using skyscraper8.Skyscraper.Net;
 using skyscraper8.Skyscraper.Scraper;
+using skyscraper8.T2MI;
 using Tsubasa.IO;
 using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
 using RntParser = skyscraper5.Dvb.TvAnytime.RntParser;
@@ -1897,31 +1899,56 @@ namespace skyscraper5.Skyscraper.Scraper
         public void OnT2MiPacketLoss(int pid, byte expectedPacket, T2MIHeader header)
         {
         }
-
-        private SkyscraperContext[][] childSkyscrapers;
+        
         public void OnT2MiPacket(int pid, byte basebandFramePlpId, byte[] basebandPacket)
         {
-            if (childSkyscrapers == null)
-                childSkyscrapers = new SkyscraperContext[0x1fff][];
-            if (childSkyscrapers[pid] == null)
-            {
-                childSkyscrapers[pid] = new SkyscraperContext[256];
-            }
-            if (childSkyscrapers[pid][basebandFramePlpId] == null)
-            {
-                childSkyscrapers[pid][basebandFramePlpId] = new SkyscraperContext(new TsContext(), DataStorage,ObjectStorage);
-                childSkyscrapers[pid][basebandFramePlpId].IsChild = true;
-                childSkyscrapers[pid][basebandFramePlpId].ChildName = String.Format("PLP {0}", basebandFramePlpId);
+	        if (subSkyscrapers == null)
+		        subSkyscrapers = new Dictionary<object, SkyscraperContext>();
 
-				childSkyscrapers[pid][basebandFramePlpId].InitalizeFilterChain();
-                LogEvent(SkyscraperContextEvent.T2MiPlpDetected, String.Format("PID {0:X4}, PLP {1}", pid, basebandFramePlpId));
-            }
+	        T2MiSubSkyscraperIdentifier subSkyscraperIdentifier = new T2MiSubSkyscraperIdentifier(pid, basebandFramePlpId);
+	        if (!subSkyscrapers.ContainsKey(subSkyscraperIdentifier))
+	        {
+		        LogEvent(SkyscraperContextEvent.T2MiPlpDetected, String.Format("PID {0:X4}, PLP {1}", pid, basebandFramePlpId));
+	        }
 
-            childSkyscrapers[pid][basebandFramePlpId].IngestSinglePacket(basebandPacket);
+	        OnSubTsPacket(subSkyscraperIdentifier, basebandPacket);
         }
 
+        private Dictionary<object, SkyscraperContext> subSkyscrapers;
+        public void OnSubTsPacket(object identifier, byte[] packet)
+        {
+	        if (subSkyscrapers == null)
+		        subSkyscrapers = new Dictionary<object, SkyscraperContext>();
+
+	        if (!subSkyscrapers.ContainsKey(identifier))
+	        {
+		        SkyscraperContext child = new SkyscraperContext(new TsContext(), DataStorage, ObjectStorage);
+		        child.IsChild = true;
+		        child.ChildName = identifier.ToString();
+		        child.ChildTimestamp = DateTime.Now.Ticks;
+
+		        List<IPacketFilter> packetFilters = new List<IPacketFilter>();
+		        StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
+		        if (storageConnectionManager.IsSubTsDumpingEnabled())
+		        {
+			        string filename = String.Format("{0}_{1}.ts", child.ChildName.SanitizeFileName(), child.ChildTimestamp);
+			        FileInfo fi = new FileInfo(filename);
+			        TsRecorder packetDumper = new TsRecorder();
+			        packetDumper.SetNextFilename(filename);
+                    packetDumper.CreateBufferedStream();
+                    packetFilters.Add(packetDumper);
+		        }
+
+		        child.InitalizeFilterChain();
+		        subSkyscrapers.Add(identifier, child);
+	        }
+
+	        SkyscraperContext context = subSkyscrapers[identifier];
+	        context.IngestSinglePacket(packet);
+        }
+        public long ChildTimestamp { get; private set; }
         public string ChildName { get; private set; }
-        public bool IsChild { get; set; }
+        public bool IsChild { get; private set; }
 
         public void OnT2MiTimestamp(int pid, _0x20_DvbT2Timestamp t2Timestamp)
         {
@@ -1931,27 +1958,28 @@ namespace skyscraper5.Skyscraper.Scraper
             if (!CurrentTransportStreamId.HasValue)
                 return;
 
-            if (childSkyscrapers == null)
+            if (subSkyscrapers == null)
                 return;
 
-            if (childSkyscrapers[pid] == null)
-                return;
-
-            bool HasEvents(SkyscraperContext[] skyscrapers)
+            List<SkyscraperContext> contexts = new List<SkyscraperContext>();
+            foreach (KeyValuePair<object, SkyscraperContext> subSkyscraper in subSkyscrapers)
             {
-                for (int i = 0; i < skyscrapers.Length; i++)
-                {
-                    if (skyscrapers[i] == null)
-                        continue;
+	            object subSkyscraperKey = subSkyscraper.Key;
+	            if (!(subSkyscraperKey is T2MiSubSkyscraperIdentifier))
+		            continue;
 
-                    if (skyscrapers[i].EventsLogged >= 2)
-                        return true;
-                }
+                T2MiSubSkyscraperIdentifier identifier = subSkyscraperKey as T2MiSubSkyscraperIdentifier;
+                if (identifier.Pid != pid)
+	                continue;
 
-                return false;
+                contexts.Add(subSkyscraper.Value);
             }
 
-            if (!HasEvents(childSkyscrapers[pid]))
+            if (contexts.Count == 0)
+                return;
+            
+            long sum = contexts.Select(x => (long)x.EventsLogged).Sum();
+            if (sum < (contexts.Count * 2))
                 return;
 
             DateTime resolveTime = t2Timestamp.ResolveTime();
@@ -2008,10 +2036,13 @@ namespace skyscraper5.Skyscraper.Scraper
             if (!CurrentTransportStreamId.HasValue)
                 return;
 
-            if (childSkyscrapers == null)
+            if (subSkyscrapers == null)
                 return;
 
-            if (childSkyscrapers[relatedPid] == null)
+            bool any = subSkyscrapers.Where(x => x.Key is T2MiSubSkyscraperIdentifier)
+	            .Select(x => (T2MiSubSkyscraperIdentifier)x.Key)
+	            .Any();
+            if (!any)
                 return;
 
             foreach (AddressingFunction function in individualAddressing.Commands)
@@ -2222,32 +2253,24 @@ namespace skyscraper5.Skyscraper.Scraper
 
         public void OnAbertisPacket(int pid, byte[] outBuffer)
         {
-            if (childSkyscrapers == null)
-                childSkyscrapers = new SkyscraperContext[0x1fff][];
-            if (childSkyscrapers[pid] == null)
-                childSkyscrapers[pid] = new SkyscraperContext[1];
-            if (childSkyscrapers[pid][0] == null)
-            {
-                childSkyscrapers[pid][0] = new SkyscraperContext(new TsContext(), DataStorage, ObjectStorage);
-                childSkyscrapers[pid][0].IsChild = true;
-                childSkyscrapers[pid][0].ChildName = String.Format("Abertis Tunnel on PID {0:X4}", pid);
-			}
-
-            childSkyscrapers[pid][0].IngestSinglePacket(outBuffer);
-        }
+	        AbertisSubSkyscraperKey key = new AbertisSubSkyscraperKey(pid);
+	        OnSubTsPacket(key, outBuffer);
+		}
 
         public void OnAbertisSyncLoss(int pid, int oldPosition = -1, long newPosition = -1)
         {
-            if (childSkyscrapers == null)
+	        if (subSkyscrapers == null)
                 DvbContext.RegisterPacketProcessor(pid, new PacketDiscarder());
-            else if (childSkyscrapers[pid] == null)
+
+	        AbertisSubSkyscraperKey key = new AbertisSubSkyscraperKey(pid);
+			if (!subSkyscrapers.ContainsKey(key) == null)
                 DvbContext.RegisterPacketProcessor(pid, new PacketDiscarder());
-            else if (childSkyscrapers[pid][0] == null)
-                DvbContext.RegisterPacketProcessor(pid, new PacketDiscarder());
-            else if (!childSkyscrapers[pid][0].firstPacketDone)
-                DvbContext.RegisterPacketProcessor(pid, new PacketDiscarder());
-            else
-                throw new NotImplementedException();
+
+			if (!subSkyscrapers[key].firstPacketDone)
+				DvbContext.RegisterPacketProcessor(pid, new PacketDiscarder());
+			else
+				throw new NotImplementedException(
+					"I couldn't sync this Abertis-styled stream, and I couldn't figure out why. Could you please share a sample of this stream?");
         }
 
 		public void OnMonochromeData(int networkId, int transportStreamId, ushort programNumber, MonochromeDataField result)

skyscraper8/Skyscraper/Scraper/Storage/StorageConnectionManager.cs

@@ -187,5 +187,29 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 			object invoke = constructorInfo.Invoke(new object[] { });
 			return (IpTrafficHandler) invoke;
 		}
+
+		private bool IsSubTsDumpingConfigured()
+		{
+			if (ini == null)
+				return false;
+			if (!ini.ContainsKey("subts"))
+				return false;
+			if (!ini["subts"].ContainsKey("dump"))
+				return false;
+			return true;
+		}
+
+		internal bool IsSubTsDumpingEnabled()
+		{
+			if (IsSubTsDumpingConfigured())
+			{
+				return ini.ReadValue("subts", "dump", true);
+			}
+			else
+			{
+				logger.WarnFormat("You didn't call the subts option before. I'm gonna assume you want me to dump inner TS packets.");
+				return true;
+			}
+		}
 	}
 }

skyscraper8/T2MI/T2MiSubSkyscraperIdentifier.cs

@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.T2MI
+{
+	internal class T2MiSubSkyscraperIdentifier
+	{
+		public int Pid { get; }
+		public byte BasebandFramePlpId { get; }
+
+		public T2MiSubSkyscraperIdentifier(int pid, byte basebandFramePlpId)
+		{
+			Pid = pid;
+			BasebandFramePlpId = basebandFramePlpId;
+		}
+
+		public override string ToString()
+		{
+			return String.Format("PID{0:X4}_PLP{1}", Pid, BasebandFramePlpId);
+		}
+
+		protected bool Equals(T2MiSubSkyscraperIdentifier other)
+		{
+			return Pid == other.Pid && BasebandFramePlpId == other.BasebandFramePlpId;
+		}
+
+		public override bool Equals(object? obj)
+		{
+			if (ReferenceEquals(null, obj)) 
+				return false;
+			if (ReferenceEquals(this, obj)) 
+				return true;
+			if (obj.GetType() != this.GetType()) 
+				return false;
+			return Equals((T2MiSubSkyscraperIdentifier)obj);
+		}
+
+		public override int GetHashCode()
+		{
+			return HashCode.Combine(Pid, BasebandFramePlpId);
+		}
+	}
+}