Added a catalogue generator option.

.gitignore

@@ -132,3 +132,4 @@ imgui.ini
 /.vs/skyscraper8/CopilotIndices/17.14.1231.31060
 /.vs/skyscraper8/CopilotIndices/17.14.1290.42047
 /Documentation/TSDuck-Samples/experiment2/*.ts
+/.vs/skyscraper8/CopilotIndices/17.14.1431.25910

skyscraper8/Mpeg2/TsContext.cs

@@ -14,12 +14,15 @@ namespace skyscraper5.Mpeg2
 		private ITsPacketProcessor[] processors;
         private uint[] continuities;
         private ulong[] pidPackets;
+        private ulong[] scrambledPackets;
+        private ulong[] teiPackets;
 
-        public long PacketLossEvents { get; private set; }
+		public long PacketLossEvents { get; private set; }
         public bool FirstPacketDone { get; private set; }
         public long PacketsRead { get; private set; }
         public long TheoreticalOffset { get; private set; }
         public List<IPacketFilter> FilterChain { get; set; }
+        public bool LikelyBrokenEncapsulation => this.encapsulationBrokenConfirmed;
 
         public void PushPacket(TsPacket packet)
         {
@@ -44,6 +47,19 @@ namespace skyscraper5.Mpeg2
 			if (FilterChain == null || FilterChain.Count == 0)
 				throw new InvalidOperationException("The filter chain has not been initialized.");
 
+			if (packet.TSC != 0)
+			{
+				if (scrambledPackets == null)
+					scrambledPackets = new ulong[0x2000];
+				scrambledPackets[packet.PID]++;
+			}
+
+			if (packet.TEI)
+			{
+				if (teiPackets == null)
+					teiPackets = new ulong[0x2000];
+				teiPackets[packet.PID]++;
+			}
 			CheckBrokenEncapsulation(packet);
 
 			for (int i = 0; i < FilterChain.Count; i++)
@@ -55,13 +71,14 @@ namespace skyscraper5.Mpeg2
 					return;
 			}
 
+			EnsureContinuity(packet);
+
 			if (processors == null)
                 return;
 
             if (processors[packet.PID] == null)
                 return;
 
-			bool continuity = EnsureContinuity(packet);
             processors[packet.PID].PushPacket(packet);
         }
 
@@ -109,6 +126,9 @@ namespace skyscraper5.Mpeg2
             if (packet.AdaptionFieldControl == 2 || packet.AdaptionFieldControl == 0)
                 return true;
 
+            if (packet.PID == 0x1fff)
+	            return true;
+
             uint pid = packet.PID;
             if (continuities[pid] == UInt32.MaxValue)
             {
@@ -189,6 +209,18 @@ namespace skyscraper5.Mpeg2
             return result;
         }
 
+        public int CountPidsWithTraffic()
+        {
+	        int result = 0;
+	        for (int i = 0; i < this.pidPackets.Length; i++)
+	        {
+		        if (this.pidPackets[i] > 0)
+			        result++;
+	        }
+
+	        return result;
+        }
+
         public ulong[] GetPidStatistics()
         {
             if (pidPackets == null)
@@ -203,10 +235,29 @@ namespace skyscraper5.Mpeg2
             return pidPackets[pid];
         }
 
+        public ulong GetScrambledPacketsOnPid(int pid)
+        {
+	        if (scrambledPackets == null)
+		        return 0;
+	        else
+		        return scrambledPackets[pid];
+        }
+
+        public ulong GetTeiPacketsOnPid(int pid)
+        {
+	        if (teiPackets == null)
+		        return 0;
+	        else
+	        {
+		        return teiPackets[pid];
+	        }
+        }
+
         #region TCP Proxy
 
         private bool _tcpProxyEnabled;
         private TcpTsProxy tcpTsProxy;
+        
 
         public bool TcpProxyEnabled
         {

skyscraper8/Program.cs

@@ -351,6 +351,16 @@ namespace skyscraper5
                     Stid135Test.Run(fi);
                     return;
                 }
+
+                if (args[0].ToLowerInvariant().Equals("make-catalogue"))
+                {
+	                DirectoryInfo di = new DirectoryInfo(args[1]);
+	                FileInfo fi = new FileInfo(args[2]);
+	                CatalogueGenerator catalogueGenerator = new CatalogueGenerator(di, fi);
+                    catalogueGenerator.Run();
+                    catalogueGenerator.Dispose();
+                    return;
+                }
 			}
 
             /*Passing passing = new Passing();
@@ -370,13 +380,15 @@ namespace skyscraper5
             Console.WriteLine("         or: .\\skyscraper8.exe pcap-off  - to write a configuration file that turns off PCAP writing during scraping.");
             Console.WriteLine("         or: .\\skyscraper8.exe subts-on  - to write a configuration file that allows extraction of nested TS.");
             Console.WriteLine("         or: .\\skyscraper8.exe subts-off - to write a configuration file that turns off extraction of nested TS.");
-			Console.WriteLine();
+            Console.WriteLine();
             Console.WriteLine("default behaviour is pcap writing and nested TS writing on.");
 			Console.WriteLine();
             Console.WriteLine("Bonus features:");
             Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC.");
             Console.WriteLine(".\\skyscraper8.exe hlsproxy-destructive \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC and delete HLS segments afterwards. (be careful!)");
             Console.WriteLine(".\\skyscraper8.exe shannon \"C:\\some\\file.bmp\" - calculates the Shannon entropy value for any given file.");
+            Console.WriteLine(".\\skyscraper8.exe make-catalogue \"C:\\path\\to\\ts\\collection\\\" \"C:\\outputted_index.csv\" - generates a catalogue with core information about your TS files.");
+            Console.WriteLine(".\\skyscraper8.exe pts2bbf2 \"C:\\path\\to\\file.ts\\\" - extracts every single BBFrame from a GS to into a small file for each. (be careful, might generate many small files!)");
 		}
 
 		private static void ToggleSubTsDumpConfiguration(bool enabled)

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "\"C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\samples\\skyscraper_20251029_1201_0150W_12596_V_44995.ts\"",
+      "commandLineArgs": "make-catalogue \"D:\\\\Stash\\\\\" \"D:\\\\index.csv\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/CatalogueGenerator.cs

@@ -0,0 +1,259 @@
+using log4net;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper.Scraper.Utils;
+using skyscraper5.src.Mpeg2.PacketFilter;
+
+namespace skyscraper8.Skyscraper
+{
+	internal class CatalogueGenerator : IDisposable
+	{
+		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		private readonly DirectoryInfo _inputDir;
+		private readonly FileStream _outputCsvStream;
+		private readonly StreamWriter _outputCsv;
+		private readonly List<IPacketFilter> _nullPacketFilter;
+		private readonly PacketDiscarder _packetDiscarder;
+
+		public CatalogueGenerator(DirectoryInfo inputDir, FileInfo outputCsv)
+		{
+			_inputDir = inputDir;
+			if (outputCsv.Exists)
+			{
+				string originalName = outputCsv.FullName;
+				string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(outputCsv.FullName);
+				string newName = Path.Combine(outputCsv.Directory.FullName,String.Format("{0}_{1}.bak", fileNameWithoutExtension, DateTime.Now.Ticks));
+				logger.WarnFormat("The file {0} already exists. I don't want to overwrite it, so I'm gonna rename it to {1}:", originalName, newName);
+				File.Move(originalName, newName);
+				outputCsv.Refresh();
+			}
+
+			_outputCsvStream = outputCsv.Open(FileMode.CreateNew, FileAccess.Write, FileShare.Read);
+			_outputCsv = new StreamWriter(_outputCsvStream, Encoding.UTF8);
+			_outputCsv.WriteLine("FILE_NAME;GUESSED_TYPE;TOTAL_PACKETS;PIDS;PAT_PACKETS;OTHER_PACKETS;NULL_PACKETS;PAT_SCRAMBLED;OTHER_SCRAMBLED;NULL_SCRAMBLED;PAT_TEI;OTHER_TEI;NULL_TEI;PCR_PID;PCR_DURATION;DISCONTINUITIES;READ_ERROR;LOWEST_PID;");
+			_nullPacketFilter = new List<IPacketFilter>();
+			_nullPacketFilter.Add(new NullPacketFilter());
+		}
+
+		public void Dispose()
+		{
+			_outputCsvStream.Dispose();
+		}
+
+		public void Run()
+		{
+			ScrapeDirectory(_inputDir);
+		}
+
+		private void ScrapeDirectory(DirectoryInfo di)
+		{
+			logger.InfoFormat("Entering directory: {0}", di.FullName);
+			foreach (FileSystemInfo fileSystemInfo in di.GetFileSystemInfos())
+			{
+				switch (fileSystemInfo)
+				{
+					case DirectoryInfo subdirectory:
+						ScrapeDirectory(subdirectory);
+						break;
+					case FileInfo fi:
+						ScrapeFile(fi);
+						break;
+					default:
+						logger.ErrorFormat("{0} is a {1}. Those are not supported yet.", fileSystemInfo.FullName, fileSystemInfo.GetType().Name);
+						break;
+				}
+			}
+			logger.InfoFormat("Leaving directory: {0}", di.FullName);
+		}
+
+		private void ScrapeFile(FileInfo fi)
+		{
+			string extension = Path.GetExtension(fi.FullName);
+			extension = extension.ToLowerInvariant();
+			if (!extension.Equals(".ts"))
+				return;
+
+			if (fi.Length == 0)
+			{
+				_outputCsv.WriteLine("\"{0}\";TOO_SMALL;0;0;0;0;0;0;0;0;0;0;0;0;0;0;", fi.FullName);
+				_outputCsv.Flush();
+				return;
+			}
+
+			TsContext tsContext = new TsContext();
+			tsContext.FilterChain = _nullPacketFilter;
+			tsContext.RegisterPacketProcessor(0x0000, _packetDiscarder);
+			int readError = 0;
+
+			logger.InfoFormat("Reading: {0}", fi.Name);
+			FileStream fileStream = fi.OpenRead();
+			byte[] buffer = new byte[188];
+			try
+			{
+				for (long l = 0; l < fileStream.Length; l += 188)
+				{
+					if (fileStream.Read(buffer, 0, 188) != 188)
+					{
+						logger.ErrorFormat("Failed to read 188 bytes from offset {0} of file {1}, aborting reading it.", l, fi.Name);
+						readError = 1;
+						break;
+					}
+					tsContext.PushPacket(buffer);
+				}
+			}
+			catch (IOException e)
+			{
+				readError = 2;
+				logger.ErrorFormat("Failed to read from {0}: {1}", fi.Name, e.ToString());
+			}
+
+			string filename = fi.FullName;
+			TsType tsType = GuessTsType(tsContext);
+			long totalPackets = tsContext.PacketsRead + 1;
+			int pids = tsContext.CountPidsWithTraffic();
+			ulong patPackets = tsContext.GetNumberOfPacketsOnPid(0);
+			ulong otherPackets = CountOtherPackets(tsContext);
+			ulong nullPackets = tsContext.GetNumberOfPacketsOnPid(8191);
+			ulong patScrambled = tsContext.GetScrambledPacketsOnPid(0);
+			ulong otherScrambled = CountOtherScrambledPacket(tsContext);
+			ulong nullScrambled = tsContext.GetScrambledPacketsOnPid(8191);
+			ulong patTei = tsContext.GetTeiPacketsOnPid(0);
+			ulong otherTei = CountOtherTeiPacket(tsContext);
+			ulong nullTei = tsContext.GetTeiPacketsOnPid(8191);
+			uint pcrPid = GetPcrPid(tsContext);
+			TimeSpan pcrDuration = GetPcrDuration(tsContext);
+			long discontinuities = tsContext.PacketLossEvents;
+			int lowestPid = GetLowestPid(tsContext);
+
+			_outputCsv.WriteLine(
+				"\"{0}\";{1};{2};{3};{4};{5};{6};{7};{8};{9};{10};{11};{12};{13};{14};{15};{16};{17}",
+				filename, tsType, totalPackets, pids,
+				patPackets, otherPackets,
+				nullPackets, patScrambled, otherScrambled, nullScrambled, patTei, otherTei, nullTei, pcrPid,
+				pcrDuration, discontinuities, readError, lowestPid); 
+			_outputCsv.Flush();
+		}
+
+		private int GetLowestPid(TsContext tsContext)
+		{
+			for (int i = 0; i < 8192; i++)
+			{
+				ulong numberOfPacketsOnPid = tsContext.GetNumberOfPacketsOnPid(i);
+				if (numberOfPacketsOnPid > 0)
+					return i;
+			}
+
+			return -1;
+		}
+
+		private TsType GuessTsType(TsContext tsContext)
+		{
+			if (tsContext.PacketsRead == 0)
+				return TsType.TOO_SMALL;
+
+			if (tsContext.LikelyBrokenEncapsulation)
+				return TsType.LIKELY_BAD_RECEPTION;
+
+			ulong brokenGsIndicators = tsContext.GetNumberOfPacketsOnPid(0x0118);
+			double brokenGsPercentage = (double)brokenGsIndicators / (double)tsContext.PacketsRead;
+			if (brokenGsPercentage >= 0.95)
+				return TsType.GS_OLD_STREAMREADER;
+
+			ulong goodGsIndicators = tsContext.GetNumberOfPacketsOnPid(0x010e);
+			double goodGsPercentage = (double)goodGsIndicators / (double)tsContext.PacketsRead;
+			if (goodGsPercentage >= 0.95)
+				return TsType.GS_STID135;
+
+			int occupiedPids = tsContext.CountPidsWithTraffic();
+			if (occupiedPids == 2)
+			{
+				ulong blockstreamIndicators = tsContext.GetNumberOfPacketsOnPid(0x0020);
+				double blockstreamPercentage = (double)blockstreamIndicators / (double)tsContext.PacketsRead;
+				if (blockstreamPercentage >= 0.95 && tsContext.GetNumberOfPacketsOnPid(0x1fff) > 0)
+					return TsType.BLOCKSTREAM;
+
+				ulong docsisIndicators = tsContext.GetNumberOfPacketsOnPid(0x1ffe);
+				double docsisPercentage = (double)docsisIndicators / (double)tsContext.PacketsRead;
+				if (docsisPercentage >= 0.08 && tsContext.GetNumberOfPacketsOnPid(0x1fff) > 0)
+					return TsType.DOCSIS;
+
+				ulong nullIndicators = tsContext.GetNumberOfPacketsOnPid(0x1fff);
+				double nullPercentage = (double)nullIndicators / (double)tsContext.PacketsRead;
+				if (nullPercentage >= 0.95 && tsContext.GetNumberOfPacketsOnPid(0) > 0 && tsContext.GetNumberOfPacketsOnPid(0x1fff) > 0)
+					return TsType.BLANK_PAT_AND_NULLS;
+
+				ulong wdrDabIndicators = tsContext.GetNumberOfPacketsOnPid(0x0bb8);
+				double wdrDabPercentage = (double)wdrDabIndicators / (double)tsContext.PacketsRead;
+				if (wdrDabPercentage >= 0.7 && nullPercentage >= 0.2)
+					return TsType.LIKELY_WDR_DAB;
+			}
+			
+			return TsType.NORMAL_TS;
+		}
+
+		private uint GetPcrPid(TsContext tsContext)
+		{
+			if (tsContext.PcrMonitor == null)
+				return 8191;
+
+			return tsContext.PcrMonitor.SourcePid;
+		}
+
+		private TimeSpan GetPcrDuration(TsContext tsContext)
+		{
+			if (tsContext.PcrMonitor == null)
+				return TimeSpan.Zero;
+			
+			if (tsContext.PcrMonitor.PcrDuration == null)
+				return TimeSpan.Zero;
+
+			return tsContext.PcrMonitor.PcrDuration.Value;
+		}
+		private ulong CountOtherPackets(TsContext tsContext)
+		{
+			ulong result = 0;
+			for (int i = 1; i < 8191; i++)
+			{
+				result += tsContext.GetNumberOfPacketsOnPid(i);
+			}
+			return result;
+		}
+
+		private ulong CountOtherScrambledPacket(TsContext tsContext)
+		{
+			ulong result = 0;
+			for (int i = 1; i < 8191; i++)
+			{
+				result += tsContext.GetScrambledPacketsOnPid(i);
+			}
+			return result;
+		}
+
+		private ulong CountOtherTeiPacket(TsContext tsContext)
+		{
+			ulong result = 0;
+			for (int i = 1; i < 8191; i++)
+			{
+				result += tsContext.GetTeiPacketsOnPid(i);
+			}
+			return result;
+		}
+
+		private enum TsType
+		{
+			NORMAL_TS,
+			TOO_SMALL,
+			GS_STID135,
+			GS_OLD_STREAMREADER,
+			BLOCKSTREAM,
+			DOCSIS,
+			LIKELY_BAD_RECEPTION,
+			BLANK_PAT_AND_NULLS,
+			LIKELY_WDR_DAB
+		}
+	}
+}