diff --git a/.gitignore b/.gitignore index 8130e63..4b31f85 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ imgui.ini /.vs/skyscraper8/CopilotIndices/17.14.1231.31060 /.vs/skyscraper8/CopilotIndices/17.14.1290.42047 /Documentation/TSDuck-Samples/experiment2/*.ts +/.vs/skyscraper8/CopilotIndices/17.14.1431.25910 diff --git a/skyscraper8/Mpeg2/TsContext.cs b/skyscraper8/Mpeg2/TsContext.cs index f94245e..5fdba55 100644 --- a/skyscraper8/Mpeg2/TsContext.cs +++ b/skyscraper8/Mpeg2/TsContext.cs @@ -14,12 +14,15 @@ namespace skyscraper5.Mpeg2 private ITsPacketProcessor[] processors; private uint[] continuities; private ulong[] pidPackets; + private ulong[] scrambledPackets; + private ulong[] teiPackets; - public long PacketLossEvents { get; private set; } + public long PacketLossEvents { get; private set; } public bool FirstPacketDone { get; private set; } public long PacketsRead { get; private set; } public long TheoreticalOffset { get; private set; } public List FilterChain { get; set; } + public bool LikelyBrokenEncapsulation => this.encapsulationBrokenConfirmed; public void PushPacket(TsPacket packet) { @@ -44,6 +47,19 @@ namespace skyscraper5.Mpeg2 if (FilterChain == null || FilterChain.Count == 0) throw new InvalidOperationException("The filter chain has not been initialized."); + if (packet.TSC != 0) + { + if (scrambledPackets == null) + scrambledPackets = new ulong[0x2000]; + scrambledPackets[packet.PID]++; + } + + if (packet.TEI) + { + if (teiPackets == null) + teiPackets = new ulong[0x2000]; + teiPackets[packet.PID]++; + } CheckBrokenEncapsulation(packet); for (int i = 0; i < FilterChain.Count; i++) @@ -55,13 +71,14 @@ namespace skyscraper5.Mpeg2 return; } + EnsureContinuity(packet); + if (processors == null) return; if (processors[packet.PID] == null) return; - bool continuity = EnsureContinuity(packet); processors[packet.PID].PushPacket(packet); } @@ -109,6 +126,9 @@ namespace skyscraper5.Mpeg2 if (packet.AdaptionFieldControl == 2 || packet.AdaptionFieldControl == 0) return true; + if (packet.PID == 0x1fff) + return true; + uint pid = packet.PID; if (continuities[pid] == UInt32.MaxValue) { @@ -189,6 +209,18 @@ namespace skyscraper5.Mpeg2 return result; } + public int CountPidsWithTraffic() + { + int result = 0; + for (int i = 0; i < this.pidPackets.Length; i++) + { + if (this.pidPackets[i] > 0) + result++; + } + + return result; + } + public ulong[] GetPidStatistics() { if (pidPackets == null) @@ -203,10 +235,29 @@ namespace skyscraper5.Mpeg2 return pidPackets[pid]; } + public ulong GetScrambledPacketsOnPid(int pid) + { + if (scrambledPackets == null) + return 0; + else + return scrambledPackets[pid]; + } + + public ulong GetTeiPacketsOnPid(int pid) + { + if (teiPackets == null) + return 0; + else + { + return teiPackets[pid]; + } + } + #region TCP Proxy private bool _tcpProxyEnabled; private TcpTsProxy tcpTsProxy; + public bool TcpProxyEnabled { diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index 0f5607f..730088b 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -351,6 +351,16 @@ namespace skyscraper5 Stid135Test.Run(fi); return; } + + if (args[0].ToLowerInvariant().Equals("make-catalogue")) + { + DirectoryInfo di = new DirectoryInfo(args[1]); + FileInfo fi = new FileInfo(args[2]); + CatalogueGenerator catalogueGenerator = new CatalogueGenerator(di, fi); + catalogueGenerator.Run(); + catalogueGenerator.Dispose(); + return; + } } /*Passing passing = new Passing(); @@ -370,13 +380,15 @@ namespace skyscraper5 Console.WriteLine(" or: .\\skyscraper8.exe pcap-off - to write a configuration file that turns off PCAP writing during scraping."); Console.WriteLine(" or: .\\skyscraper8.exe subts-on - to write a configuration file that allows extraction of nested TS."); Console.WriteLine(" or: .\\skyscraper8.exe subts-off - to write a configuration file that turns off extraction of nested TS."); - Console.WriteLine(); + Console.WriteLine(); Console.WriteLine("default behaviour is pcap writing and nested TS writing on."); Console.WriteLine(); Console.WriteLine("Bonus features:"); Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC."); Console.WriteLine(".\\skyscraper8.exe hlsproxy-destructive \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC and delete HLS segments afterwards. (be careful!)"); Console.WriteLine(".\\skyscraper8.exe shannon \"C:\\some\\file.bmp\" - calculates the Shannon entropy value for any given file."); + Console.WriteLine(".\\skyscraper8.exe make-catalogue \"C:\\path\\to\\ts\\collection\\\" \"C:\\outputted_index.csv\" - generates a catalogue with core information about your TS files."); + Console.WriteLine(".\\skyscraper8.exe pts2bbf2 \"C:\\path\\to\\file.ts\\\" - extracts every single BBFrame from a GS to into a small file for each. (be careful, might generate many small files!)"); } private static void ToggleSubTsDumpConfiguration(bool enabled) diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index 07c01ef..8660a46 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\samples\\skyscraper_20251029_1201_0150W_12596_V_44995.ts\"", + "commandLineArgs": "make-catalogue \"D:\\\\Stash\\\\\" \"D:\\\\index.csv\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/CatalogueGenerator.cs b/skyscraper8/Skyscraper/CatalogueGenerator.cs new file mode 100644 index 0000000..a900e29 --- /dev/null +++ b/skyscraper8/Skyscraper/CatalogueGenerator.cs @@ -0,0 +1,259 @@ +using log4net; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using skyscraper5.Mpeg2; +using skyscraper5.Skyscraper.Scraper.Utils; +using skyscraper5.src.Mpeg2.PacketFilter; + +namespace skyscraper8.Skyscraper +{ + internal class CatalogueGenerator : IDisposable + { + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private readonly DirectoryInfo _inputDir; + private readonly FileStream _outputCsvStream; + private readonly StreamWriter _outputCsv; + private readonly List _nullPacketFilter; + private readonly PacketDiscarder _packetDiscarder; + + public CatalogueGenerator(DirectoryInfo inputDir, FileInfo outputCsv) + { + _inputDir = inputDir; + if (outputCsv.Exists) + { + string originalName = outputCsv.FullName; + string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(outputCsv.FullName); + string newName = Path.Combine(outputCsv.Directory.FullName,String.Format("{0}_{1}.bak", fileNameWithoutExtension, DateTime.Now.Ticks)); + logger.WarnFormat("The file {0} already exists. I don't want to overwrite it, so I'm gonna rename it to {1}:", originalName, newName); + File.Move(originalName, newName); + outputCsv.Refresh(); + } + + _outputCsvStream = outputCsv.Open(FileMode.CreateNew, FileAccess.Write, FileShare.Read); + _outputCsv = new StreamWriter(_outputCsvStream, Encoding.UTF8); + _outputCsv.WriteLine("FILE_NAME;GUESSED_TYPE;TOTAL_PACKETS;PIDS;PAT_PACKETS;OTHER_PACKETS;NULL_PACKETS;PAT_SCRAMBLED;OTHER_SCRAMBLED;NULL_SCRAMBLED;PAT_TEI;OTHER_TEI;NULL_TEI;PCR_PID;PCR_DURATION;DISCONTINUITIES;READ_ERROR;LOWEST_PID;"); + _nullPacketFilter = new List(); + _nullPacketFilter.Add(new NullPacketFilter()); + } + + public void Dispose() + { + _outputCsvStream.Dispose(); + } + + public void Run() + { + ScrapeDirectory(_inputDir); + } + + private void ScrapeDirectory(DirectoryInfo di) + { + logger.InfoFormat("Entering directory: {0}", di.FullName); + foreach (FileSystemInfo fileSystemInfo in di.GetFileSystemInfos()) + { + switch (fileSystemInfo) + { + case DirectoryInfo subdirectory: + ScrapeDirectory(subdirectory); + break; + case FileInfo fi: + ScrapeFile(fi); + break; + default: + logger.ErrorFormat("{0} is a {1}. Those are not supported yet.", fileSystemInfo.FullName, fileSystemInfo.GetType().Name); + break; + } + } + logger.InfoFormat("Leaving directory: {0}", di.FullName); + } + + private void ScrapeFile(FileInfo fi) + { + string extension = Path.GetExtension(fi.FullName); + extension = extension.ToLowerInvariant(); + if (!extension.Equals(".ts")) + return; + + if (fi.Length == 0) + { + _outputCsv.WriteLine("\"{0}\";TOO_SMALL;0;0;0;0;0;0;0;0;0;0;0;0;0;0;", fi.FullName); + _outputCsv.Flush(); + return; + } + + TsContext tsContext = new TsContext(); + tsContext.FilterChain = _nullPacketFilter; + tsContext.RegisterPacketProcessor(0x0000, _packetDiscarder); + int readError = 0; + + logger.InfoFormat("Reading: {0}", fi.Name); + FileStream fileStream = fi.OpenRead(); + byte[] buffer = new byte[188]; + try + { + for (long l = 0; l < fileStream.Length; l += 188) + { + if (fileStream.Read(buffer, 0, 188) != 188) + { + logger.ErrorFormat("Failed to read 188 bytes from offset {0} of file {1}, aborting reading it.", l, fi.Name); + readError = 1; + break; + } + tsContext.PushPacket(buffer); + } + } + catch (IOException e) + { + readError = 2; + logger.ErrorFormat("Failed to read from {0}: {1}", fi.Name, e.ToString()); + } + + string filename = fi.FullName; + TsType tsType = GuessTsType(tsContext); + long totalPackets = tsContext.PacketsRead + 1; + int pids = tsContext.CountPidsWithTraffic(); + ulong patPackets = tsContext.GetNumberOfPacketsOnPid(0); + ulong otherPackets = CountOtherPackets(tsContext); + ulong nullPackets = tsContext.GetNumberOfPacketsOnPid(8191); + ulong patScrambled = tsContext.GetScrambledPacketsOnPid(0); + ulong otherScrambled = CountOtherScrambledPacket(tsContext); + ulong nullScrambled = tsContext.GetScrambledPacketsOnPid(8191); + ulong patTei = tsContext.GetTeiPacketsOnPid(0); + ulong otherTei = CountOtherTeiPacket(tsContext); + ulong nullTei = tsContext.GetTeiPacketsOnPid(8191); + uint pcrPid = GetPcrPid(tsContext); + TimeSpan pcrDuration = GetPcrDuration(tsContext); + long discontinuities = tsContext.PacketLossEvents; + int lowestPid = GetLowestPid(tsContext); + + _outputCsv.WriteLine( + "\"{0}\";{1};{2};{3};{4};{5};{6};{7};{8};{9};{10};{11};{12};{13};{14};{15};{16};{17}", + filename, tsType, totalPackets, pids, + patPackets, otherPackets, + nullPackets, patScrambled, otherScrambled, nullScrambled, patTei, otherTei, nullTei, pcrPid, + pcrDuration, discontinuities, readError, lowestPid); + _outputCsv.Flush(); + } + + private int GetLowestPid(TsContext tsContext) + { + for (int i = 0; i < 8192; i++) + { + ulong numberOfPacketsOnPid = tsContext.GetNumberOfPacketsOnPid(i); + if (numberOfPacketsOnPid > 0) + return i; + } + + return -1; + } + + private TsType GuessTsType(TsContext tsContext) + { + if (tsContext.PacketsRead == 0) + return TsType.TOO_SMALL; + + if (tsContext.LikelyBrokenEncapsulation) + return TsType.LIKELY_BAD_RECEPTION; + + ulong brokenGsIndicators = tsContext.GetNumberOfPacketsOnPid(0x0118); + double brokenGsPercentage = (double)brokenGsIndicators / (double)tsContext.PacketsRead; + if (brokenGsPercentage >= 0.95) + return TsType.GS_OLD_STREAMREADER; + + ulong goodGsIndicators = tsContext.GetNumberOfPacketsOnPid(0x010e); + double goodGsPercentage = (double)goodGsIndicators / (double)tsContext.PacketsRead; + if (goodGsPercentage >= 0.95) + return TsType.GS_STID135; + + int occupiedPids = tsContext.CountPidsWithTraffic(); + if (occupiedPids == 2) + { + ulong blockstreamIndicators = tsContext.GetNumberOfPacketsOnPid(0x0020); + double blockstreamPercentage = (double)blockstreamIndicators / (double)tsContext.PacketsRead; + if (blockstreamPercentage >= 0.95 && tsContext.GetNumberOfPacketsOnPid(0x1fff) > 0) + return TsType.BLOCKSTREAM; + + ulong docsisIndicators = tsContext.GetNumberOfPacketsOnPid(0x1ffe); + double docsisPercentage = (double)docsisIndicators / (double)tsContext.PacketsRead; + if (docsisPercentage >= 0.08 && tsContext.GetNumberOfPacketsOnPid(0x1fff) > 0) + return TsType.DOCSIS; + + ulong nullIndicators = tsContext.GetNumberOfPacketsOnPid(0x1fff); + double nullPercentage = (double)nullIndicators / (double)tsContext.PacketsRead; + if (nullPercentage >= 0.95 && tsContext.GetNumberOfPacketsOnPid(0) > 0 && tsContext.GetNumberOfPacketsOnPid(0x1fff) > 0) + return TsType.BLANK_PAT_AND_NULLS; + + ulong wdrDabIndicators = tsContext.GetNumberOfPacketsOnPid(0x0bb8); + double wdrDabPercentage = (double)wdrDabIndicators / (double)tsContext.PacketsRead; + if (wdrDabPercentage >= 0.7 && nullPercentage >= 0.2) + return TsType.LIKELY_WDR_DAB; + } + + return TsType.NORMAL_TS; + } + + private uint GetPcrPid(TsContext tsContext) + { + if (tsContext.PcrMonitor == null) + return 8191; + + return tsContext.PcrMonitor.SourcePid; + } + + private TimeSpan GetPcrDuration(TsContext tsContext) + { + if (tsContext.PcrMonitor == null) + return TimeSpan.Zero; + + if (tsContext.PcrMonitor.PcrDuration == null) + return TimeSpan.Zero; + + return tsContext.PcrMonitor.PcrDuration.Value; + } + private ulong CountOtherPackets(TsContext tsContext) + { + ulong result = 0; + for (int i = 1; i < 8191; i++) + { + result += tsContext.GetNumberOfPacketsOnPid(i); + } + return result; + } + + private ulong CountOtherScrambledPacket(TsContext tsContext) + { + ulong result = 0; + for (int i = 1; i < 8191; i++) + { + result += tsContext.GetScrambledPacketsOnPid(i); + } + return result; + } + + private ulong CountOtherTeiPacket(TsContext tsContext) + { + ulong result = 0; + for (int i = 1; i < 8191; i++) + { + result += tsContext.GetTeiPacketsOnPid(i); + } + return result; + } + + private enum TsType + { + NORMAL_TS, + TOO_SMALL, + GS_STID135, + GS_OLD_STREAMREADER, + BLOCKSTREAM, + DOCSIS, + LIKELY_BAD_RECEPTION, + BLANK_PAT_AND_NULLS, + LIKELY_WDR_DAB + } + } +}