From f83b920af463d8087cddc334671e8b785e09d2e6 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Fri, 19 Sep 2025 08:48:16 +0200 Subject: [PATCH] GSE testing --- Documentation/storage_system_ids.md | 7 + .../Jobs/InheritedBlindscanConfigWindow.cs | 2 + skyscraper8/DvbNip/DvbNipReceiver.cs | 1 - skyscraper8/Program.cs | 68 ++++++-- skyscraper8/Properties/launchSettings.json | 2 +- skyscraper8/QuickAndDirtySatIpClient.cs | 20 ++- .../Skyscraper/BbframeDeencapsulator.cs | 156 ++++++++++++++++++ .../Skyscraper/DigitalDevicesBbFrameReader.cs | 29 +++- skyscraper8/Skyscraper/IO/M3U8Stream.cs | 2 +- skyscraper8/Skyscraper/IpPacketFinder.cs | 37 ++++- .../Math/EntropyCalculatorStream.cs | 110 ++++++++++++ .../Skyscraper/Net/IpTrafficHandler.cs | 13 ++ .../Skyscraper/Net/NullIpTrafficHandler.cs | 27 +++ .../Net/Pcap/PcapIpTrafficHandler.cs | 46 ++++++ .../Skyscraper/Plugins/PluginManager.cs | 55 +++++- .../Skyscraper/Scraper/SkyscraperContext.cs | 37 ++--- .../Storage/StorageConnectionManager.cs | 48 ++++++ 17 files changed, 610 insertions(+), 50 deletions(-) create mode 100644 skyscraper8/Skyscraper/BbframeDeencapsulator.cs create mode 100644 skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs create mode 100644 skyscraper8/Skyscraper/Net/IpTrafficHandler.cs create mode 100644 skyscraper8/Skyscraper/Net/NullIpTrafficHandler.cs create mode 100644 skyscraper8/Skyscraper/Net/Pcap/PcapIpTrafficHandler.cs diff --git a/Documentation/storage_system_ids.md b/Documentation/storage_system_ids.md index 6103b1e..9f99f27 100644 --- a/Documentation/storage_system_ids.md +++ b/Documentation/storage_system_ids.md @@ -12,3 +12,10 @@ These contain blob-like data, like Screenshots, DSM-CC Carousels, etc. |--|----------| | 1|MinIO | | 2|Filesystem| + +#IP Traffic handlers +These will have IP packets forwarded to. +|ID|Type | +|--|-----------| +| 1|PCAP Writer| +| 2|Discard | \ No newline at end of file diff --git a/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanConfigWindow.cs b/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanConfigWindow.cs index 27e7c14..bb6f137 100644 --- a/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanConfigWindow.cs +++ b/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanConfigWindow.cs @@ -170,6 +170,8 @@ namespace SDL2Demo.Jobs ImGui.Text("This assumes that both tuners are connected to the same antenna set-up, using e.g. a splitter."); ImGui.PushID("tunerB"); + if (settingsWindowSetFilterTunerSelection - 1 < tuners.Count) + settingsWindowSetFilterTunerSelection = 0; if (ImGui.BeginCombo("Tuner for BLScanEx and RFScan", tuners[settingsWindowSetFilterTunerSelection].Name)) { for (int i = 0; i < tuners.Count; i++) diff --git a/skyscraper8/DvbNip/DvbNipReceiver.cs b/skyscraper8/DvbNip/DvbNipReceiver.cs index 408ea7d..530b9eb 100644 --- a/skyscraper8/DvbNip/DvbNipReceiver.cs +++ b/skyscraper8/DvbNip/DvbNipReceiver.cs @@ -187,7 +187,6 @@ namespace skyscraper8.DvbNip rawSlepString = rawSlepString.Replace(" ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList(); foreach (SsdpDevice ssdpDevice in ssdpDevices) { @@ -282,7 +273,40 @@ namespace skyscraper5 qadsipc.Run(); return; } - } + + if (args[0].ToLowerInvariant().Equals("shannon")) + { + if (args.Length != 2) + { + Console.WriteLine("Specify ONE file, please."); + return; + } + FileInfo fi = new FileInfo(args[1]); + if (!fi.Exists) + { + Console.WriteLine("{0} doesn't exist.", fi.FullName); + return; + } + FileStream fileStream = fi.OpenRead(); + EntropyCalculatorStream entropyCalculatorStream = new EntropyCalculatorStream(); + fileStream.CopyTo(entropyCalculatorStream); + fileStream.Close(); + fileStream.Dispose(); + Console.WriteLine("Entropy value: {0} ({1}%)", entropyCalculatorStream.Entropy, entropyCalculatorStream.Percentage); + return; + } + + if (args[0].ToLowerInvariant().Equals("pcapon")) + { + TogglePcapConfiguration(1); + return; + } + if (args[0].ToLowerInvariant().Equals("pcapoff")) + { + TogglePcapConfiguration(2); + return; + } + } /*Passing passing = new Passing(); if (!passing.Boot()) @@ -296,12 +320,28 @@ namespace skyscraper5 Console.WriteLine(" or: .\\skyscraper8.exe \"C:\\path\\to\\file.ts\\"); Console.WriteLine(" or: .\\skyscraper8.exe \"C:\\path\\to\\my\\folder\\with\\ts\\files\\\" (for batch extraction)"); Console.WriteLine(" or: .\\skyscraper8.exe satip IP_ADDRESS DISEQC POLARITY FREQUENCY SYSTEM SYMBOL_RATE (see README file)"); + Console.WriteLine(" or: .\\skyscraper8.exe pcapon - to write a configuration file that allows PCAP writing during scraping."); + Console.WriteLine(" or: .\\skyscraper8.exe pcapoff - to write a configuration file that turns off PCAP writing during scraping."); Console.WriteLine(); - Console.WriteLine("Bonus feature:"); + Console.WriteLine("default behaviour is pcap writing on."); + Console.WriteLine(); + Console.WriteLine("Bonus features:"); Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC."); Console.WriteLine(".\\skyscraper8.exe hlsproxy-destructive \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC and delete HLS segments afterwards. (be careful!)"); + Console.WriteLine(".\\skyscraper8.exe shannon \"C:\\some\\file.bmp\" - calculates the Shannon entropy value for any given file."); + } + + private static void TogglePcapConfiguration(int value) + { + PluginManager pluginManager = PluginManager.GetInstance(); + if (!pluginManager.Ini.ContainsKey("ip_handler")) + pluginManager.Ini.Add("ip_handler", new IniSection()); + + pluginManager.Ini["ip_handler"]["type"] = value.ToString(); + pluginManager.SaveConfiguration(); + + Console.WriteLine("Wrote skyscraper5.ini."); } - private static void ProcessDirectory(DirectoryInfo di) { DataStorage dataStorage = new InMemoryScraperStorage(); diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index 4075ac8..6ebaf7b 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "cscan tcp://172.20.20.203:6969", + "commandLineArgs": "\"C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\638938290099956387.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/QuickAndDirtySatIpClient.cs b/skyscraper8/QuickAndDirtySatIpClient.cs index 360bd1f..80af382 100644 --- a/skyscraper8/QuickAndDirtySatIpClient.cs +++ b/skyscraper8/QuickAndDirtySatIpClient.cs @@ -20,6 +20,7 @@ namespace skyscraper8 internal class QuickAndDirtySatIpClient { private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private const bool ENABLE_TS_WRITER = true; public QuickAndDirtySatIpClient(string[] args) { @@ -114,7 +115,7 @@ namespace skyscraper8 objectStorage = new FilesystemStorage(new DirectoryInfo(".")); context = new SkyscraperContext(new TsContext(), dataStorage, objectStorage); context.EnableTimeout = true; - context.TimeoutSeconds = 10; + context.TimeoutSeconds = 60; context.InitalizeFilterChain(); RtspPlayResponse play = rtspClient.GetPlay(setup); @@ -150,6 +151,7 @@ namespace skyscraper8 rtspClient.Dispose(); } + private FileStream fs; private uint stuffingBytes; private Queue packetQueue; private ObjectStorage objectStorage; @@ -167,6 +169,8 @@ namespace skyscraper8 { packetQueue.Enqueue(buffer); } + + DumpPacket(buffer); i += 187; } else if (data[i] == 0xff) @@ -176,6 +180,20 @@ namespace skyscraper8 } } + private void DumpPacket(byte[] buffer) + { + if (!ENABLE_TS_WRITER) + return; + + if (fs == null) + { + string fname = String.Format("{0}.ts", DateTime.Now.Ticks); + fs = File.OpenWrite(fname); + } + + fs.Write(buffer, 0, buffer.Length); + } + private int rtcps; private void Setup_OnRtcpPacket(byte[] data, int length) { diff --git a/skyscraper8/Skyscraper/BbframeDeencapsulator.cs b/skyscraper8/Skyscraper/BbframeDeencapsulator.cs new file mode 100644 index 0000000..595b1f8 --- /dev/null +++ b/skyscraper8/Skyscraper/BbframeDeencapsulator.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.NetworkInformation; +using System.Text; +using System.Threading.Tasks; +using skyscraper5.Mpeg2; +using skyscraper5.Skyscraper; +using skyscraper5.Skyscraper.IO; + +namespace skyscraper8.Skyscraper +{ + internal class BbframeDeencapsulator + { + private bool interruptedGseHem; + private MemoryStream interruptedGseHemBuffer; + private bool interruptedGseHemWantsCrc32; + + public void PushPacket(byte[] bbframe) + { + MemoryStream ms = new MemoryStream(bbframe, false); + //BBHeader + byte matype1 = ms.ReadUInt8(); + int tsGsField = (matype1 & 0xc0) >> 6; + bool sisMisField = (matype1 & 0x20) != 0; + bool ccmAcmField = (matype1 & 0x10) != 0; + bool issyi = (matype1 & 0x08) != 0; + bool npd = (matype1 & 0x04) != 0; + int ro = (matype1 & 0x03); + + byte matype2 = ms.ReadUInt8(); + + ushort userPacketLength = ms.ReadUInt16BE(); + ushort dataFieldLength = ms.ReadUInt16BE(); + byte sync = ms.ReadUInt8(); + ushort syncd = ms.ReadUInt16BE(); + byte crc8 = ms.ReadUInt8(); + + switch (tsGsField) + { + case 2: + int bytes = dataFieldLength / 8; + if (ms.GetAvailableBytes() < bytes) + return; + HandleGse(ms); + break; + default: //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts + throw new NotImplementedException(String.Format("TS/GS field says 0x{0:X2}", tsGsField)); + } + } + + private bool isGseSynced; + private int gseNeeded; + private MemoryStream gseNeededBuffer; + private void HandleGse(MemoryStream ms) + { + if (!isGseSynced) + { + ms.Position = 10; + byte syncByte = ms.ReadUInt8(); + ms.Position = 10; + if ((syncByte & 0xc0) == 0xc0) + { + isGseSynced = true; + } + else + { + return; + } + } + if (gseNeeded > 0) + { + int blockSize = (int)System.Math.Min(gseNeeded, ms.GetAvailableBytes()); + gseNeededBuffer.Write(ms.ReadBytes(blockSize), 0, blockSize); + gseNeeded -= blockSize; + if (gseNeeded == 0) + { + int payloadSize = (int)gseNeededBuffer.Position; + gseNeededBuffer.Position = 0; + byte[] payloadBuffer = gseNeededBuffer.ReadBytes(payloadSize); + } + } + while (ms.GetAvailableBytes() > 2) + { + //GSE-Header + byte byteA = ms.ReadUInt8(); + bool startIndicator = (byteA & 0x80) != 0; + bool endIndicator = (byteA & 0x40) != 0; + int labelIndicator = (byteA & 0x30) >> 4; + if (!startIndicator && !endIndicator && labelIndicator == 0) + { + //end of base band frame + return; + } + else + { + byte byteB = ms.ReadUInt8(); + int gseLength = byteA & 0x0f; + gseLength <<= 8; + gseLength += byteB; + + if (!startIndicator || !endIndicator) + { + byte fragId = ms.ReadUInt8(); + gseLength--; + } + + if (startIndicator && !endIndicator) + { + ushort totalLength = ms.ReadUInt16BE(); + gseLength -= 2; + } + + if (startIndicator) + { + ushort protocolType = ms.ReadUInt16BE(); + gseLength -= 2; + if (labelIndicator == 0) + { + PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6)); + gseLength -= 6; + } + else if (labelIndicator == 1) + { + byte[] threeByteLabel = ms.ReadBytes(3); + gseLength -= 3; + } + } + + if (!startIndicator && endIndicator) + gseLength -= 4; + + int startCrc32 = (int)ms.Position; + + if (gseLength > ms.GetAvailableBytes()) + { + gseNeeded = gseLength; + gseNeededBuffer = new MemoryStream(); + gseNeeded -= (int)ms.GetAvailableBytes(); + gseNeededBuffer.Write(ms.ReadBytes(ms.GetAvailableBytes())); + return; + } + byte[] payload = ms.ReadBytes(gseLength); + if (!startIndicator && endIndicator) + { + uint crc32 = ms.ReadUInt32BE(); + int endCrc32 = (int)ms.Position; + DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32); + } + + + } + } + } + } +} diff --git a/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs b/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs index d1c039f..03872c8 100644 --- a/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs +++ b/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs @@ -6,18 +6,22 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using skyscraper8.Skyscraper; namespace skyscraper5.src.Skyscraper { internal class DigitalDevicesBbFrameReader : ITsPacketProcessor { private IGsEventHandler mpeEventHandler; + private BbframeDeencapsulator deencapsulator; public DigitalDevicesBbFrameReader(IGsEventHandler mpeEventHandler) { this.mpeEventHandler = mpeEventHandler; } + private long packetsRecovered; + private long packetsLost; private MemoryStream outbuf; private bool annoncementDone; public void PushPacket(TsPacket packet) @@ -35,7 +39,23 @@ namespace skyscraper5.src.Skyscraper if (outbuf != null) { byte[] chi = outbuf.ToArray(); - byte[] ipPacket = IpPacketFinder.LookForIpPacket(outbuf.ToArray(), 32); + if (deencapsulator == null) + deencapsulator = new BbframeDeencapsulator(); + deencapsulator.PushPacket(chi); + /*List ipPackets = IpPacketFinder.LookForIpPackets(outbuf.ToArray()); + if (ipPackets != null) + { + foreach (byte[] ipPacket in ipPackets) + { + mpeEventHandler.OnIpDatagram(0x118, ipPacket); + packetsRecovered++; + } + } + else + { + packetsLost++; + }*/ + /*byte[] ipPacket = IpPacketFinder.LookForIpPacket(outbuf.ToArray(), 32); if (ipPacket != null) { if (!annoncementDone) @@ -44,10 +64,15 @@ namespace skyscraper5.src.Skyscraper annoncementDone = true; } mpeEventHandler.OnIpDatagram(0x0118, ipPacket); + packetsRecovered++; } + else + { + packetsLost++; + }*/ } outbuf = new MemoryStream(); - outbuf.Write(packets,8, packets[7]); + outbuf.Write(packets,9, packets[7] - 1); } else { diff --git a/skyscraper8/Skyscraper/IO/M3U8Stream.cs b/skyscraper8/Skyscraper/IO/M3U8Stream.cs index f02f9f9..59ac4b7 100644 --- a/skyscraper8/Skyscraper/IO/M3U8Stream.cs +++ b/skyscraper8/Skyscraper/IO/M3U8Stream.cs @@ -131,7 +131,7 @@ namespace skyscraper8.Skyscraper.IO segmentNames = newLines; return true; } - for (int i = 0; i < Math.Min(newLines.Length, segmentNames.Length); i++) + for (int i = 0; i < System.Math.Min(newLines.Length, segmentNames.Length); i++) { if (!segmentNames[i].Equals(newLines[i])) { diff --git a/skyscraper8/Skyscraper/IpPacketFinder.cs b/skyscraper8/Skyscraper/IpPacketFinder.cs index 915cba1..45dd2bc 100644 --- a/skyscraper8/Skyscraper/IpPacketFinder.cs +++ b/skyscraper8/Skyscraper/IpPacketFinder.cs @@ -13,7 +13,7 @@ namespace skyscraper5.Skyscraper { byte[] ipBuffer = new byte[20]; - for (int i = 1; i < searchDepth; i++) + for (int i = 0; i < searchDepth; i++) { int v4HeaderEnd = i + 20; if (v4HeaderEnd > gsPacket.Length) @@ -39,5 +39,40 @@ namespace skyscraper5.Skyscraper return null; } + + public static List LookForIpPackets(byte[] gsPacket) + { + List result = null; + + byte[] ipBuffer = new byte[20]; + for (int offset = 0; offset < gsPacket.Length; offset++) + { + int v4HeaderEnd = offset + 20; + if (v4HeaderEnd > gsPacket.Length) + break; + + if (gsPacket[offset] == 0x45) + { + Array.Copy(gsPacket, offset, ipBuffer, 0, ipBuffer.Length); + InternetHeader ipv4 = new(ipBuffer); + if (!ipv4.ChecksumValid) + continue; + int payloadStart = offset + ipBuffer.Length; + int payloadEnd = offset + ipv4.TotalLength; + if (payloadEnd > gsPacket.Length) + break; + int packetLength = payloadEnd - offset; + + byte[] finalPacket = new byte[packetLength]; + Array.Copy(gsPacket, offset, finalPacket, 0, packetLength); + if (result == null) + result = new List(); + result.Add(finalPacket); + offset = payloadEnd; + } + } + + return result; + } } } diff --git a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs new file mode 100644 index 0000000..0b8d80c --- /dev/null +++ b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Math +{ + internal class EntropyCalculatorStream : Stream + { + public override void Flush() + { + //no actual writes done, so nothing to flush. + } + + public override int Read(byte[] buffer, int offset, int count) + { + //this is a sink, so nothing to read here. + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + if (offset == 0) + return 0; + + //this is a sink, so we can't do actual seeks + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + //TODO: allow this later, although I have no idea why one would actually want this. + throw new NotSupportedException(); + } + + private double _entropy; + private long[] freq; + private long internalPosition; + public override void Write(byte[] buffer, int offset, int count) + { + //Count the bytes + if (freq == null) + freq = new long[byte.MaxValue + 1]; + for (int i = 0; i < count; i++) + freq[buffer[i + offset]]++; + internalPosition += count; + + + //Actually calculate the entropy here. + double newEntropy = 0.0; + for (int i = 0; i < freq.Length; i++) + { + double p_i = (double)freq[i] / (double)internalPosition; + if (p_i > 0) + newEntropy -= p_i * System.Math.Log2(p_i); + } + + //Raise events if we feel like it + if (newEntropy != Entropy) + { + double oldEntropy = Entropy; + double oldPercentage = Percentage; + double newPercentage = (newEntropy / 8.0) * 100.0; + if (newEntropy > Entropy) + EntropyRising?.Invoke(oldEntropy, newEntropy, oldPercentage, newPercentage); + else if (newEntropy < Entropy) + EntropyFalling?.Invoke(oldEntropy, newEntropy, oldPercentage, newPercentage); + } + + _entropy = newEntropy; + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => internalPosition; + public override long Position + { + get + { + return internalPosition; + } + set + { + throw new NotSupportedException(); + } + } + public double Entropy + { + get + { + return _entropy; + } + } + + public double Percentage + { + get + { + return (Entropy / 8.0) * 100.0; + } + } + + public delegate void EntropyCallback(double oldEntropy, double newEntropy, double oldPercentage, double newPercentage); + + public event EntropyCallback EntropyRising; + public event EntropyCallback EntropyFalling; + } +} diff --git a/skyscraper8/Skyscraper/Net/IpTrafficHandler.cs b/skyscraper8/Skyscraper/Net/IpTrafficHandler.cs new file mode 100644 index 0000000..6346c9f --- /dev/null +++ b/skyscraper8/Skyscraper/Net/IpTrafficHandler.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Net +{ + internal interface IpTrafficHandler : IDisposable + { + void HandlePacket(int pid, byte[] payload); + } +} diff --git a/skyscraper8/Skyscraper/Net/NullIpTrafficHandler.cs b/skyscraper8/Skyscraper/Net/NullIpTrafficHandler.cs new file mode 100644 index 0000000..6b84024 --- /dev/null +++ b/skyscraper8/Skyscraper/Net/NullIpTrafficHandler.cs @@ -0,0 +1,27 @@ +using log4net; +using skyscraper5.Skyscraper.Plugins; +using skyscraper8.Skyscraper.Scraper.Storage; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Net +{ + [SkyscraperPlugin] + [StorageId(2)] + [StorageName("Packet Discarder")] + internal class NullIpTrafficHandler : IpTrafficHandler + { + public void Dispose() + { + + } + + public void HandlePacket(int pid, byte[] payload) + { + + } + } +} diff --git a/skyscraper8/Skyscraper/Net/Pcap/PcapIpTrafficHandler.cs b/skyscraper8/Skyscraper/Net/Pcap/PcapIpTrafficHandler.cs new file mode 100644 index 0000000..b2faa0b --- /dev/null +++ b/skyscraper8/Skyscraper/Net/Pcap/PcapIpTrafficHandler.cs @@ -0,0 +1,46 @@ +using log4net; +using skyscraper5.Skyscraper.Net.Pcap; +using skyscraper5.Skyscraper.Plugins; +using skyscraper8.Skyscraper.Scraper.Storage; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Net.Pcap +{ + [SkyscraperPlugin] + [StorageId(1)] + [StorageName("PCAP Writer")] + internal class PcapIpTrafficHandler : IpTrafficHandler + { + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private PcapWriter[] pcapWriters; + public void HandlePacket(int pid, byte[] payload) + { + if (pcapWriters == null) + pcapWriters = new PcapWriter[0x1fff]; + if (pcapWriters[pid] == null) + { + string fname = String.Format("{0:X4},{1}.pcap", pid, DateTime.Now.Ticks); + logger.InfoFormat("Opening file for writing: {0}", fname); + FileStream fs = File.OpenWrite(fname); + pcapWriters[pid] = new PcapWriter(fs, TcpdumpNetworkType.RawIp); + } + pcapWriters[pid].WritePacket(payload); + } + + public void Dispose() + { + if (pcapWriters != null) + { + for (int i = 0; i < pcapWriters.Length; i++) + { + if (pcapWriters[i] != null) + pcapWriters[i].Dispose(); + } + } + } + } +} diff --git a/skyscraper8/Skyscraper/Plugins/PluginManager.cs b/skyscraper8/Skyscraper/Plugins/PluginManager.cs index fcadb7b..05e920a 100644 --- a/skyscraper8/Skyscraper/Plugins/PluginManager.cs +++ b/skyscraper8/Skyscraper/Plugins/PluginManager.cs @@ -17,6 +17,9 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; +using skyscraper8.Skyscraper.Net; +using skyscraper8.Skyscraper.Net.Pcap; +using skyscraper8.Skyscraper.Plugins; using skyscraper8.Skyscraper.Scraper.Storage; using skyscraper8.Skyscraper.Text; @@ -42,8 +45,11 @@ namespace skyscraper5.Skyscraper.Plugins _mpeg2ExtensionDescriptors = new ConstructorInfo[256]; ScanAssembly(this.GetType().Assembly); - - FileInfo iniFileInfo = new FileInfo(iniFilename); + + FileInfo skyscraperMainAssembly = GetSkyscraperMainAssembly(); + DirectoryInfo skyscraperHome = skyscraperMainAssembly.Directory; + + iniFileInfo = new FileInfo(Path.Combine(skyscraperHome.FullName, iniFilename)); if (iniFileInfo.Exists) { Ini = new Ini(iniFileInfo.FullName); @@ -76,12 +82,13 @@ namespace skyscraper5.Skyscraper.Plugins else { Debug.WriteLine(String.Format("{0} was not found. Create it using the UI!", iniFileInfo.FullName)); + Ini = new Ini(); } - FileInfo fi = GetSkyscraperMainAssembly(); - if (fi != null) + + if (skyscraperMainAssembly != null) { - DirectoryInfo directory = fi.Directory; + DirectoryInfo directory = skyscraperMainAssembly.Directory; logger.DebugFormat("Found skyscraper main assembly at: {0}", directory.FullName); FileInfo[] fileInfos = directory.GetFiles("skyscraper5.*.dll"); foreach (FileInfo fileInfo in fileInfos) @@ -173,6 +180,7 @@ namespace skyscraper5.Skyscraper.Plugins } + private FileInfo iniFileInfo; private const string iniFilename = "skyscraper5.ini"; private static PluginManager _instance; @@ -199,6 +207,7 @@ namespace skyscraper5.Skyscraper.Plugins private Type filesystemProcessorType = typeof(FilesystemProcessorPlugin); private Type textDecoderType = typeof(TextDecoder); private PluginPrioritySorter sorter = new PluginPrioritySorter(); + private Type ipTrafficHandler = typeof(IpTrafficHandler); private List _dnsParsers; private List _mpePlugins; @@ -214,6 +223,7 @@ namespace skyscraper5.Skyscraper.Plugins private ConstructorInfo[] _dsmCcMessages; private ConstructorInfo[] _mpeg2ExtensionDescriptors; private TextDecoder[] _textDecoders; + private Dictionary _ipTrafficHandles; private void ScanAssembly(Assembly assembly) { @@ -319,13 +329,36 @@ namespace skyscraper5.Skyscraper.Plugins HandleTextDecoder(type); continue; } + else if (type.IsAssignableTo(ipTrafficHandler)) + { + HandleIpTrafficHandler(type); + continue; + } - throw new NotImplementedException(); + throw new NotImplementedException(); } _mpePlugins.Sort(sorter); } + private void HandleIpTrafficHandler(Type type) + { + Attribute attribute = type.GetCustomAttribute(typeof(StorageIdAttribute)); + if (attribute == null) + throw new PluginsException(String.Format("The type {0} does not have a {1}", type.Name, typeof(StorageIdAttribute))); + + if (_ipTrafficHandles == null) + _ipTrafficHandles = new Dictionary(); + + StorageIdAttribute sia = (StorageIdAttribute)attribute; + if (_ipTrafficHandles.ContainsKey(sia.Id)) + { + throw new PluginsException(String.Format("Multiple {0} claim to have ID {1}. {2} <-> {3}", nameof(IpTrafficHandler), sia.Id, _ipTrafficHandles[sia.Id].Name,type.Name)); + } + + _ipTrafficHandles.Add(sia.Id, type); + } + private void HandleTextDecoder(Type type) { List idAttributes = type.GetCustomAttributes().ToList(); @@ -701,6 +734,11 @@ namespace skyscraper5.Skyscraper.Plugins return _dataStorages.AsReadOnly(); } + public ReadOnlyDictionary GetIpTrafficHandlers() + { + return _ipTrafficHandles.AsReadOnly(); + } + public void AutoconfigureObject(string categoryName, object targetObject) { if (Ini == null) @@ -743,5 +781,10 @@ namespace skyscraper5.Skyscraper.Plugins TextDecoder[] saneClone = (TextDecoder[])clone; return saneClone; } + + public void SaveConfiguration() + { + Ini.Export(iniFileInfo); + } } } diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index e1adfc2..4a7bdbb 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -78,6 +78,7 @@ using System.Security.Policy; using System.Text; using skyscraper8.Experimentals.NdsSsu; using skyscraper8.Experimentals.OtvSsu; +using skyscraper8.Skyscraper.Net; using skyscraper8.Skyscraper.Scraper; using Tsubasa.IO; using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform; @@ -95,7 +96,7 @@ namespace skyscraper5.Skyscraper.Scraper { public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true; public const bool ALLOW_FFMPEG_FRAMEGRABBER = true; - public const bool ENABLE_MPE_TO_PCAP = true; + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); public TsContext DvbContext { get; } @@ -1658,20 +1659,17 @@ namespace skyscraper5.Skyscraper.Scraper return; } - private PcapWriter[] pcapWriters; + + private IpTrafficHandler ipTrafficHandler; public void OnIpDatagram(int pid, byte[] payload) { - if (ENABLE_MPE_TO_PCAP) - { - if (pcapWriters == null) - pcapWriters = new PcapWriter[0x1fff]; - if (pcapWriters[pid] == null) - { - FileStream fs = File.OpenWrite(String.Format("{0:X4},{1}.pcap", pid,DateTime.Now.Ticks)); - pcapWriters[pid] = new PcapWriter(fs, TcpdumpNetworkType.RawIp); - } - pcapWriters[pid].WritePacket(payload); - } + if (ipTrafficHandler == null) + { + StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance(); + ipTrafficHandler = storageConnectionManager.GetDefaultIpTrafficHandler(); + } + + ipTrafficHandler?.HandlePacket(pid, payload); int ipVersion = (payload[0] & 0xf0) >> 4; if (ipVersion == 4) @@ -2039,19 +2037,12 @@ namespace skyscraper5.Skyscraper.Scraper if (TcpProxyEnabled) TcpProxyEnabled = false; - - if (ENABLE_MPE_TO_PCAP) + if (ipTrafficHandler != null) { - if (pcapWriters != null) - { - for (int i = 0; i < pcapWriters.Length; i++) - { - if (pcapWriters[i] != null) - pcapWriters[i].Dispose(); - } - } + ipTrafficHandler.Dispose(); } + if (DvbContext.TcpProxyEnabled) { diff --git a/skyscraper8/Skyscraper/Scraper/Storage/StorageConnectionManager.cs b/skyscraper8/Skyscraper/Scraper/Storage/StorageConnectionManager.cs index 7102f45..02e1a3b 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/StorageConnectionManager.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/StorageConnectionManager.cs @@ -8,7 +8,10 @@ using System.Linq; using System.Reflection; using System.Text; using System.Threading.Tasks; +using skyscraper5.Skyscraper.Net; using skyscraper5.Skyscraper.Scraper.Storage; +using skyscraper8.Skyscraper.Net; +using skyscraper8.Skyscraper.Net.Pcap; namespace skyscraper8.Skyscraper.Scraper.Storage { @@ -139,5 +142,50 @@ namespace skyscraper8.Skyscraper.Scraper.Storage { return plugins.GetDataStorages(); } + + private bool IsIpTafficHandlerConfigured() + { + if (ini == null) + return false; + if (!ini.ContainsKey("ip_handler")) + return false; + if (!ini["ip_handler"].ContainsKey("type")) + return false; + return true; + } + + + internal IpTrafficHandler GetDefaultIpTrafficHandler() + { + Type targetType; + if (IsIpTafficHandlerConfigured()) + { + ReadOnlyDictionary handlers = plugins.GetIpTrafficHandlers(); + int type = Int32.Parse(ini["ip_handler"]["type"]); + if (!handlers.ContainsKey(type)) + { + logger.WarnFormat("The IP traffic handler with ID {0} was not found. I'm gonna assume you want the pcap files.", type); + targetType = typeof(PcapIpTrafficHandler); + } + else + { + targetType = handlers[type]; + } + } + else + { + logger.WarnFormat("You didn't call \"pcapon\" or \"pcapoff\" before. I'm gonna assume you want the pcap files."); + targetType = typeof(PcapIpTrafficHandler); + } + + ConstructorInfo constructorInfo = targetType.GetConstructor(new Type[] { }); + if (constructorInfo == null) + { + throw new ScraperStorageException(String.Format("The type {0} doesn't contain a parameterless constructor.", targetType.Name)); + } + + object invoke = constructorInfo.Invoke(new object[] { }); + return (IpTrafficHandler) invoke; + } } }