From 3595b51fcc147b6c1475afc9c94eeba6f4161c7b Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Fri, 5 Sep 2025 23:00:13 +0200 Subject: [PATCH] Implemented some missing storage methods. --- .gitignore | 1 + .../Jobs/InheritedBlindscanUiJunction.cs | 2 + skyscraper8/Program.cs | 84 ++++++++++++------- skyscraper8/Properties/launchSettings.json | 2 +- .../Skyscraper/Scraper/CaSystemNames.cs | 3 +- .../Storage/Filesystem/FilesystemStorage.cs | 82 +++++++++++++++--- .../InMemory/InMemoryScraperStorage.cs | 28 ++++++- skyscraper8/skyscraper8.csproj | 10 +++ 8 files changed, 168 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index cdef35e..749f173 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,4 @@ imgui.ini /GUIs/skyscraper8.UI.ImGui.MonoGame/obj/Debug/net8.0 /GUIs/skyscraper8.UI.ImGui.MonoGame/obj /.vs/skyscraper8/CopilotIndices/17.14.995.13737 +/.vs/skyscraper8/config diff --git a/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs b/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs index fe66dca..93b3204 100644 --- a/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs +++ b/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs @@ -1742,6 +1742,8 @@ namespace SDL2Demo.Jobs jobContext.Renderables.Add(this); }); foundFrequenciesWindow.allowZapNow = true; + foundFrequenciesWindow.statusPacketsInQueue = 0; + foundFrequenciesWindow.statusPacketsInTotal = 0; } public void OnBlindscanScrapeStopCondition() diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index 74e5bed..ce2a669 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -24,7 +24,9 @@ using System.Diagnostics; using System.Net; using System.Net.NetworkInformation; using System.Net.Sockets; +using System.Reflection; using System.Runtime.InteropServices; +using skyscraper5.Mpeg2.Descriptors; using skyscraper5.Skyscraper.Scraper.StreamAutodetection; using skyscraper8; using skyscraper8.SatIp; @@ -37,6 +39,7 @@ namespace skyscraper5 class Program { private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private const int PUBLIC_RELEASE = 4; private static void IntegrationTest() { /*List ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList(); @@ -44,8 +47,7 @@ namespace skyscraper5 { Console.WriteLine("SSDP device: {0}", ssdpDevice.Server); }*/ - - Console.WriteLine("yeet!"); + /*RtspClient rtspClient = new RtspClient("172.20.20.121", 554); rtspClient.AutoReconnect = true; RtspOptionsResponse options = rtspClient.GetOptions("/"); @@ -100,8 +102,8 @@ namespace skyscraper5 { IntegrationTest(); - logger.Info("Hello!"); - ffmpegFrameGrabber.CanStart(); + logger.InfoFormat(String.Format("Hello! This is skyscraper8, public release #{0}, code version {1}", PUBLIC_RELEASE, GetCurrentAssemblyDisplayVersion())); + logger.DebugFormat("Found {0}-bit Operating system.", Environment.Is64BitOperatingSystem ? 64 : 32); logger.DebugFormat("I'm a {0}-bit Process.", Environment.Is64BitProcess ? 64 : 32); @@ -286,6 +288,7 @@ namespace skyscraper5 Console.WriteLine("You're not supposed to run me directly. Run me from the commandline using the following:"); Console.WriteLine("for example: .\\skyscraper8.exe cscan tcp://127.0.0.1:6969"); Console.WriteLine(" or: .\\skyscraper8.exe \"C:\\path\\to\\file.ts\\"); + Console.WriteLine(" or: .\\skyscraper8.exe \"C:\\path\\to\\my\\folder\\with\\ts\\files\\\" (for batch extraction)"); Console.WriteLine(); Console.WriteLine("Bonus feature:"); Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC."); @@ -295,30 +298,39 @@ namespace skyscraper5 { DataStorage dataStorage = new InMemoryScraperStorage(); FilesystemStorage filesystemStorage = new FilesystemStorage(new DirectoryInfo(".")); - - //DirectoryInfo di = new DirectoryInfo(@"E:\Skyscraper\Astra 19.2"); - FileInfo[] fileInfos = di.GetFiles("*.ts"); - foreach (FileInfo fileInfo in fileInfos) - { - Console.WriteLine(new string('_', Console.WindowWidth - 1)); - Console.WriteLine("Processing: {0}", fileInfo.Name); - SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), dataStorage,filesystemStorage); - //StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest(); - FileStream fileStream = fileInfo.OpenRead(); - - Stopwatch stopwatch = new Stopwatch(); - stopwatch.Start(); - skyscraper.IngestFromStream(fileStream); - //streamTypeAutodetectionTest.IngestFromStream(fileStream); - stopwatch.Stop(); - - Console.WriteLine("Time to process: " + stopwatch.Elapsed.TotalSeconds); - - Console.WriteLine("File was: {0}", fileInfo.Name); - PrintPidStatistics(skyscraper); - } + ProcessDirectory(di, dataStorage, filesystemStorage); } + private static void ProcessDirectory(DirectoryInfo di, DataStorage dataStorage, ObjectStorage objectStorage) + { + DirectoryInfo[] directoryInfos = di.GetDirectories(); + foreach (DirectoryInfo subdir in directoryInfos) + { + ProcessDirectory(subdir, dataStorage, objectStorage); + } + FileInfo[] fileInfos = di.GetFiles("*.ts"); + foreach (FileInfo fileInfo in fileInfos) + { + Console.WriteLine(new string('_', Console.WindowWidth - 1)); + Console.WriteLine("Processing: {0}", fileInfo.Name); + SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), dataStorage, objectStorage); + skyscraper.InitalizeFilterChain(); + //StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest(); + FileStream fileStream = fileInfo.OpenRead(); + + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + skyscraper.IngestFromStream(fileStream); + //streamTypeAutodetectionTest.IngestFromStream(fileStream); + stopwatch.Stop(); + + Console.WriteLine("Time to process: " + stopwatch.Elapsed.TotalSeconds); + + Console.WriteLine("File was: {0}", fileInfo.Name); + PrintPidStatistics(skyscraper); + } + } + private static void PrintPidStatistics(SkyscraperContext skyscraper) { Console.WriteLine("Unattached PIDs:"); @@ -440,8 +452,6 @@ namespace skyscraper5 skyscraper.Dispose(); } - - private static void HandleCrazyScanToLiveSystem(string url, bool withTimeout = true) { StorageConnectionManager connectionManager = StorageConnectionManager.GetInstance(); @@ -798,5 +808,23 @@ namespace skyscraper5 } } + + private static string GetCurrentAssemblyDisplayVersion() + { + try + { + Assembly executingAssembly = System.Reflection.Assembly.GetExecutingAssembly(); + AssemblyName assemblyName = executingAssembly.GetName(); + Version version = assemblyName.Version; + DateTime buildDate = new DateTime(2000, 1, 1) + .AddDays(version.Build).AddSeconds(version.Revision * 2); + string displayableVersion = $"{version} ({buildDate})"; + return displayableVersion; + } + catch (Exception e) + { + return "???"; + } + } } } diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index 256aaef..35dcc33 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"C:\\Users\\Sascha Schiemann\\Downloads\\36E-12226L(1)\\36E-12226L.ts\"", + "commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\incoming\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/Scraper/CaSystemNames.cs b/skyscraper8/Skyscraper/Scraper/CaSystemNames.cs index f3e8b09..6be4a5e 100644 --- a/skyscraper8/Skyscraper/Scraper/CaSystemNames.cs +++ b/skyscraper8/Skyscraper/Scraper/CaSystemNames.cs @@ -32,7 +32,8 @@ namespace skyscraper5.Skyscraper.Scraper 0x18fe, 0x1813, 0x1817, 0x1818, 0x1819, 0x1863, 0x1801, 0x1810, 0x1702, 0x1722, 0x1762, 0x1834, - 0x1879 + 0x1879, + 0x185f }; diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs index 4c44445..4d48f1f 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs @@ -1460,7 +1460,11 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public void InsertSgtList(SgtList list) { - throw new NotImplementedException(); + string path = Path.Combine(rootDirectory.FullName, "Astra-SGT", list.ServiceListId.ToString() + ".json"); + FileInfo fi = new FileInfo(path); + fi.Directory.EnsureExists(); + string json = JsonConvert.SerializeObject(list, Formatting.Indented); + File.WriteAllText(fi.FullName, json); } public bool TestForSgtService(SgtService child) @@ -1610,7 +1614,8 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem } } - string newJson = JsonConvert.SerializeObject(incomingObject); + fileInfo.Directory.EnsureExists(); + string newJson = JsonConvert.SerializeObject(incomingObject); File.WriteAllText(path, newJson); return true; } @@ -1651,26 +1656,62 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem File.WriteAllText(path, json); } + private static string CleanFilename(string input) + { + char[] forbiddenOnes = Path.GetInvalidPathChars(); + char[] alsoForbbiden = Path.GetInvalidFileNameChars(); + char[] charArray = input.ToCharArray(); + for (int i = 0; i < charArray.Length; i++) + { + if (forbiddenOnes.Contains(charArray[i])) + { + charArray[i] = '_'; + } + if (alsoForbbiden.Contains(charArray[i])) + { + charArray[i] = '_'; + } + } + + return new string(charArray); + } public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession) { - throw new NotImplementedException(); + string jsonFileName = CleanFilename(multicastSession.serviceIdentifier) + ".json"; + string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_sessions", jsonFileName); + FileInfo fi = new FileInfo(path); + return fi.Exists; } public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession) { - throw new NotImplementedException(); + string jsonFileName = CleanFilename(multicastSession.serviceIdentifier) + ".json"; + string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_sessions", jsonFileName); + FileInfo fi = new FileInfo(path); + fi.Directory.EnsureExists(); + string json = JsonConvert.SerializeObject(multicastSession, Formatting.Indented); + File.WriteAllText(fi.FullName, json); } public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastEndpointAddressType multicastEndpointAddressType) { - throw new NotImplementedException(); + string jsonFileName = String.Format("{0}_{1}_{2}_{3}_{4}.json", carrier.NipNetworkId, carrier.NipCarrierId, carrier.NipLinkId, carrier.NipServiceId, carrier.NipStreamProviderName); + string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_gateway_configuration_transport_sessions", jsonFileName); + FileInfo fi = new FileInfo(path); + return fi.Exists; } public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastEndpointAddressType multicastGatewayConfigurationTransportSession) { - throw new NotImplementedException(); + string jsonFileName = String.Format("{0}_{1}_{2}_{3}_{4}.json", carrier.NipNetworkId, carrier.NipCarrierId, carrier.NipLinkId, carrier.NipServiceId, carrier.NipStreamProviderName); + string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_gateway_configuration_transport_sessions", jsonFileName); + FileInfo fi = new FileInfo(path); + fi.Directory.EnsureExists(); + string json = + JsonConvert.SerializeObject(multicastGatewayConfigurationTransportSession, Formatting.Indented); + File.WriteAllText(fi.FullName, json); } public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation) @@ -1690,12 +1731,21 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash) { - throw new NotImplementedException(); + string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag"); + FileInfo fi = new FileInfo(path); + if (!fi.Exists) + { + return DateTime.MinValue; + } + return fi.LastWriteTime; } public void InsertDvbiServiceListEntryPoint(long sourceHash) { - throw new NotImplementedException(); + string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag"); + FileInfo fi = new FileInfo(path); + fi.Directory.EnsureExists(); + File.WriteAllText(fi.FullName, ""); } public bool TestForServiceListEntryPoints(long sourceHash) @@ -1710,7 +1760,12 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public void InsertDvbiServiceList(DvbiServiceList serviceList) { - throw new NotImplementedException(); + string jsonFileName = CleanFilename(serviceList.Name) + ".json"; + string path = Path.Combine(rootDirectory.FullName, "DVB-I", "services", jsonFileName); + FileInfo fi = new FileInfo(path); + fi.Directory.EnsureExists(); + string json = JsonConvert.SerializeObject(serviceList, Formatting.Indented); + File.WriteAllText(fi.FullName, json); } public bool TestForDvbiService(string id) @@ -1720,7 +1775,9 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public bool TestForDvbiServiceListEntryPoints(long sourceHash) { - throw new NotImplementedException(); + string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag"); + FileInfo fi = new FileInfo(path); + return fi.Exists; } public void UpdateDvbiServiceListLastCheckedDate(string id, DateTime currentTime) @@ -1760,7 +1817,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public bool TestForDvbiServiceList(string id) { - throw new NotImplementedException(); + string jsonFileName = CleanFilename(id) + ".json"; + string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_lists", id); + FileInfo fi = new FileInfo(path); + return fi.Exists; } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs index 00136bf..958f3cd 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs @@ -861,22 +861,44 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory //Since we're working solely in memory, we won't need to remember this. } + class T2MiTransmitterData + { + public ushort TimeOffset { get; set; } + } + + private Dictionary,T2MiTransmitterData> t2miTransmitters; public bool T2MiTestForTransmitter(int? currentNetworkId, int? currentTransportStreamId, int relatedPid, ushort txIdentifier) { - throw new NotImplementedException(); + if (t2miTransmitters == null) + return false; + + Tuple t2miTransmitterId = new Tuple(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier); + return t2miTransmitters.ContainsKey(t2miTransmitterId); } public void T2MiRememberTransmitter(int? currentNetworkId, int? currentTransportStreamId, int relatedPid, ushort txIdentifier) { - throw new NotImplementedException(); + Tuple t2miTransmitterId = + new Tuple(currentNetworkId, currentTransportStreamId, relatedPid, + txIdentifier); + + if (t2miTransmitters == null) + t2miTransmitters = new Dictionary, T2MiTransmitterData>(); + + if (t2miTransmitters.ContainsKey(t2miTransmitterId)) + return; + + t2miTransmitters.Add(t2miTransmitterId, new T2MiTransmitterData()); } public void T2MiSetTransmitterTimeOffset(int? currentNetworkId, int? currentTransportStreamId, int relatedPid, ushort txIdentifier, ushort timeOffset) { - throw new NotImplementedException(); + T2MiRememberTransmitter(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier); + Tuple t2miTransmitterId = new Tuple(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier); + t2miTransmitters[t2miTransmitterId].TimeOffset = timeOffset; } private List _lnbTypes; diff --git a/skyscraper8/skyscraper8.csproj b/skyscraper8/skyscraper8.csproj index 8b531c1..7e848a4 100644 --- a/skyscraper8/skyscraper8.csproj +++ b/skyscraper8/skyscraper8.csproj @@ -6,6 +6,16 @@ enable enable Linux + 8.0.* + 5.6.7.8 + + + + False + + + + False