Implemented some missing storage methods.

.gitignore

@@ -123,3 +123,4 @@ imgui.ini
 /GUIs/skyscraper8.UI.ImGui.MonoGame/obj/Debug/net8.0
 /GUIs/skyscraper8.UI.ImGui.MonoGame/obj
 /.vs/skyscraper8/CopilotIndices/17.14.995.13737
+/.vs/skyscraper8/config

GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs

@@ -1742,6 +1742,8 @@ namespace SDL2Demo.Jobs
 				jobContext.Renderables.Add(this);
 			});
 			foundFrequenciesWindow.allowZapNow = true;
+			foundFrequenciesWindow.statusPacketsInQueue = 0;
+			foundFrequenciesWindow.statusPacketsInTotal = 0;
 		}
 
 		public void OnBlindscanScrapeStopCondition()

skyscraper8/Program.cs

@@ -24,7 +24,9 @@ using System.Diagnostics;
 using System.Net;
 using System.Net.NetworkInformation;
 using System.Net.Sockets;
+using System.Reflection;
 using System.Runtime.InteropServices;
+using skyscraper5.Mpeg2.Descriptors;
 using skyscraper5.Skyscraper.Scraper.StreamAutodetection;
 using skyscraper8;
 using skyscraper8.SatIp;
@@ -37,6 +39,7 @@ namespace skyscraper5
     class Program
     {
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+		private const int PUBLIC_RELEASE = 4;
 		private static void IntegrationTest()
 		{
 			/*List<SsdpDevice> ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList();
@@ -44,8 +47,7 @@ namespace skyscraper5
 			{
 				Console.WriteLine("SSDP device: {0}", ssdpDevice.Server);
 			}*/
-
+            
-			Console.WriteLine("yeet!");
 			/*RtspClient rtspClient = new RtspClient("172.20.20.121", 554);
 			rtspClient.AutoReconnect = true;
 			RtspOptionsResponse options = rtspClient.GetOptions("/");
@@ -100,8 +102,8 @@ namespace skyscraper5
 		{
             IntegrationTest();
 
-			logger.Info("Hello!");
+            logger.InfoFormat(String.Format("Hello! This is skyscraper8, public release #{0}, code version {1}", PUBLIC_RELEASE, GetCurrentAssemblyDisplayVersion()));
-			ffmpegFrameGrabber.CanStart();
+
             logger.DebugFormat("Found {0}-bit Operating system.", Environment.Is64BitOperatingSystem ? 64 : 32);
             logger.DebugFormat("I'm a {0}-bit Process.", Environment.Is64BitProcess ? 64 : 32);
 
@@ -286,6 +288,7 @@ namespace skyscraper5
             Console.WriteLine("You're not supposed to run me directly. Run me from the commandline using the following:");
             Console.WriteLine("for example: .\\skyscraper8.exe cscan tcp://127.0.0.1:6969");
             Console.WriteLine("         or: .\\skyscraper8.exe \"C:\\path\\to\\file.ts\\");
+            Console.WriteLine("         or: .\\skyscraper8.exe \"C:\\path\\to\\my\\folder\\with\\ts\\files\\\" (for batch extraction)");
             Console.WriteLine();
             Console.WriteLine("Bonus feature:");
             Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC.");
@@ -295,30 +298,39 @@ namespace skyscraper5
         {
             DataStorage dataStorage = new InMemoryScraperStorage();
             FilesystemStorage filesystemStorage = new FilesystemStorage(new DirectoryInfo("."));
-
+            ProcessDirectory(di, dataStorage, filesystemStorage);
-            //DirectoryInfo di = new DirectoryInfo(@"E:\Skyscraper\Astra 19.2");
-            FileInfo[] fileInfos = di.GetFiles("*.ts");
-            foreach (FileInfo fileInfo in fileInfos)
-            {
-                Console.WriteLine(new string('_', Console.WindowWidth - 1));
-                Console.WriteLine("Processing: {0}", fileInfo.Name);
-                SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), dataStorage,filesystemStorage);
-                //StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest();
-                FileStream fileStream = fileInfo.OpenRead();
-
-                Stopwatch stopwatch = new Stopwatch();
-                stopwatch.Start();
-                skyscraper.IngestFromStream(fileStream);
-                //streamTypeAutodetectionTest.IngestFromStream(fileStream);
-                stopwatch.Stop();
-
-                Console.WriteLine("Time to process: " + stopwatch.Elapsed.TotalSeconds);
-
-                Console.WriteLine("File was: {0}", fileInfo.Name);
-                PrintPidStatistics(skyscraper);
-            }
         }
 
+        private static void ProcessDirectory(DirectoryInfo di, DataStorage dataStorage, ObjectStorage objectStorage)
+        {
+	        DirectoryInfo[] directoryInfos = di.GetDirectories();
+	        foreach (DirectoryInfo subdir in directoryInfos)
+	        {
+		        ProcessDirectory(subdir, dataStorage, objectStorage);
+	        }
+	        FileInfo[] fileInfos = di.GetFiles("*.ts");
+	        foreach (FileInfo fileInfo in fileInfos)
+	        {
+		        Console.WriteLine(new string('_', Console.WindowWidth - 1));
+		        Console.WriteLine("Processing: {0}", fileInfo.Name);
+		        SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
+		        skyscraper.InitalizeFilterChain();
+		        //StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest();
+		        FileStream fileStream = fileInfo.OpenRead();
+
+		        Stopwatch stopwatch = new Stopwatch();
+		        stopwatch.Start();
+		        skyscraper.IngestFromStream(fileStream);
+		        //streamTypeAutodetectionTest.IngestFromStream(fileStream);
+		        stopwatch.Stop();
+
+		        Console.WriteLine("Time to process: " + stopwatch.Elapsed.TotalSeconds);
+
+		        Console.WriteLine("File was: {0}", fileInfo.Name);
+		        PrintPidStatistics(skyscraper);
+	        }
+		}
+
         private static void PrintPidStatistics(SkyscraperContext skyscraper)
         {
             Console.WriteLine("Unattached PIDs:");
@@ -440,8 +452,6 @@ namespace skyscraper5
             skyscraper.Dispose();
 		}
 
-        
-
         private static void HandleCrazyScanToLiveSystem(string url, bool withTimeout = true)
         {
 	        StorageConnectionManager connectionManager = StorageConnectionManager.GetInstance();
@@ -798,5 +808,23 @@ namespace skyscraper5
             }
 
         }
+
+        private static string GetCurrentAssemblyDisplayVersion()
+        {
+	        try
+	        {
+		        Assembly executingAssembly = System.Reflection.Assembly.GetExecutingAssembly();
+		        AssemblyName assemblyName = executingAssembly.GetName();
+		        Version version = assemblyName.Version;
+		        DateTime buildDate = new DateTime(2000, 1, 1)
+			        .AddDays(version.Build).AddSeconds(version.Revision * 2);
+		        string displayableVersion = $"{version} ({buildDate})";
+		        return displayableVersion;
+	        }
+	        catch (Exception e)
+	        {
+		        return "???";
+	        }
+        }
     }
 }

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "\"C:\\Users\\Sascha Schiemann\\Downloads\\36E-12226L(1)\\36E-12226L.ts\"",
+      "commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\incoming\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/Scraper/CaSystemNames.cs

@@ -32,7 +32,8 @@ namespace skyscraper5.Skyscraper.Scraper
             0x18fe,
 			0x1813, 0x1817, 0x1818, 0x1819, 0x1863, 0x1801, 0x1810, 0x1702, 0x1722, 0x1762,
 			0x1834,
-            0x1879
+            0x1879,
+            0x185f
 		};
 
 

skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs

@@ -1460,7 +1460,11 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 
 		public void InsertSgtList(SgtList list)
 		{
-			throw new NotImplementedException();
+			string path = Path.Combine(rootDirectory.FullName, "Astra-SGT", list.ServiceListId.ToString() + ".json");
+			FileInfo fi = new FileInfo(path);
+            fi.Directory.EnsureExists();
+            string json = JsonConvert.SerializeObject(list, Formatting.Indented);
+            File.WriteAllText(fi.FullName, json);
 		}
 
 		public bool TestForSgtService(SgtService child)
@@ -1610,7 +1614,8 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 				}
 			}
 
-			string newJson = JsonConvert.SerializeObject(incomingObject);
+            fileInfo.Directory.EnsureExists();
+            string newJson = JsonConvert.SerializeObject(incomingObject);
 			File.WriteAllText(path, newJson);
 			return true;
 		}
@@ -1651,26 +1656,62 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 			File.WriteAllText(path, json);
 		}
 
+		private static string CleanFilename(string input)
+		{
+			char[] forbiddenOnes = Path.GetInvalidPathChars();
+			char[] alsoForbbiden = Path.GetInvalidFileNameChars();
+			char[] charArray = input.ToCharArray();
+			for (int i = 0; i < charArray.Length; i++)
+			{
+				if (forbiddenOnes.Contains(charArray[i]))
+				{
+					charArray[i] = '_';
+				}
+				if (alsoForbbiden.Contains(charArray[i]))
+				{
+					charArray[i] = '_';
+				}
+			}
+
+			return new string(charArray);
+		}
 		public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession)
 		{
-			throw new NotImplementedException();
+			string jsonFileName = CleanFilename(multicastSession.serviceIdentifier) + ".json";
+			string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_sessions", jsonFileName);
+			FileInfo fi = new FileInfo(path);
+			return fi.Exists;
 		}
 
 		public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession)
 		{
-			throw new NotImplementedException();
+			string jsonFileName = CleanFilename(multicastSession.serviceIdentifier) + ".json";
+			string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_sessions", jsonFileName);
+			FileInfo fi = new FileInfo(path);
+            fi.Directory.EnsureExists();
+            string json = JsonConvert.SerializeObject(multicastSession, Formatting.Indented);
+            File.WriteAllText(fi.FullName, json);
 		}
 
 		public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier,
 			MulticastEndpointAddressType multicastEndpointAddressType)
 		{
-			throw new NotImplementedException();
+			string jsonFileName = String.Format("{0}_{1}_{2}_{3}_{4}.json", carrier.NipNetworkId, carrier.NipCarrierId, carrier.NipLinkId, carrier.NipServiceId, carrier.NipStreamProviderName);
+			string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_gateway_configuration_transport_sessions", jsonFileName);
+			FileInfo fi = new FileInfo(path);
+			return fi.Exists;
 		}
 
 		public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier,
 			MulticastEndpointAddressType multicastGatewayConfigurationTransportSession)
 		{
-			throw new NotImplementedException();
+			string jsonFileName = String.Format("{0}_{1}_{2}_{3}_{4}.json", carrier.NipNetworkId, carrier.NipCarrierId, carrier.NipLinkId, carrier.NipServiceId, carrier.NipStreamProviderName);
+			string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_gateway_configuration_transport_sessions", jsonFileName);
+			FileInfo fi = new FileInfo(path);
+            fi.Directory.EnsureExists();
+            string json =
+	            JsonConvert.SerializeObject(multicastGatewayConfigurationTransportSession, Formatting.Indented);
+            File.WriteAllText(fi.FullName, json);
 		}
 
 		public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation)
@@ -1690,12 +1731,21 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 
 		public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash)
 		{
-			throw new NotImplementedException();
+			string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag");
+			FileInfo fi = new FileInfo(path);
+			if (!fi.Exists)
+			{
+                return DateTime.MinValue;
+			}
+			return fi.LastWriteTime;
 		}
 
 		public void InsertDvbiServiceListEntryPoint(long sourceHash)
 		{
-			throw new NotImplementedException();
+			string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag");
+			FileInfo fi = new FileInfo(path);
+            fi.Directory.EnsureExists();
+			File.WriteAllText(fi.FullName, "");
 		}
 
 		public bool TestForServiceListEntryPoints(long sourceHash)
@@ -1710,7 +1760,12 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 
 		public void InsertDvbiServiceList(DvbiServiceList serviceList)
 		{
-			throw new NotImplementedException();
+			string jsonFileName = CleanFilename(serviceList.Name) + ".json";
+			string path = Path.Combine(rootDirectory.FullName, "DVB-I", "services", jsonFileName);
+			FileInfo fi = new FileInfo(path);
+			fi.Directory.EnsureExists();
+			string json = JsonConvert.SerializeObject(serviceList, Formatting.Indented);
+			File.WriteAllText(fi.FullName, json);
 		}
 
 		public bool TestForDvbiService(string id)
@@ -1720,7 +1775,9 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 
 		public bool TestForDvbiServiceListEntryPoints(long sourceHash)
 		{
-			throw new NotImplementedException();
+			string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag");
+			FileInfo fi = new FileInfo(path);
+			return fi.Exists;
 		}
 
 		public void UpdateDvbiServiceListLastCheckedDate(string id, DateTime currentTime)
@@ -1760,7 +1817,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 
 		public bool TestForDvbiServiceList(string id)
 		{
-			throw new NotImplementedException();
+			string jsonFileName = CleanFilename(id) + ".json";
+			string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_lists", id);
+			FileInfo fi = new FileInfo(path);
+			return fi.Exists;
 		}
 	}
 }

skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs

@@ -861,22 +861,44 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
 			//Since we're working solely in memory, we won't need to remember this.
 		}
 
+		class T2MiTransmitterData
+		{
+			public ushort TimeOffset { get; set; }
+		}
+
+		private Dictionary<Tuple<int?, int?, int, ushort>,T2MiTransmitterData> t2miTransmitters;
 		public bool T2MiTestForTransmitter(int? currentNetworkId, int? currentTransportStreamId, int relatedPid,
 			ushort txIdentifier)
 		{
-			throw new NotImplementedException();
+			if (t2miTransmitters == null)
+				return false;
+
+			Tuple<int?, int?, int, ushort> t2miTransmitterId = new Tuple<int?, int?, int, ushort>(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier);
+			return t2miTransmitters.ContainsKey(t2miTransmitterId);
 		}
 
 		public void T2MiRememberTransmitter(int? currentNetworkId, int? currentTransportStreamId, int relatedPid,
 			ushort txIdentifier)
 		{
-			throw new NotImplementedException();
+			Tuple<int?, int?, int, ushort> t2miTransmitterId =
+				new Tuple<int?, int?, int, ushort>(currentNetworkId, currentTransportStreamId, relatedPid,
+					txIdentifier);
+
+			if (t2miTransmitters == null)
+				t2miTransmitters = new Dictionary<Tuple<int?, int?, int, ushort>, T2MiTransmitterData>();
+
+			if (t2miTransmitters.ContainsKey(t2miTransmitterId))
+				return;
+
+			t2miTransmitters.Add(t2miTransmitterId, new T2MiTransmitterData());
 		}
 
 		public void T2MiSetTransmitterTimeOffset(int? currentNetworkId, int? currentTransportStreamId, int relatedPid,
 			ushort txIdentifier, ushort timeOffset)
 		{
-			throw new NotImplementedException();
+			T2MiRememberTransmitter(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier);
+			Tuple<int?, int?, int, ushort> t2miTransmitterId = new Tuple<int?, int?, int, ushort>(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier);
+			t2miTransmitters[t2miTransmitterId].TimeOffset = timeOffset;
 		}
 
 		private List<LnbType> _lnbTypes;

skyscraper8/skyscraper8.csproj

@@ -6,6 +6,16 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+    <AssemblyVersion>8.0.*</AssemblyVersion>
+    <FileVersion>5.6.7.8</FileVersion>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <Deterministic>False</Deterministic>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <Deterministic>False</Deterministic>
   </PropertyGroup>
 
   <ItemGroup>