From 82a05769035e396a507c02eac5d9fe477e89305e Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Tue, 24 Jun 2025 22:05:44 +0200 Subject: [PATCH] Added DvbNipTestForCarrier, DvbNipInsertCarrier along with an implementation in the PostgreSQL driver. --- .../MinioObjectStorage.cs | 15 +++ .../skyscraper5.Data.PostgreSql/DvbNip.cs | 120 ++++++++++++++++++ .../skyscraper5.Data.PostgreSql/Dvbi.cs | 2 + .../PostgresqlDataStore.cs | 7 +- skyscraper8/DvbNip/DvbNipUtilities.cs | 2 +- skyscraper8/Ietf/FLUTE/FluteListener.cs | 2 +- skyscraper8/Properties/launchSettings.json | 1 - .../Skyscraper/Scraper/SkyscraperContext.cs | 4 + .../Filesystem/FilesystemScraperStorage.cs | 10 ++ .../Scraper/Storage/IScraperStorage.cs | 2 + .../InMemory/InMemoryScraperStorage.cs | 10 ++ .../Scraper/Storage/Split/DataStorage.cs | 34 +++-- .../Scraper/Storage/Split/ObjectStorage.cs | 9 -- .../Storage/Split/SplitScraperStorage.cs | 30 +++-- 14 files changed, 213 insertions(+), 35 deletions(-) create mode 100644 DataTableStorages/skyscraper5.Data.PostgreSql/DvbNip.cs diff --git a/BlobStorages/skyscraper5.Data.Minio/MinioObjectStorage.cs b/BlobStorages/skyscraper5.Data.Minio/MinioObjectStorage.cs index fd80c26..f16530b 100644 --- a/BlobStorages/skyscraper5.Data.Minio/MinioObjectStorage.cs +++ b/BlobStorages/skyscraper5.Data.Minio/MinioObjectStorage.cs @@ -9,6 +9,8 @@ using Minio.DataModel.Args; using Minio.Exceptions; using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs; using skyscraper5.Skyscraper.Scraper.Storage.Split; +using skyscraper8.DvbNip; +using skyscraper8.Ietf.FLUTE; namespace skyscraper5.Data { @@ -239,5 +241,18 @@ namespace skyscraper5.Data GetVersioningArgs args = new GetVersioningArgs().WithBucket(_minioBucket); VersioningConfiguration vc = _minioClient.GetVersioningAsync(args).Result; } + + public bool DvbNipTestForFile(string announcedFileContentLocation) + { + string path = "/nip/" + announcedFileContentLocation; + return FileExists(path); + } + + public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener) + { + string path = "/nip/" + DvbNipUtilities.MakeFilename(listener.FileAssociation.ContentLocation); + Stream stream = listener.ToStream(); + WriteObject(path, stream); + } } } diff --git a/DataTableStorages/skyscraper5.Data.PostgreSql/DvbNip.cs b/DataTableStorages/skyscraper5.Data.PostgreSql/DvbNip.cs new file mode 100644 index 0000000..ce4a338 --- /dev/null +++ b/DataTableStorages/skyscraper5.Data.PostgreSql/DvbNip.cs @@ -0,0 +1,120 @@ +using skyscraper8.DvbNip; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Npgsql; +using NpgsqlTypes; + +namespace skyscraper5.Data.PostgreSql +{ + public partial class PostgresqlDataStore + { + private Dictionary _knownNipCarriers; + + public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation) + { + if (_knownNipCarriers != null) + { + if (_knownNipCarriers.ContainsKey(currentCarrierInformation)) + return true; + } + + bool result = false; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + connection.Open(); + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT serial FROM dvbnip_carrier_information WHERE nid = @nid AND cid = @cid AND lid = @lid AND sid = @sid"; + command.Parameters.AddWithValue("@nid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipNetworkId); + command.Parameters.AddWithValue("@cid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipCarrierId); + command.Parameters.AddWithValue("@lid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipLinkId); + command.Parameters.AddWithValue("@sid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipServiceId); + NpgsqlDataReader dataReader = command.ExecuteReader(); + if (dataReader.Read()) + { + if (_knownNipCarriers == null) + _knownNipCarriers = new Dictionary(); + + int value = dataReader.GetInt32(0); + _knownNipCarriers.Add(currentCarrierInformation, value); + result = true; + } + dataReader.Close(); + } + + return result; + } + + public void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation) + { + EnqueueTask(x => DvbNipInsertCarrierEx(x, currentCarrierInformation)); + EnqueueTask(CommitTransaction); + } + + private void DvbNipInsertCarrierEx(NpgsqlConnection connection, NipActualCarrierInformation currentCarrierInformation) + { + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "INSERT INTO dvbnip_carrier_information VALUES (DEFAULT,DEFAULT,@nid,@cid,@lid,@sid,@spn)"; + command.Parameters.AddWithValue("@nid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipNetworkId); + command.Parameters.AddWithValue("@cid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipCarrierId); + command.Parameters.AddWithValue("@lid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipLinkId); + command.Parameters.AddWithValue("@sid", NpgsqlDbType.Integer, (int)currentCarrierInformation.NipServiceId); + command.Parameters.AddWithValue("@spn", NpgsqlDbType.Text, currentCarrierInformation.NipStreamProviderName); + int a = command.ExecuteNonQuery(); + if (a != 1) + throw new DataException(String.Format("Wanted to insert 1 row, got {0}", a)); + command.Dispose(); + } + + public bool DvbNipPrivateDataSpecifier(NipActualCarrierInformation currentCarrierInformation, DateTime versionUpdate, + uint privateDataSpecifier, List privateDataSessions) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForNetwork(BroadcastNetworkType network) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertNetwork(BroadcastNetworkType network) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForService(BroadcastMediaStreamType broadcastMediaStreamType) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertService(BroadcastMediaStreamType broadcastMediaStreamType) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, + MulticastEndpointAddressType multicastEndpointAddressType) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, + MulticastGatewayConfigurationTransportSessionType multicastGatewayConfigurationTransportSession) + { + throw new NotImplementedException(); + } + } +} diff --git a/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs b/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs index 119d12c..ad28f42 100644 --- a/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs +++ b/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs @@ -8,6 +8,7 @@ using System.Data; using System.Linq; using System.Text; using System.Threading.Tasks; +using skyscraper8.DvbNip; namespace skyscraper5.Data.PostgreSql { @@ -305,6 +306,7 @@ namespace skyscraper5.Data.PostgreSql return result; } + private Dictionary _knownDvbiServiceListUpdateDates; public DateTime GetDvbiServiceListLastUpdateDate(string id) { diff --git a/DataTableStorages/skyscraper5.Data.PostgreSql/PostgresqlDataStore.cs b/DataTableStorages/skyscraper5.Data.PostgreSql/PostgresqlDataStore.cs index 59a5d5e..3318786 100644 --- a/DataTableStorages/skyscraper5.Data.PostgreSql/PostgresqlDataStore.cs +++ b/DataTableStorages/skyscraper5.Data.PostgreSql/PostgresqlDataStore.cs @@ -1,14 +1,13 @@ using System; +using System.Collections.Generic; using System.Data; using Npgsql; using NpgsqlTypes; using skyscraper5.Dvb.Psi.Model; using skyscraper5.Dvb.TvAnytime; -using skyscraper5.Rds.Messages; using skyscraper5.Skyscraper.Headless; using skyscraper5.Skyscraper.Scraper.Storage.Split; -using skyscraper8.DvbI; -using skyscraper8.Ses; +using skyscraper8.DvbNip; namespace skyscraper5.Data.PostgreSql { @@ -164,7 +163,7 @@ namespace skyscraper5.Data.PostgreSql return new object[] { connectionStringBuilder }; } - + } } diff --git a/skyscraper8/DvbNip/DvbNipUtilities.cs b/skyscraper8/DvbNip/DvbNipUtilities.cs index 2296c0a..795cabe 100644 --- a/skyscraper8/DvbNip/DvbNipUtilities.cs +++ b/skyscraper8/DvbNip/DvbNipUtilities.cs @@ -8,7 +8,7 @@ using skyscraper8.Ietf.FLUTE; namespace skyscraper8.DvbNip { - internal class DvbNipUtilities + public class DvbNipUtilities { private DvbNipUtilities() { } diff --git a/skyscraper8/Ietf/FLUTE/FluteListener.cs b/skyscraper8/Ietf/FLUTE/FluteListener.cs index 26137b4..142b1e7 100644 --- a/skyscraper8/Ietf/FLUTE/FluteListener.cs +++ b/skyscraper8/Ietf/FLUTE/FluteListener.cs @@ -145,7 +145,7 @@ namespace skyscraper8.Ietf.FLUTE } } - internal Stream ToStream() + public Stream ToStream() { if (!IsComplete()) throw new InvalidOperationException(); diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index b746a2f..21e5d76 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,6 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"C:\\Temp\\Hotbird130_12380_V_NIP.ts\"\r\n", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 4149d25..b2ef1b8 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -2500,6 +2500,10 @@ namespace skyscraper5.Skyscraper.Scraper public void OnNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation) { LogEvent(SkyscraperContextEvent.NipCarrierDetected, String.Format("{0}", currentCarrierInformation.NipStreamProviderName)); + if (!ScraperStorage.DvbNipTestForCarrier(currentCarrierInformation)) + { + ScraperStorage.DvbNipInsertCarrier(currentCarrierInformation); + } } public bool IsFileNeeded(string announcedFileContentLocation) diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs index 7bfadc5..586e4ea 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs @@ -1501,6 +1501,16 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem throw new NotImplementedException(); } + public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash) { throw new NotImplementedException(); diff --git a/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs index 0404d72..a5c6901 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs @@ -194,5 +194,7 @@ namespace skyscraper5.Skyscraper.Scraper.Storage void DvbNipInsertMulticastSession(MulticastSessionType multicastSession); bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastEndpointAddressType multicastEndpointAddressType); void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastGatewayConfigurationTransportSessionType multicastGatewayConfigurationTransportSession); + bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation); + void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation); } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs index 52647d7..dbddc4d 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs @@ -1565,6 +1565,16 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory _nipMulticastGatewayConfigurationTransportSessions.Add(key, multicastGatewayConfigurationTransportSession); } + public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + public void InsertDvbiServiceListEntryPoint(long sourceHash) { if (_dvbiServiceListEntryPointsCoordinates == null) diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs index e250dcd..fa8f620 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs @@ -1,12 +1,4 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.NetworkInformation; -using System.Text; -using System.Threading.Tasks; -using skyscraper5.Docsis.MacManagement; +using skyscraper5.Docsis.MacManagement; using skyscraper5.DsmCc.Descriptors; using skyscraper5.Dvb.DataBroadcasting.IntModel; using skyscraper5.Dvb.Descriptors; @@ -32,7 +24,16 @@ using skyscraper5.src.Skyscraper.FrequencyListGenerator; using skyscraper5.src.Skyscraper.Scraper.Dns; using skyscraper5.Teletext; using skyscraper8.DvbI; +using skyscraper8.DvbNip; using skyscraper8.Ses; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.NetworkInformation; +using System.Text; +using System.Threading.Tasks; using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform; namespace skyscraper5.Skyscraper.Scraper.Storage.Split @@ -181,5 +182,18 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Split void UpdateDvbiServiceListEntryPointUpdateDate(long hash, DateTime currentTime); void InsertDvbiService(DvbIService service); bool TestForDvbiServiceList(string id); - } + + bool DvbNipPrivateDataSpecifier(NipActualCarrierInformation currentCarrierInformation, DateTime versionUpdate, uint privateDataSpecifier, List privateDataSessions); + bool DvbNipTestForNetwork(BroadcastNetworkType network); + void DvbNipInsertNetwork(BroadcastNetworkType network); + bool DvbNipTestForService(BroadcastMediaStreamType broadcastMediaStreamType); + void DvbNipInsertService(BroadcastMediaStreamType broadcastMediaStreamType); + bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession); + void DvbNipInsertMulticastSession(MulticastSessionType multicastSession); + bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastEndpointAddressType multicastEndpointAddressType); + void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastGatewayConfigurationTransportSessionType multicastGatewayConfigurationTransportSession); + + bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation); + void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation); + } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Split/ObjectStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Split/ObjectStorage.cs index fa58c6d..7c8433f 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Split/ObjectStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Split/ObjectStorage.cs @@ -25,14 +25,5 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Split void Ping(); bool DvbNipTestForFile(string announcedFileContentLocation); void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener); - bool DvbNipPrivateDataSpecifier(NipActualCarrierInformation currentCarrierInformation, DateTime versionUpdate, uint privateDataSpecifier, List privateDataSessions); - bool DvbNipTestForNetwork(BroadcastNetworkType network); - void DvbNipInsertNetwork(BroadcastNetworkType network); - bool DvbNipTestForService(BroadcastMediaStreamType broadcastMediaStreamType); - void DvbNipInsertService(BroadcastMediaStreamType broadcastMediaStreamType); - bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession); - void DvbNipInsertMulticastSession(MulticastSessionType multicastSession); - bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastEndpointAddressType multicastEndpointAddressType); - void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastGatewayConfigurationTransportSessionType multicastGatewayConfigurationTransportSession); } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs index e8a38df..3cf796d 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs @@ -906,56 +906,68 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Split [DebuggerStepThrough] public bool DvbNipPrivateDataSpecifier(NipActualCarrierInformation currentCarrierInformation, DateTime versionUpdate, uint privateDataSpecifier, List privateDataSessions) { - return objectStorage.DvbNipPrivateDataSpecifier(currentCarrierInformation, versionUpdate, privateDataSpecifier, privateDataSessions); + return dataStorage.DvbNipPrivateDataSpecifier(currentCarrierInformation, versionUpdate, privateDataSpecifier, privateDataSessions); } [DebuggerStepThrough] public bool DvbNipTestForNetwork(BroadcastNetworkType network) { - return objectStorage.DvbNipTestForNetwork(network); + return dataStorage.DvbNipTestForNetwork(network); } [DebuggerStepThrough] public void DvbNipInsertNetwork(BroadcastNetworkType network) { - objectStorage.DvbNipInsertNetwork(network); + dataStorage.DvbNipInsertNetwork(network); } [DebuggerStepThrough] public bool DvbNipTestForService(BroadcastMediaStreamType broadcastMediaStreamType) { - return objectStorage.DvbNipTestForService(broadcastMediaStreamType); + return dataStorage.DvbNipTestForService(broadcastMediaStreamType); } [DebuggerStepThrough] public void DvbNipInsertService(BroadcastMediaStreamType broadcastMediaStreamType) { - objectStorage.DvbNipInsertService(broadcastMediaStreamType); + dataStorage.DvbNipInsertService(broadcastMediaStreamType); } [DebuggerStepThrough] public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession) { - return objectStorage.DvbNipTestForMulticastSession(multicastSession); + return dataStorage.DvbNipTestForMulticastSession(multicastSession); } [DebuggerStepThrough] public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession) { - objectStorage.DvbNipInsertMulticastSession(multicastSession); + dataStorage.DvbNipInsertMulticastSession(multicastSession); } [DebuggerStepThrough] public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastEndpointAddressType multicastEndpointAddressType) { - return objectStorage.DvbNipTestForMulticastGatewayConfigurationTransportSession(carrier, multicastEndpointAddressType); + return dataStorage.DvbNipTestForMulticastGatewayConfigurationTransportSession(carrier, multicastEndpointAddressType); } [DebuggerStepThrough] public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, MulticastGatewayConfigurationTransportSessionType multicastGatewayConfigurationTransportSession) { - objectStorage.DvbNipInsertMulticastGatewayConfigurationTransportSession(carrier, multicastGatewayConfigurationTransportSession); + dataStorage.DvbNipInsertMulticastGatewayConfigurationTransportSession(carrier, multicastGatewayConfigurationTransportSession); + } + + [DebuggerStepThrough] + public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation) + { + return dataStorage.DvbNipTestForCarrier(currentCarrierInformation); + } + + [DebuggerStepThrough] + public void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation) + { + dataStorage.DvbNipInsertCarrier(currentCarrierInformation); } [DebuggerStepThrough]