From 3c1c28c4bed0279145f2eaeab3806806a9dbef12 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Mon, 9 Jun 2025 22:25:20 +0200 Subject: [PATCH] Added DVB-I Database operations to Postgresql. --- .../skyscraper5.Data.PostgreSql/Dvbi.cs | 296 +++++++++++++++++- .../skyscraper5.Data.PostgreSql/Sgt.cs | 30 +- .../Scraper/Storage/Split/DataStorage.cs | 1 + .../Storage/Split/SplitScraperStorage.cs | 4 +- 4 files changed, 313 insertions(+), 18 deletions(-) diff --git a/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs b/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs index 9d0181f..119d12c 100644 --- a/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs +++ b/DataTableStorages/skyscraper5.Data.PostgreSql/Dvbi.cs @@ -1,7 +1,10 @@ -using skyscraper5.Skyscraper.Scraper.Storage.Split; +using Npgsql; +using NpgsqlTypes; +using skyscraper5.Skyscraper.Scraper.Storage.Split; using skyscraper8.DvbI; using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -13,7 +16,18 @@ namespace skyscraper5.Data.PostgreSql public void InsertDvbiServiceListEntryPoint(long sourceHash) { - throw new NotImplementedException(); + EnqueueTask(x => InsertDvbiServiceListEntryPoint(x, sourceHash)); + _knownDvbiServiceListEntryPoints.Add(sourceHash); + } + + private void InsertDvbiServiceListEntryPoint(NpgsqlConnection x, long sourceHash) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "INSERT INTO dvbi_service_list_entry_points VALUES (@shash,DEFAULT,NULL)"; + command.Parameters.AddWithValue("@shash", NpgsqlTypes.NpgsqlDbType.Bigint, sourceHash); + int result = command.ExecuteNonQuery(); + if (result == -1) + throw new NotImplementedException(result.ToString()); } public bool TestForServiceListEntryPoints(long sourceHash) @@ -23,47 +37,180 @@ namespace skyscraper5.Data.PostgreSql public void InsertDvbiServiceList(DvbiServiceList serviceList) { - throw new NotImplementedException(); + EnqueueTask(x => InsertDvbiServiceListEx(x, serviceList)); + _knownDvbiServiceLists.Add(serviceList.Id); } + private void InsertDvbiServiceListEx(NpgsqlConnection x, DvbiServiceList serviceList) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "INSERT INTO dvbi_service_list VALUES (@id,DEFAULT,@name,@dadelivery,@dvdelivery,@rmaterial,@uri) ON CONFLICT (id) DO NOTHING;"; + command.Parameters.AddWithValue("@id", NpgsqlDbType.Text, serviceList.Id); + command.Parameters.AddWithValue("@name", NpgsqlDbType.Text, serviceList.Name); + command.Parameters.AddWithValue("@dadelivery", NpgsqlDbType.Text, serviceList.DashDelivery); + command.Parameters.AddWithValue("@dvdelivery", NpgsqlDbType.Text, serviceList.DvbSDelivery); + command.Parameters.AddWithValue("@rmaterial", NpgsqlDbType.Text, serviceList.RelatedMaterial); + command.Parameters.AddWithValue("@uri", NpgsqlDbType.Text, serviceList.URI); + SetNulls(command); + command.ExecuteNonQuery(); + } + + private HashSet _knownDvbiServices; public bool TestForDvbiService(string id) { - throw new NotImplementedException(); + if (_knownDvbiServices == null) + _knownDvbiServices = new HashSet(); + + bool result; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + connection.Open(); + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT dateadded FROM dvbi_services WHERE id = @id"; + command.Parameters.AddWithValue("@id", NpgsqlDbType.Text, id); + NpgsqlDataReader dataReader = command.ExecuteReader(); + if (result = dataReader.Read()) + _knownDvbiServices.Add(id); + dataReader.Close(); + command.Dispose(); + connection.Close(); + } + return result; } + private HashSet _knownDvbiServiceListEntryPoints; public bool TestForDvbiServiceListEntryPoints(long sourceHash) { - throw new NotImplementedException(); + if (_knownDvbiServiceListEntryPoints == null) + _knownDvbiServiceListEntryPoints = new HashSet(); + + bool result; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + connection.Open(); + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT dateadded FROM dvbi_service_list_entry_points WHERE sourcehash = @shash"; + command.Parameters.AddWithValue("@shash", NpgsqlTypes.NpgsqlDbType.Bigint, sourceHash); + NpgsqlDataReader dataReader = command.ExecuteReader(); + if (result = dataReader.Read()) + _knownDvbiServiceListEntryPoints.Add(sourceHash); + dataReader.Close(); + command.Dispose(); + connection.Close(); + } + return result; } public void UpdateDvbiServiceListLastCheckedDate(string id, DateTime currentTime) { - throw new NotImplementedException(); + EnqueueTask(x => UpdateDvbiServiceListLastCheckedDateEx(x, id, currentTime)); + _knownDvbiServiceListUpdateDates[id] = currentTime; + } + + private void UpdateDvbiServiceListLastCheckedDateEx(NpgsqlConnection x, string id, DateTime currentTime) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "UPDATE dvbi_service_list SET last_updated = @lupdated WHERE id = @id"; + command.Parameters.AddWithValue("@lupdated", NpgsqlDbType.Timestamp, currentTime); + command.Parameters.AddWithValue("@id", NpgsqlDbType.Text, id); + command.ExecuteNonQuery(); } public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash) { - throw new NotImplementedException(); - } + if (_knownDvbiServiceListEntryPointUpdateDates == null) + _knownDvbiServiceListEntryPointUpdateDates = new Dictionary(); - public DateTime GetDvbiServiceListLastUpdateDate(string id) - { - throw new NotImplementedException(); + if (_knownDvbiServiceListEntryPointUpdateDates.ContainsKey(sourceHash)) + return _knownDvbiServiceListEntryPointUpdateDates[sourceHash]; + + DateTime result; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + connection.Open(); + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT dateupdated FROM dvbi_service_list_entry_points WHERE sourcehash = @shash"; + command.Parameters.AddWithValue("@shash", NpgsqlDbType.Bigint, sourceHash); + NpgsqlDataReader dataReader = command.ExecuteReader(); + if (dataReader.Read()) + { + if (dataReader.IsDBNull(0)) + { + result = DateTime.MinValue; + } + else + { + result = dataReader.GetDateTime(0); + } + } + else + { + result = DateTime.MinValue; + } + dataReader.Close(); + command.Dispose(); + connection.Close(); + } + _knownDvbiServiceListEntryPointUpdateDates[sourceHash] = result; + return result; } public void AddDvbiServiceListToServiceListEntryPoint(DvbiServiceList serviceList, long sourceHash) { - throw new NotImplementedException(); + EnqueueTask(x => AddDvbiServiceListToServiceListEntryPointEx(x, serviceList, sourceHash)); + } + + private void AddDvbiServiceListToServiceListEntryPointEx(NpgsqlConnection x, DvbiServiceList serviceList, long sourceHash) + { + if (!TestForDvbiServiceList(serviceList.Id)) + { + return; + } + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "INSERT INTO dvbi_service_lists_to_entry_points VALUES (@slist,@epoint,DEFAULT)"; + command.Parameters.AddWithValue("@slist", NpgsqlDbType.Text, serviceList.Id); + command.Parameters.AddWithValue("@epoint", NpgsqlDbType.Bigint, sourceHash); + command.ExecuteNonQuery(); } public void AddDvbiServiceToServiceList(string id, string serviceListId) { - throw new NotImplementedException(); + EnqueueTask(x => AddDvbiServiceToServiceList(x, id, serviceListId)); } + private void AddDvbiServiceToServiceList(NpgsqlConnection x, string id, string serviceListId) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "INSERT INTO dvbi_services_to_service_lists VALUES (@service,@servicelist,DEFAULT)"; + command.Parameters.AddWithValue("@service", NpgsqlDbType.Text, id); + command.Parameters.AddWithValue("@servicelist", NpgsqlDbType.Text, serviceListId); + command.ExecuteNonQuery(); + } + + private Dictionary _knownDvbiServiceVersions; public int GetDvbiServiceVersion(string id) { - throw new NotImplementedException(); + if (_knownDvbiServiceVersions == null) + _knownDvbiServiceVersions = new Dictionary(); + + if (_knownDvbiServiceVersions.ContainsKey(id)) + return _knownDvbiServiceVersions[id]; + + int version; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT version FROM dvbi_services WHERE id = @id"; + command.Parameters.AddWithValue("@id", NpgsqlDbType.Text, id); + NpgsqlDataReader dataReader = command.ExecuteReader(); + dataReader.Read(); + version = dataReader.GetInt32(0); + dataReader.Close(); + command.Dispose(); + connection.Close(); + } + _knownDvbiServiceVersions[id] = version; + return version; } public void UpdateDvbiService(DvbIService service) @@ -71,14 +218,131 @@ namespace skyscraper5.Data.PostgreSql throw new NotImplementedException(); } + private Dictionary _knownDvbiServiceListEntryPointUpdateDates; public void UpdateDvbiServiceListEntryPointUpdateDate(long hash, DateTime currentTime) { - throw new NotImplementedException(); + EnqueueTask(x => UpdateDvbiServiceListEntryPointUpdateDateEx(x, hash, currentTime)); + if (_knownDvbiServiceListEntryPointUpdateDates == null) + _knownDvbiServiceListEntryPointUpdateDates = new Dictionary(); + + _knownDvbiServiceListEntryPointUpdateDates[hash] = currentTime; + } + + private void UpdateDvbiServiceListEntryPointUpdateDateEx(NpgsqlConnection x, long hash, DateTime currentTime) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "UPDATE dvbi_service_list_entry_points SET dateupdated = @dupdated WHERE sourcehash = @shash"; + command.Parameters.AddWithValue("@dupdated", NpgsqlDbType.Timestamp, currentTime); + command.Parameters.AddWithValue("@shash",NpgsqlDbType.Bigint, hash); + command.ExecuteNonQuery(); } public void InsertDvbiService(DvbIService service) { - throw new NotImplementedException(); + EnqueueTask(x => InsertDvbiServiceEx(x, service)); + _knownDvbiServices.Add(service.Id); + + if (_knownDvbiServiceVersions == null) + _knownDvbiServiceVersions = new Dictionary(); + _knownDvbiServiceVersions[service.Id] = service.Version; + } + + private void InsertDvbiServiceEx(NpgsqlConnection x, DvbIService service) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "insert into dvbi_services VALUES " + + "(@id, DEFAULT, @provider_name, @logo_url, @onid, @sid, @tsid, @fec, @frequency, @modcod, @modulation_system, @modulation_type, @orbital_position, @polarization, @roll_off, " + + "@symbol_rate, @sat_ip_query_parameters, @service_name, @service_type, @version, @related_item, @country, @ranking, @region, @service_description) " + + "ON CONFLICT (id) DO NOTHING;"; + command.Parameters.AddWithValue("@id", NpgsqlDbType.Text, service.Id); + command.Parameters.AddWithValue("@provider_name", NpgsqlDbType.Text, service.ProviderName); + command.Parameters.AddWithValue("@logo_url", NpgsqlDbType.Text, service.LogoURL); + command.Parameters.AddWithValue("@onid", NpgsqlDbType.Integer, (int)service.OriginalNetworkId); + command.Parameters.AddWithValue("@sid", NpgsqlDbType.Integer, (int)service.ServiceId); + command.Parameters.AddWithValue("@tsid", NpgsqlDbType.Integer, (int)service.TransportStreamId); + command.Parameters.AddWithValue("@fec", NpgsqlDbType.Integer, (int)service.FEC); + command.Parameters.AddWithValue("@frequency", NpgsqlDbType.Text, service.Frequency); + command.Parameters.AddWithValue("@modcod", NpgsqlDbType.Integer, (int)service.Modcod); + command.Parameters.AddWithValue("@modulation_system", NpgsqlDbType.Integer, (int)service.ModulationSystem); + command.Parameters.AddWithValue("@modulation_type", NpgsqlDbType.Integer, (int)service.ModulationType); + command.Parameters.AddWithValue("@orbital_position", NpgsqlDbType.Double, service.OrbitalPosition); + command.Parameters.AddWithValue("@polarization", NpgsqlDbType.Integer, (int)service.Polarization); + command.Parameters.AddWithValue("@roll_off", NpgsqlDbType.Integer, (int)service.RollOff); + command.Parameters.AddWithValue("@symbol_rate", NpgsqlDbType.Text, service.SymbolRate); + command.Parameters.AddWithValue("@sat_ip_query_parameters", NpgsqlDbType.Text, service.SatIpQueryParameters); + command.Parameters.AddWithValue("@service_name", NpgsqlDbType.Text, service.ServiceName); + command.Parameters.AddWithValue("@service_type", NpgsqlDbType.Text, service.ServiceType); + command.Parameters.AddWithValue("@version", NpgsqlDbType.Integer, service.Version); + command.Parameters.AddWithValue("@related_item", NpgsqlDbType.Text, service.RelatedItem); + command.Parameters.AddWithValue("@country", NpgsqlDbType.Text, service.Country); + command.Parameters.AddWithValue("@ranking", NpgsqlDbType.Integer, service.Ranking); + command.Parameters.AddWithValue("@region", NpgsqlDbType.Text, service.Region); + command.Parameters.AddWithValue("@service_description", NpgsqlDbType.Text, service.ServiceDescription); + SetNulls(command); + command.ExecuteNonQuery(); + } + + private HashSet _knownDvbiServiceLists; + public bool TestForDvbiServiceList(string id) + { + if (_knownDvbiServiceLists == null) + _knownDvbiServiceLists = new HashSet(); + + bool result; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + connection.Open(); + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT dateadded FROM dvbi_service_list WHERE id = @id"; + command.Parameters.AddWithValue("@id", NpgsqlTypes.NpgsqlDbType.Text, id); + NpgsqlDataReader dataReader = command.ExecuteReader(); + if (result = dataReader.Read()) + _knownDvbiServiceLists.Add(id); + dataReader.Close(); + command.Dispose(); + connection.Close(); + } + return result; + } + + private Dictionary _knownDvbiServiceListUpdateDates; + public DateTime GetDvbiServiceListLastUpdateDate(string id) + { + if (_knownDvbiServiceListUpdateDates == null) + _knownDvbiServiceListUpdateDates = new Dictionary(); + + if (_knownDvbiServiceListUpdateDates.ContainsKey(id)) + return _knownDvbiServiceListUpdateDates[id]; + + DateTime result; + using (NpgsqlConnection connection = new NpgsqlConnection(connectionStringBuilder.ToString())) + { + connection.Open(); + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT last_updated FROM dvbi_service_list WHERE id = @id"; + command.Parameters.AddWithValue("@id", NpgsqlTypes.NpgsqlDbType.Text, id); + NpgsqlDataReader dataReader = command.ExecuteReader(); + if (dataReader.Read()) + { + if (dataReader.IsDBNull(0)) + { + result = DateTime.MinValue; + } + else + { + result = dataReader.GetDateTime(0); + } + } + else + { + result = DateTime.MinValue; + } + dataReader.Close(); + command.Dispose(); + connection.Close(); + } + _knownDvbiServiceListUpdateDates[id] = result; + return result; } } } diff --git a/DataTableStorages/skyscraper5.Data.PostgreSql/Sgt.cs b/DataTableStorages/skyscraper5.Data.PostgreSql/Sgt.cs index 0a63ece..0d1fe92 100644 --- a/DataTableStorages/skyscraper5.Data.PostgreSql/Sgt.cs +++ b/DataTableStorages/skyscraper5.Data.PostgreSql/Sgt.cs @@ -100,7 +100,35 @@ namespace skyscraper5.Data.PostgreSql public void InsertSgtService(SgtService child) { - throw new NotImplementedException(); + DatabaseKeySgtService key = child.GetDatabaseKey(); + if (_knownSgtServices.Contains(key)) + return; + + EnqueueTask(x => InsertSgtServiceEx(x, child)); + _knownSgtServices.Add(key); + } + + private void InsertSgtServiceEx(NpgsqlConnection x, SgtService child) + { + NpgsqlCommand command = x.CreateCommand(); + command.CommandText = "INSERT INTO astra_sgt_services " + + "VALUES (@slid,@sid,@tsid,@onid,DEFAULT,@lcn,@visible,@newService,@genrecode,@csids,@sname,@spname,@stype,@vsids,@blist)"; + command.Parameters.AddWithValue("@slid", NpgsqlDbType.Integer, (int)child.ServiceListId); + command.Parameters.AddWithValue("@sid", NpgsqlDbType.Integer, (int)child.ServiceId); + command.Parameters.AddWithValue("@tsid", NpgsqlDbType.Integer, (int)child.TransportStreamId); + command.Parameters.AddWithValue("@onid", NpgsqlDbType.Integer, (int)child.OriginalNetworkId); + command.Parameters.AddWithValue("@lcn", NpgsqlDbType.Integer, child.Lcn); + command.Parameters.AddWithValue("@visible", NpgsqlDbType.Boolean, child.VisibleServiceFlag); + command.Parameters.AddWithValue("@newService", NpgsqlDbType.Boolean, child.NewServiceFlag); + command.Parameters.AddWithValue("@genrecode", NpgsqlDbType.Integer, (int)child.GenreCode); + command.Parameters.AddWithValue("@csids",NpgsqlDbType.Json, JsonConvert.SerializeObject(child.CaSystemIds)); + command.Parameters.AddWithValue("@sname", NpgsqlDbType.Text, child.ServiceDescriptor?.ServiceName); + command.Parameters.AddWithValue("@spname", NpgsqlDbType.Text, child.ServiceDescriptor?.ServiceProviderName); + command.Parameters.AddWithValue("@stype", NpgsqlDbType.Integer, (int)child.ServiceDescriptor?.ServiceType); + command.Parameters.AddWithValue("@vsids", NpgsqlDbType.Json, JsonConvert.SerializeObject(child.VirtualServiceIds)); + command.Parameters.AddWithValue("@blist", NpgsqlDbType.Text, child.BouquetList); + SetNulls(command); + command.ExecuteNonQuery(); } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs index 40ad829..e250dcd 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Split/DataStorage.cs @@ -180,5 +180,6 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Split void UpdateDvbiService(DvbIService service); void UpdateDvbiServiceListEntryPointUpdateDate(long hash, DateTime currentTime); void InsertDvbiService(DvbIService service); + bool TestForDvbiServiceList(string id); } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs index e52ba8d..6b1a3ef 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs @@ -967,14 +967,16 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Split dataStorage.UpdateDvbiServiceListEntryPointUpdateDate(hash, currentTime); } + [DebuggerStepThrough] public void InsertDvbiService(DvbIService service) { dataStorage.InsertDvbiService(service); } + [DebuggerStepThrough] public bool TestForDvbiServiceList(string id) { - throw new NotImplementedException(); + return dataStorage.TestForDvbiServiceList(id); } } }