From 0453a7115a9b7adf3821614f305ce965d14bf568 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Thu, 17 Jul 2025 22:01:57 +0200 Subject: [PATCH] Fixed the Lyngsat Map Scraper to work with the new architecture. --- .../skyscraper5.Data.MySql/Ait.cs | 157 +++++++++++++++++- .../skyscraper5.LyngsatMapsScraper/Program.cs | 27 +-- .../AprsStorage/AprsPostgresqlStorage.cs | 7 +- MpePlugins/skyscraper5.Aprs/LX9SESPlugin.cs | 50 ++---- .../skyscraper5.Aprs/skyscraper5.Aprs.csproj | 1 + skyscraper8.Tests/DocsisTestsLong.cs | 9 +- skyscraper8.Tests/SanityTests.cs | 28 ++++ skyscraper8/Passing.cs | 4 +- 8 files changed, 227 insertions(+), 56 deletions(-) create mode 100644 skyscraper8.Tests/SanityTests.cs diff --git a/DataTableStorages/skyscraper5.Data.MySql/Ait.cs b/DataTableStorages/skyscraper5.Data.MySql/Ait.cs index 944a628..78bc5ef 100644 --- a/DataTableStorages/skyscraper5.Data.MySql/Ait.cs +++ b/DataTableStorages/skyscraper5.Data.MySql/Ait.cs @@ -8,6 +8,9 @@ using skyscraper5.Mhp.Descriptors; using skyscraper5.Mhp.Descriptors.InteractionTransportSelectors; using System; using skyscraper5.Teletext; +using skyscraper8.DvbI; +using skyscraper8.DvbNip; +using skyscraper8.Ses; using skyscraper8.Skyscraper.Scraper.Storage; namespace skyscraper5.Data.MySql @@ -275,5 +278,157 @@ namespace skyscraper5.Data.MySql } } - } + public bool TestForSgtList(SgtList list) + { + throw new NotImplementedException(); + } + + public void InsertSgtList(SgtList list) + { + throw new NotImplementedException(); + } + + public bool TestForSgtService(SgtService child) + { + throw new NotImplementedException(); + } + + public void InsertSgtService(SgtService child) + { + throw new NotImplementedException(); + } + + public void InsertDvbiService(DvbIService service) + { + throw new NotImplementedException(); + } + + public bool TestForDvbiServiceList(string id) + { + throw new NotImplementedException(); + } + + public bool DvbNipPrivateDataSpecifier(NipActualCarrierInformation currentCarrierInformation, DateTime versionUpdate, + uint privateDataSpecifier, List privateDataSessions) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForNetwork(BroadcastNetworkType network) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertNetwork(BroadcastNetworkType network) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForService(BroadcastMediaStreamType broadcastMediaStreamType) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertService(BroadcastMediaStreamType broadcastMediaStreamType) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, + MulticastEndpointAddressType multicastEndpointAddressType) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier, + MulticastEndpointAddressType multicastGatewayConfigurationTransportSession) + { + throw new NotImplementedException(); + } + + public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + + public void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + + public void InsertDvbiServiceListEntryPoint(long sourceHash) + { + throw new NotImplementedException(); + } + + public bool TestForDvbiService(string id) + { + throw new NotImplementedException(); + } + + public bool TestForDvbiServiceListEntryPoints(long sourceHash) + { + throw new NotImplementedException(); + } + + public void UpdateDvbiService(DvbIService service) + { + throw new NotImplementedException(); + } + + public void UpdateDvbiServiceListEntryPointUpdateDate(long hash, DateTime currentTime) + { + throw new NotImplementedException(); + } + + public void UpdateDvbiServiceListLastCheckedDate(string id, DateTime currentTime) + { + throw new NotImplementedException(); + } + + public int GetDvbiServiceVersion(string id) + { + throw new NotImplementedException(); + } + + public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash) + { + throw new NotImplementedException(); + } + + public void AddDvbiServiceToServiceList(string id, string serviceListId) + { + throw new NotImplementedException(); + } + + public DateTime GetDvbiServiceListLastUpdateDate(string id) + { + throw new NotImplementedException(); + } + + public void AddDvbiServiceListToServiceListEntryPoint(DvbiServiceList serviceList, long sourceHash) + { + throw new NotImplementedException(); + } + + public bool TestForServiceListEntryPoints(long sourceHash) + { + throw new NotImplementedException(); + } + + public void InsertDvbiServiceList(DvbiServiceList serviceList) + { + throw new NotImplementedException(); + } + } } diff --git a/Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs b/Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs index 8bf1241..00b2a4d 100644 --- a/Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs +++ b/Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs @@ -7,6 +7,8 @@ using SharpKml.Dom; using SharpKml.Engine; using skyscraper5.LyngsatMapsScraper.Model; using skyscraper5.Skyscraper.Scraper.Storage; +using skyscraper8.Skyscraper.Plugins; +using skyscraper8.Skyscraper.Scraper.Storage; namespace skyscraper5.LyngsatMapsScraper { @@ -21,6 +23,7 @@ namespace skyscraper5.LyngsatMapsScraper private bool gotHeaders; private const string rootUrl = "https://genmap.lyngsat.org/server2/kml"; private Random rng; + private void SafeThrottle() { if (rng == null) @@ -30,6 +33,7 @@ namespace skyscraper5.LyngsatMapsScraper private void Run() { + PluginLogger logger = PluginLogManager.GetLogger(typeof(Program)); WebClient webClient = new WebClient(); DirectoryInfo outputDirectoryInfo = new DirectoryInfo("temp"); if (!outputDirectoryInfo.Exists) @@ -37,17 +41,16 @@ namespace skyscraper5.LyngsatMapsScraper FileInfo indexJsonFileInfo = outputDirectoryInfo.GetSubfile(String.Format("index_{0}_{1}_{2}.json", DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day)); if (!indexJsonFileInfo.Exists) { - Console.WriteLine("download index json"); + logger.Log(PluginLogLevel.Info,"download index json"); byte[] kmlByteArray = webClient.DownloadData(rootUrl); indexJsonFileInfo.WriteAllBytes(kmlByteArray); SafeThrottle(); } - - - ScraperStorageFactoryConnectionManager connectionManager = ScraperStorageFactoryConnectionManager.GetInstance(); - IScraperStorageFactory storageFactory = connectionManager.AutoGetDefaultFactory(); - IScraperStroage storage = storageFactory.CreateScraperStroage(); - Console.WriteLine("mark all beams as disabled"); + + StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance(); + DataStorageFactory dataStorageFactory = storageConnectionManager.GetDefaultDataStorageFactory(); + DataStorage storage = dataStorageFactory.CreateDataStorage(); + logger.Log(PluginLogLevel.Info,"mark all beams as disabled"); storage.BeamsDisableAll(); @@ -55,7 +58,7 @@ namespace skyscraper5.LyngsatMapsScraper foreach (IndexJsonKml kml in indexJson.kml_list) { SaneSatellitePointer databasePointer = kml.Sanitize(); - Console.WriteLine("mark beam {0} enabled", kml.name); + logger.Log(PluginLogLevel.Info,"mark beam {0} enabled", kml.name); storage.BeamsEnable(databasePointer.id, databasePointer.satpos, databasePointer.name, databasePointer.beamsProcessTimestamp); FileInfo kmlFileInfo = outputDirectoryInfo.GetSubfile(String.Format("{0}_{1}_{2}_{3}.kml", kml.satpos, kml.name, kml.id, kml.beams_process_timestamp)); @@ -67,7 +70,7 @@ namespace skyscraper5.LyngsatMapsScraper gotHeaders = true; } - Console.WriteLine("download {0}", kml.url); + logger.Log(PluginLogLevel.Info,"download {0}", kml.url); try { byte[] kmlData = webClient.DownloadData(kml.url); @@ -85,13 +88,13 @@ namespace skyscraper5.LyngsatMapsScraper } else { - Console.WriteLine(we); + logger.Log(PluginLogLevel.Error,we.ToString()); throw; } } catch (Exception e) { - Console.WriteLine(e); + logger.Log(PluginLogLevel.Error, e.ToString()); throw; } } @@ -130,7 +133,7 @@ namespace skyscraper5.LyngsatMapsScraper if (!storage.TestForBeamFootprint(databasePointer.id, databasePointer.beamsProcessTimestamp, name, id)) { - Console.WriteLine("write footprint {0}", name); + logger.Log(PluginLogLevel.Info,"write footprint {0}", name); storage.BeamFootprintStore(databasePointer.id, databasePointer.beamsProcessTimestamp, name, polygonString2, id); } } diff --git a/MpePlugins/skyscraper5.Aprs/AprsStorage/AprsPostgresqlStorage.cs b/MpePlugins/skyscraper5.Aprs/AprsStorage/AprsPostgresqlStorage.cs index b83a44a..c542d7c 100644 --- a/MpePlugins/skyscraper5.Aprs/AprsStorage/AprsPostgresqlStorage.cs +++ b/MpePlugins/skyscraper5.Aprs/AprsStorage/AprsPostgresqlStorage.cs @@ -9,16 +9,17 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using skyscraper5.Data.PostgreSql; namespace skyscraper5.Aprs.AprsStorage { internal class AprsPostgresqlStorage : LX9SESStorage { - private readonly NpgsqlConnectionStringBuilder connectionStringBuilder; + private readonly PostgresqlToken _postgresql; - public AprsPostgresqlStorage(NpgsqlConnectionStringBuilder npgsqlConnectionStringBuilder) + public AprsPostgresqlStorage(PostgresqlToken postgresql) { - this.connectionStringBuilder = npgsqlConnectionStringBuilder; + _postgresql = postgresql; } #region Background Worker diff --git a/MpePlugins/skyscraper5.Aprs/LX9SESPlugin.cs b/MpePlugins/skyscraper5.Aprs/LX9SESPlugin.cs index efabbaa..ec2bfe9 100644 --- a/MpePlugins/skyscraper5.Aprs/LX9SESPlugin.cs +++ b/MpePlugins/skyscraper5.Aprs/LX9SESPlugin.cs @@ -9,6 +9,7 @@ using GeoCoordinatePortable; using Npgsql; using skyscraper5.Aprs.AprsSharp; using skyscraper5.Aprs.AprsStorage; +using skyscraper5.Data.PostgreSql; using skyscraper5.Ietf.Rfc971; using skyscraper5.Skyscraper.Plugins; using skyscraper5.Skyscraper.Scraper.Storage; @@ -21,45 +22,26 @@ namespace skyscraper5.Aprs [PluginPriority(1)] internal class Lx9SesPlugin : ISkyscraperMpePlugin { - public void ConnectToStorage(object[] connector, PluginLogMessage logger) + private LX9SESStorage storage; + public void ConnectToStorage(object[] connector) { - bool skipNext = false; - foreach (object o in connector) + object o = connector[0]; + switch (o) { - if (skipNext) - { - skipNext = false; - continue; - } - Type connectorType = o.GetType(); - if (connectorType == typeof(InMemoryPluginToken)) - { - _storage = new AprsInMemoryStorage((InMemoryPluginToken)o); + case InMemoryPluginToken t1: + storage = new AprsInMemoryStorage(t1); break; - } - else if (connectorType == typeof(NpgsqlConnectionStringBuilder)) - { - _storage = new AprsPostgresqlStorage((NpgsqlConnectionStringBuilder)o); - } - else - { - if (connectorType.Name.Equals("MinioClient")) - { - skipNext = true; - continue; - } - - if (connectorType.Name.Equals("DirectoryInfo")) - { - skipNext = true; - continue; - } - - throw new NotImplementedException(); - } + case PostgresqlToken t2: + storage = new AprsPostgresqlStorage(t2); + break; + default: + throw new NotImplementedException(o.GetType().FullName); } + } - messageReceiver = new LX9SESMessageReceiver(_storage, logger); + public void SetContext(DateTime? currentTime, object skyscraperContext) + { + } public void SetContext(DateTime? currentDateTime) diff --git a/MpePlugins/skyscraper5.Aprs/skyscraper5.Aprs.csproj b/MpePlugins/skyscraper5.Aprs/skyscraper5.Aprs.csproj index b83c3e9..6beda27 100644 --- a/MpePlugins/skyscraper5.Aprs/skyscraper5.Aprs.csproj +++ b/MpePlugins/skyscraper5.Aprs/skyscraper5.Aprs.csproj @@ -13,6 +13,7 @@ + diff --git a/skyscraper8.Tests/DocsisTestsLong.cs b/skyscraper8.Tests/DocsisTestsLong.cs index b1467f0..7048a4c 100644 --- a/skyscraper8.Tests/DocsisTestsLong.cs +++ b/skyscraper8.Tests/DocsisTestsLong.cs @@ -11,6 +11,7 @@ using System.Diagnostics; using System.Linq; using System.Text; using System.Threading.Tasks; +using skyscraper8.Skyscraper.Scraper.Storage; namespace skyscraper8.Tests { @@ -30,7 +31,7 @@ namespace skyscraper8.Tests storageFactory.Directory = "docsis_longboi"; Passing passing = new Passing(); - passing.ScraperStorage = storageFactory.CreateScraperStroage(); + passing.DataStorage = storageFactory.CreateDataStorage(); passing.MassImportDirectory(di); } @@ -55,9 +56,9 @@ namespace skyscraper8.Tests M3U8Stream m3u8 = new M3U8Stream(docsisM3u8.FullName); TsContext tsContext = new TsContext(); - InMemoryScraperStorageFactory storageFactory = new InMemoryScraperStorageFactory(); - IScraperStroage scraperStorage = storageFactory.CreateScraperStroage(); - SkyscraperContext skyscraperContext = new SkyscraperContext(tsContext, null, scraperStorage); + InMemoryScraperStorageFactory storageFactory = new InMemoryScraperStorageFactory(); + DataStorage scraperStorage = storageFactory.CreateDataStorage(); + SkyscraperContext skyscraperContext = new SkyscraperContext(tsContext, scraperStorage, null); skyscraperContext.InitalizeFilterChain(); skyscraperContext.IngestFromStream(m3u8); diff --git a/skyscraper8.Tests/SanityTests.cs b/skyscraper8.Tests/SanityTests.cs new file mode 100644 index 0000000..4072ada --- /dev/null +++ b/skyscraper8.Tests/SanityTests.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using skyscraper5.Skyscraper.IO; + +namespace skyscraper8.Tests +{ + public class SanityTests + { + [Fact] + public void Test15bits() + { + byte[] bcd = new byte[] { 0xa9, 0x23 }; + MemoryStream ms = new MemoryStream(bcd); + byte readUInt8 = ms.ReadUInt8(); + + bool msbSet = (readUInt8 & 0x80) != 0; + Assert.True(msbSet); + + int result = ms.ReadUInt8(); + result <<= 8; + result += readUInt8; + Assert.Equal(9001, result); + } + } +} diff --git a/skyscraper8/Passing.cs b/skyscraper8/Passing.cs index bfee833..9792854 100644 --- a/skyscraper8/Passing.cs +++ b/skyscraper8/Passing.cs @@ -27,8 +27,8 @@ namespace skyscraper5 { public class Passing { - public DataStorage DataStorage { get; private set; } - public ObjectStorage ObjectStorage { get; private set; } + public DataStorage DataStorage { get; set; } + public ObjectStorage ObjectStorage { get; set; } private IStreamReader streamReader; private List tuners; private List satellitePositions;