Fixed the Lyngsat Map Scraper to work with the new architecture.

DataTableStorages/skyscraper5.Data.MySql/Ait.cs

@@ -8,6 +8,9 @@ using skyscraper5.Mhp.Descriptors;
 using skyscraper5.Mhp.Descriptors.InteractionTransportSelectors;
 using System;
 using skyscraper5.Teletext;
+using skyscraper8.DvbI;
+using skyscraper8.DvbNip;
+using skyscraper8.Ses;
 using skyscraper8.Skyscraper.Scraper.Storage;
 
 namespace skyscraper5.Data.MySql
@@ -275,5 +278,157 @@ namespace skyscraper5.Data.MySql
 			}
 		}
 
-	}
+		public bool TestForSgtList(SgtList list)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void InsertSgtList(SgtList list)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool TestForSgtService(SgtService child)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void InsertSgtService(SgtService child)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void InsertDvbiService(DvbIService service)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool TestForDvbiServiceList(string id)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool DvbNipPrivateDataSpecifier(NipActualCarrierInformation currentCarrierInformation, DateTime versionUpdate,
+			uint privateDataSpecifier, List<string> privateDataSessions)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool DvbNipTestForNetwork(BroadcastNetworkType network)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void DvbNipInsertNetwork(BroadcastNetworkType network)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool DvbNipTestForService(BroadcastMediaStreamType broadcastMediaStreamType)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void DvbNipInsertService(BroadcastMediaStreamType broadcastMediaStreamType)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier,
+			MulticastEndpointAddressType multicastEndpointAddressType)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier,
+			MulticastEndpointAddressType multicastGatewayConfigurationTransportSession)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void DvbNipInsertCarrier(NipActualCarrierInformation currentCarrierInformation)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void InsertDvbiServiceListEntryPoint(long sourceHash)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool TestForDvbiService(string id)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool TestForDvbiServiceListEntryPoints(long sourceHash)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void UpdateDvbiService(DvbIService service)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void UpdateDvbiServiceListEntryPointUpdateDate(long hash, DateTime currentTime)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void UpdateDvbiServiceListLastCheckedDate(string id, DateTime currentTime)
+		{
+			throw new NotImplementedException();
+		}
+
+		public int GetDvbiServiceVersion(string id)
+		{
+			throw new NotImplementedException();
+		}
+
+		public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void AddDvbiServiceToServiceList(string id, string serviceListId)
+		{
+			throw new NotImplementedException();
+		}
+
+		public DateTime GetDvbiServiceListLastUpdateDate(string id)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void AddDvbiServiceListToServiceListEntryPoint(DvbiServiceList serviceList, long sourceHash)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool TestForServiceListEntryPoints(long sourceHash)
+		{
+			throw new NotImplementedException();
+		}
+
+		public void InsertDvbiServiceList(DvbiServiceList serviceList)
+		{
+			throw new NotImplementedException();
+		}
+    }
 }

Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs

@@ -7,6 +7,8 @@ using SharpKml.Dom;
 using SharpKml.Engine;
 using skyscraper5.LyngsatMapsScraper.Model;
 using skyscraper5.Skyscraper.Scraper.Storage;
+using skyscraper8.Skyscraper.Plugins;
+using skyscraper8.Skyscraper.Scraper.Storage;
 
 namespace skyscraper5.LyngsatMapsScraper
 {
@@ -21,6 +23,7 @@ namespace skyscraper5.LyngsatMapsScraper
 		private bool gotHeaders;
 		private const string rootUrl = "https://genmap.lyngsat.org/server2/kml";
 		private Random rng;
+		
 		private void SafeThrottle()
 		{
 			if (rng == null)
@@ -30,6 +33,7 @@ namespace skyscraper5.LyngsatMapsScraper
 
 		private void Run()
 		{
+			PluginLogger logger = PluginLogManager.GetLogger(typeof(Program));
 			WebClient webClient = new WebClient();
 			DirectoryInfo outputDirectoryInfo = new DirectoryInfo("temp");
 			if (!outputDirectoryInfo.Exists)
@@ -37,17 +41,16 @@ namespace skyscraper5.LyngsatMapsScraper
 			FileInfo indexJsonFileInfo = outputDirectoryInfo.GetSubfile(String.Format("index_{0}_{1}_{2}.json", DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day));
 			if (!indexJsonFileInfo.Exists)
 			{
-				Console.WriteLine("download index json");
+				logger.Log(PluginLogLevel.Info,"download index json");
 				byte[] kmlByteArray = webClient.DownloadData(rootUrl);
 				indexJsonFileInfo.WriteAllBytes(kmlByteArray);
 				SafeThrottle();
 			}
 			
-
+			StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
-			ScraperStorageFactoryConnectionManager connectionManager = ScraperStorageFactoryConnectionManager.GetInstance();
+			DataStorageFactory dataStorageFactory = storageConnectionManager.GetDefaultDataStorageFactory();
-			IScraperStorageFactory storageFactory = connectionManager.AutoGetDefaultFactory();
+			DataStorage storage = dataStorageFactory.CreateDataStorage();
-			IScraperStroage storage = storageFactory.CreateScraperStroage();
+			logger.Log(PluginLogLevel.Info,"mark all beams as disabled");
-			Console.WriteLine("mark all beams as disabled");
 			storage.BeamsDisableAll();
 
 
@@ -55,7 +58,7 @@ namespace skyscraper5.LyngsatMapsScraper
 			foreach (IndexJsonKml kml in indexJson.kml_list)
 			{
 				SaneSatellitePointer databasePointer = kml.Sanitize();
-				Console.WriteLine("mark beam {0} enabled", kml.name);
+				logger.Log(PluginLogLevel.Info,"mark beam {0} enabled", kml.name);
 				storage.BeamsEnable(databasePointer.id, databasePointer.satpos, databasePointer.name, databasePointer.beamsProcessTimestamp);
 
 				FileInfo kmlFileInfo = outputDirectoryInfo.GetSubfile(String.Format("{0}_{1}_{2}_{3}.kml", kml.satpos, kml.name, kml.id, kml.beams_process_timestamp));
@@ -67,7 +70,7 @@ namespace skyscraper5.LyngsatMapsScraper
 						gotHeaders = true;
 					}
 
-					Console.WriteLine("download {0}", kml.url);
+					logger.Log(PluginLogLevel.Info,"download {0}", kml.url);
 					try
 					{
 						byte[] kmlData = webClient.DownloadData(kml.url);
@@ -85,13 +88,13 @@ namespace skyscraper5.LyngsatMapsScraper
 						}
 						else
 						{
-							Console.WriteLine(we);
+							logger.Log(PluginLogLevel.Error,we.ToString());
 							throw;
 						}
 					}
 					catch (Exception e)
 					{
-						Console.WriteLine(e);
+						logger.Log(PluginLogLevel.Error, e.ToString());
 						throw;
 					}
 				}
@@ -130,7 +133,7 @@ namespace skyscraper5.LyngsatMapsScraper
 
 					if (!storage.TestForBeamFootprint(databasePointer.id, databasePointer.beamsProcessTimestamp, name, id))
 					{
-						Console.WriteLine("write footprint {0}", name);
+						logger.Log(PluginLogLevel.Info,"write footprint {0}", name);
 						storage.BeamFootprintStore(databasePointer.id, databasePointer.beamsProcessTimestamp, name, polygonString2, id);
 					}
 				}

MpePlugins/skyscraper5.Aprs/AprsStorage/AprsPostgresqlStorage.cs

@@ -9,16 +9,17 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper5.Data.PostgreSql;
 
 namespace skyscraper5.Aprs.AprsStorage
 {
 	internal class AprsPostgresqlStorage : LX9SESStorage
 	{
-		private readonly NpgsqlConnectionStringBuilder connectionStringBuilder;
+		private readonly PostgresqlToken _postgresql;
 
-		public AprsPostgresqlStorage(NpgsqlConnectionStringBuilder npgsqlConnectionStringBuilder)
+		public AprsPostgresqlStorage(PostgresqlToken postgresql)
 		{
-			this.connectionStringBuilder = npgsqlConnectionStringBuilder;
+			_postgresql = postgresql;
 		}
 
 		#region Background Worker

MpePlugins/skyscraper5.Aprs/LX9SESPlugin.cs

@@ -9,6 +9,7 @@ using GeoCoordinatePortable;
 using Npgsql;
 using skyscraper5.Aprs.AprsSharp;
 using skyscraper5.Aprs.AprsStorage;
+using skyscraper5.Data.PostgreSql;
 using skyscraper5.Ietf.Rfc971;
 using skyscraper5.Skyscraper.Plugins;
 using skyscraper5.Skyscraper.Scraper.Storage;
@@ -21,45 +22,26 @@ namespace skyscraper5.Aprs
 	[PluginPriority(1)]
 	internal class Lx9SesPlugin : ISkyscraperMpePlugin
 	{
-		public void ConnectToStorage(object[] connector, PluginLogMessage logger)
+		private LX9SESStorage storage;
+		public void ConnectToStorage(object[] connector)
 		{
-			bool skipNext = false;
+			object o = connector[0];
-			foreach (object o in connector)
+			switch (o)
 			{
-				if (skipNext)
+				case InMemoryPluginToken t1:
-				{
+					storage = new AprsInMemoryStorage(t1);
-					skipNext = false;
-					continue;
-				}
-				Type connectorType = o.GetType();
-				if (connectorType == typeof(InMemoryPluginToken))
-				{
-					_storage = new AprsInMemoryStorage((InMemoryPluginToken)o);
 					break;
-				}
+				case PostgresqlToken t2:
-				else if (connectorType == typeof(NpgsqlConnectionStringBuilder))
+					storage = new AprsPostgresqlStorage(t2);
-				{
+					break;
-					_storage = new AprsPostgresqlStorage((NpgsqlConnectionStringBuilder)o);
+				default:
-				}
+					throw new NotImplementedException(o.GetType().FullName);
-				else
-				{
-					if (connectorType.Name.Equals("MinioClient"))
-					{
-						skipNext = true;
-						continue;
-					}
-
-					if (connectorType.Name.Equals("DirectoryInfo"))
-					{
-						skipNext = true;
-						continue;
-					}
-
-					throw new NotImplementedException();
-				}
 			}
+		}
+
+		public void SetContext(DateTime? currentTime, object skyscraperContext)
+		{
 			
-			messageReceiver = new LX9SESMessageReceiver(_storage, logger);
 		}
 
 		public void SetContext(DateTime? currentDateTime)

MpePlugins/skyscraper5.Aprs/skyscraper5.Aprs.csproj

@@ -13,6 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\DataTableStorages\skyscraper5.Data.PostgreSql\skyscraper5.Data.PostgreSql.csproj" />
     <ProjectReference Include="..\..\skyscraper8\skyscraper8.csproj" />
   </ItemGroup>
 

skyscraper8.Tests/DocsisTestsLong.cs

@@ -11,6 +11,7 @@ using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using skyscraper8.Skyscraper.Scraper.Storage;
 
 namespace skyscraper8.Tests
 {
@@ -30,7 +31,7 @@ namespace skyscraper8.Tests
 			storageFactory.Directory = "docsis_longboi";
 
 			Passing passing = new Passing();
-			passing.ScraperStorage = storageFactory.CreateScraperStroage();
+			passing.DataStorage = storageFactory.CreateDataStorage();
 			passing.MassImportDirectory(di);
 		}
 
@@ -56,8 +57,8 @@ namespace skyscraper8.Tests
 
 			TsContext tsContext = new TsContext();
 			InMemoryScraperStorageFactory storageFactory = new InMemoryScraperStorageFactory(); 
-			IScraperStroage scraperStorage = storageFactory.CreateScraperStroage();
+			DataStorage scraperStorage = storageFactory.CreateDataStorage();
-			SkyscraperContext skyscraperContext = new SkyscraperContext(tsContext, null, scraperStorage);
+			SkyscraperContext skyscraperContext = new SkyscraperContext(tsContext, scraperStorage, null);
 			skyscraperContext.InitalizeFilterChain();
 			skyscraperContext.IngestFromStream(m3u8);
 

skyscraper8.Tests/SanityTests.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using skyscraper5.Skyscraper.IO;
+
+namespace skyscraper8.Tests
+{
+	public class SanityTests
+	{
+		[Fact]
+		public void Test15bits()
+		{
+			byte[] bcd = new byte[] { 0xa9, 0x23 };
+			MemoryStream ms = new MemoryStream(bcd);
+			byte readUInt8 = ms.ReadUInt8();
+
+			bool msbSet = (readUInt8 & 0x80) != 0;
+			Assert.True(msbSet);
+
+			int result = ms.ReadUInt8();
+			result <<= 8;
+			result += readUInt8;
+			Assert.Equal(9001, result);
+		}
+	}
+}

skyscraper8/Passing.cs

@@ -27,8 +27,8 @@ namespace skyscraper5
 {
     public class Passing
     {
-        public DataStorage DataStorage { get; private set; }
+        public DataStorage DataStorage { get; set; }
-        public ObjectStorage ObjectStorage { get; private set; }
+        public ObjectStorage ObjectStorage { get; set; }
         private IStreamReader streamReader;
         private List<TunerMetadata> tuners;
         private List<SatellitePosition> satellitePositions;