skyscraper8/Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs

using System.Globalization;
using System.Net;
using System.Text;
using System.Xml.Serialization;
using SharpKml.Base;
using SharpKml.Dom;
using SharpKml.Engine;
using skyscraper5.LyngsatMapsScraper.Model;
using skyscraper5.Skyscraper.Scraper.Storage;
using skyscraper8.Skyscraper.Plugins;
using skyscraper8.Skyscraper.Scraper.Storage;

namespace skyscraper5.LyngsatMapsScraper
{
	internal class Program
	{
		static void Main(string[] args)
		{
			new Program().Run();
		}

		private XmlSerializer xmlSerializer;
		private bool gotHeaders;
		private const string rootUrl = "https://genmap.lyngsat.org/server2/kml";
		private Random rng;
		
		private void SafeThrottle()
		{
			if (rng == null)
				rng = new Random();
			Thread.Sleep(rng.Next(500, 1500));
		}

		private void Run()
		{
			PluginLogger logger = PluginLogManager.GetLogger(typeof(Program));
			WebClient webClient = new WebClient();
			DirectoryInfo outputDirectoryInfo = new DirectoryInfo("temp");
			if (!outputDirectoryInfo.Exists)
				outputDirectoryInfo.Create();
			FileInfo indexJsonFileInfo = outputDirectoryInfo.GetSubfile(String.Format("index_{0}_{1}_{2}.json", DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day));
			if (!indexJsonFileInfo.Exists)
			{
				logger.Log(PluginLogLevel.Info,"download index json");
				byte[] kmlByteArray = webClient.DownloadData(rootUrl);
				indexJsonFileInfo.WriteAllBytes(kmlByteArray);
				SafeThrottle();
			}
			
			StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
			DataStorageFactory dataStorageFactory = storageConnectionManager.GetDefaultDataStorageFactory();
			DataStorage storage = dataStorageFactory.CreateDataStorage();
			logger.Log(PluginLogLevel.Info,"mark all beams as disabled");
			storage.BeamsDisableAll();


			IndexJson indexJson = Newtonsoft.Json.JsonConvert.DeserializeObject<IndexJson>(indexJsonFileInfo.ReadAllText());
			foreach (IndexJsonKml kml in indexJson.kml_list)
			{
				SaneSatellitePointer databasePointer = kml.Sanitize();
				logger.Log(PluginLogLevel.Info,"mark beam {0} enabled", kml.name);
				storage.BeamsEnable(databasePointer.id, databasePointer.satpos, databasePointer.name, databasePointer.beamsProcessTimestamp);

				FileInfo kmlFileInfo = outputDirectoryInfo.GetSubfile(String.Format("{0}_{1}_{2}_{3}.kml", kml.satpos, kml.name, kml.id, kml.beams_process_timestamp));
				if (!kmlFileInfo.Exists)
				{
					if (!gotHeaders)
					{
						webClient.Headers.Add("Referer", rootUrl);
						gotHeaders = true;
					}

					logger.Log(PluginLogLevel.Info,"download {0}", kml.url);
					try
					{
						byte[] kmlData = webClient.DownloadData(kml.url);
						kmlFileInfo.WriteAllBytes(kmlData);
						SafeThrottle();
					}
					catch (WebException we)
					{
						HttpWebResponse httpWebResponse = we.Response as HttpWebResponse;
						if (httpWebResponse.StatusCode == HttpStatusCode.NotFound)
						{
							storage.BeamsDisableSpecific(databasePointer.id, databasePointer.satpos, databasePointer.name, databasePointer.beamsProcessTimestamp);
							SafeThrottle();
							continue;
						}
						else
						{
							logger.Log(PluginLogLevel.Error,we.ToString());
							throw;
						}
					}
					catch (Exception e)
					{
						logger.Log(PluginLogLevel.Error, e.ToString());
						throw;
					}
				}

				FileStream kmlStream = kmlFileInfo.OpenRead();
				KmlFile kmlFile = KmlFile.Load(kmlStream);
				Document rootFeature = (SharpKml.Dom.Document)((SharpKml.Dom.Kml)kmlFile.Root).Feature;
				foreach (Feature feature in rootFeature.Features)
				{
					Placemark placemark = feature as Placemark;
					if (placemark == null)
						continue;
					string name = placemark.Name;

					Polygon polygon = placemark.Geometry as Polygon;
					if (polygon == null)
						continue;
					string id = polygon.Id;

					CoordinateCollection linearRingCoordinates = polygon.OuterBoundary.LinearRing.Coordinates;
					StringBuilder sb = new StringBuilder();
					foreach (Vector vector in linearRingCoordinates)
					{
						sb.AppendFormat("{0},{1}\t", vector.Longitude.ToString(CultureInfo.InvariantCulture), vector.Latitude.ToString(CultureInfo.InvariantCulture));
					}

					string polygonString = sb.ToString().Trim('\t');
					if (string.IsNullOrEmpty(polygonString))
						continue;
					Skyscraper.Drawing.Polygon testPolygon = new Skyscraper.Drawing.Polygon(polygonString);
					string polygonString2 = testPolygon.GetPolygonString();
					if (!polygonString2.Equals(polygonString))
					{
						throw new ApplicationException("Oh dear, I think we're encountering a bug.");
					}

					if (!storage.TestForBeamFootprint(databasePointer.id, databasePointer.beamsProcessTimestamp, name, id))
					{
						logger.Log(PluginLogLevel.Info,"write footprint {0}", name);
						storage.BeamFootprintStore(databasePointer.id, databasePointer.beamsProcessTimestamp, name, polygonString2, id);
					}
				}
				kmlStream.Close();
			}
		}
	}

	internal static class DirectoryInfoExtensions
	{
		public static FileInfo GetSubfile(this DirectoryInfo directory, string filename)
		{
			string path = Path.Combine(directory.FullName, filename);
			return new FileInfo(path);
		}
	}

	internal static class FileInfoExtensionis
	{
		public static void WriteAllBytes(this FileInfo fileInfo, byte[] bytes)
		{
			File.WriteAllBytes(fileInfo.FullName, bytes);
		}

		public static byte[] ReadAllBytes(this FileInfo fileInfo)
		{
			return File.ReadAllBytes(fileInfo.FullName);
		}

		public static string ReadAllText(this FileInfo fileInfo)
		{
			return File.ReadAllText(fileInfo.FullName);
		}
	}
}