using System.Globalization; using System.Net; using System.Text; using System.Xml.Serialization; using SharpKml.Base; using SharpKml.Dom; using SharpKml.Engine; using skyscraper5.LyngsatMapsScraper.Model; using skyscraper5.Skyscraper.Scraper.Storage; using skyscraper8.Skyscraper.Plugins; using skyscraper8.Skyscraper.Scraper.Storage; namespace skyscraper5.LyngsatMapsScraper { internal class Program { static void Main(string[] args) { new Program().Run(); } private XmlSerializer xmlSerializer; private bool gotHeaders; private const string rootUrl = "https://genmap.lyngsat.org/server2/kml"; private Random rng; private void SafeThrottle() { if (rng == null) rng = new Random(); Thread.Sleep(rng.Next(500, 1500)); } private void Run() { PluginLogger logger = PluginLogManager.GetLogger(typeof(Program)); WebClient webClient = new WebClient(); DirectoryInfo outputDirectoryInfo = new DirectoryInfo("temp"); if (!outputDirectoryInfo.Exists) outputDirectoryInfo.Create(); FileInfo indexJsonFileInfo = outputDirectoryInfo.GetSubfile(String.Format("index_{0}_{1}_{2}.json", DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day)); if (!indexJsonFileInfo.Exists) { logger.Log(PluginLogLevel.Info,"download index json"); byte[] kmlByteArray = webClient.DownloadData(rootUrl); indexJsonFileInfo.WriteAllBytes(kmlByteArray); SafeThrottle(); } StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance(); DataStorageFactory dataStorageFactory = storageConnectionManager.GetDefaultDataStorageFactory(); DataStorage storage = dataStorageFactory.CreateDataStorage(); logger.Log(PluginLogLevel.Info,"mark all beams as disabled"); storage.BeamsDisableAll(); IndexJson indexJson = Newtonsoft.Json.JsonConvert.DeserializeObject(indexJsonFileInfo.ReadAllText()); foreach (IndexJsonKml kml in indexJson.kml_list) { SaneSatellitePointer databasePointer = kml.Sanitize(); logger.Log(PluginLogLevel.Info,"mark beam {0} enabled", kml.name); storage.BeamsEnable(databasePointer.id, databasePointer.satpos, databasePointer.name, databasePointer.beamsProcessTimestamp); FileInfo kmlFileInfo = outputDirectoryInfo.GetSubfile(String.Format("{0}_{1}_{2}_{3}.kml", kml.satpos, kml.name, kml.id, kml.beams_process_timestamp)); if (!kmlFileInfo.Exists) { if (!gotHeaders) { webClient.Headers.Add("Referer", rootUrl); gotHeaders = true; } logger.Log(PluginLogLevel.Info,"download {0}", kml.url); try { byte[] kmlData = webClient.DownloadData(kml.url); kmlFileInfo.WriteAllBytes(kmlData); SafeThrottle(); } catch (WebException we) { HttpWebResponse httpWebResponse = we.Response as HttpWebResponse; if (httpWebResponse.StatusCode == HttpStatusCode.NotFound) { storage.BeamsDisableSpecific(databasePointer.id, databasePointer.satpos, databasePointer.name, databasePointer.beamsProcessTimestamp); SafeThrottle(); continue; } else { logger.Log(PluginLogLevel.Error,we.ToString()); throw; } } catch (Exception e) { logger.Log(PluginLogLevel.Error, e.ToString()); throw; } } FileStream kmlStream = kmlFileInfo.OpenRead(); KmlFile kmlFile = KmlFile.Load(kmlStream); Document rootFeature = (SharpKml.Dom.Document)((SharpKml.Dom.Kml)kmlFile.Root).Feature; foreach (Feature feature in rootFeature.Features) { Placemark placemark = feature as Placemark; if (placemark == null) continue; string name = placemark.Name; Polygon polygon = placemark.Geometry as Polygon; if (polygon == null) continue; string id = polygon.Id; CoordinateCollection linearRingCoordinates = polygon.OuterBoundary.LinearRing.Coordinates; StringBuilder sb = new StringBuilder(); foreach (Vector vector in linearRingCoordinates) { sb.AppendFormat("{0},{1}\t", vector.Longitude.ToString(CultureInfo.InvariantCulture), vector.Latitude.ToString(CultureInfo.InvariantCulture)); } string polygonString = sb.ToString().Trim('\t'); if (string.IsNullOrEmpty(polygonString)) continue; Skyscraper.Drawing.Polygon testPolygon = new Skyscraper.Drawing.Polygon(polygonString); string polygonString2 = testPolygon.GetPolygonString(); if (!polygonString2.Equals(polygonString)) { throw new ApplicationException("Oh dear, I think we're encountering a bug."); } if (!storage.TestForBeamFootprint(databasePointer.id, databasePointer.beamsProcessTimestamp, name, id)) { logger.Log(PluginLogLevel.Info,"write footprint {0}", name); storage.BeamFootprintStore(databasePointer.id, databasePointer.beamsProcessTimestamp, name, polygonString2, id); } } kmlStream.Close(); } } } internal static class DirectoryInfoExtensions { public static FileInfo GetSubfile(this DirectoryInfo directory, string filename) { string path = Path.Combine(directory.FullName, filename); return new FileInfo(path); } } internal static class FileInfoExtensionis { public static void WriteAllBytes(this FileInfo fileInfo, byte[] bytes) { File.WriteAllBytes(fileInfo.FullName, bytes); } public static byte[] ReadAllBytes(this FileInfo fileInfo) { return File.ReadAllBytes(fileInfo.FullName); } public static string ReadAllText(this FileInfo fileInfo) { return File.ReadAllText(fileInfo.FullName); } } }