diff --git a/GUIs/skyscraper5.UI/Form1.cs b/GUIs/skyscraper5.UI/Form1.cs index 15ed172..2091d2d 100644 --- a/GUIs/skyscraper5.UI/Form1.cs +++ b/GUIs/skyscraper5.UI/Form1.cs @@ -19,7 +19,7 @@ namespace skyscraper5.UI AcquiredStream = acquiredStream; tsContext = new TsContext(); tsContext.OnPacketLoss += TsContext_OnPacketLoss; - skyscraperContext = new SkyscraperContext(tsContext, eventLogger, storage); + skyscraperContext = new SkyscraperContext(tsContext, storage); skyscraperContext.UiJunction = uiJunction; } @@ -31,7 +31,6 @@ namespace skyscraper5.UI public StreamSource AcquiredStream { get; } private TsContext tsContext; private SkyscraperContext skyscraperContext; - private ISkyscraperEventLogger eventLogger; private IScraperStroage storage; private Form1UiJunction uiJunction; private DateTime windowOpenedTimestamp; diff --git a/skyscraper8/Abertis/AbertisDemoProgram.cs b/skyscraper8/Abertis/AbertisDemoProgram.cs index def6fd9..4948172 100644 --- a/skyscraper8/Abertis/AbertisDemoProgram.cs +++ b/skyscraper8/Abertis/AbertisDemoProgram.cs @@ -20,9 +20,8 @@ namespace skyscraper5.Abertis AbertisDecoder decoder = new AbertisDecoder(0x02be,this); ts.RegisterPacketProcessor(0x02be, decoder); - ConsoleEventLogger logger = new ConsoleEventLogger(); InMemoryScraperStorage scraper = new InMemoryScraperStorage(); - SkyscraperContext context = new SkyscraperContext(ts, logger, scraper); + SkyscraperContext context = new SkyscraperContext(ts, scraper); FileStream fileStream = fi.OpenRead(); context.IngestFromStream(fileStream); } diff --git a/skyscraper8/DvbI/DvbIDataStorage.cs b/skyscraper8/DvbI/DvbIDataStorage.cs new file mode 100644 index 0000000..820f495 --- /dev/null +++ b/skyscraper8/DvbI/DvbIDataStorage.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.DvbI +{ + public interface DvbIDataStorage + { + DateTime GetLastServiceListEntryPointUpdateDate(long sourceHash); + void InsertDvbiServiceListEntryPoint(long sourceHash); + bool TestForServiceListEntryPoints(long sourceHash); + } +} diff --git a/skyscraper8/DvbI/DvbIFilesystemProcessor.cs b/skyscraper8/DvbI/DvbIFilesystemProcessor.cs index 4215758..c8a4256 100644 --- a/skyscraper8/DvbI/DvbIFilesystemProcessor.cs +++ b/skyscraper8/DvbI/DvbIFilesystemProcessor.cs @@ -1,4 +1,6 @@ -using skyscraper5.Skyscraper.Plugins; +using log4net; +using moe.yo3explorer.skyscraper8.DVBI.Model; +using skyscraper5.Skyscraper.Plugins; using skyscraper8.yo3explorer; using System; using System.Collections.Generic; @@ -7,7 +9,6 @@ using System.Text; using System.Threading.Tasks; using System.Xml.Serialization; using Tsubasa.IO; -using moe.yo3explorer.skyscraper8.DVBI.Model; namespace skyscraper8.DvbI { @@ -20,7 +21,7 @@ namespace skyscraper8.DvbI internal class DvbIFilesystemProcessor : FilesystemProcessorPlugin { private const string INDEX_FILENAME = "\\DVB-I_slep.xml"; - + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); private XmlSerializer serviceListEntryPointSerializer; public void ProcessFilesystem(IFilesystem vfs, yo3explorerToSkyscraperContextBridge context, yo3explorerState state) @@ -28,8 +29,43 @@ namespace skyscraper8.DvbI if (!vfs.FileExists(INDEX_FILENAME)) return; + if (context.Source == ServiceListEntryPointSource.TransportStream) + { + if (!context.TransportStreamId.HasValue) + return; + + if (!context.OriginalNetworkId.HasValue) + return; + } + else + { + throw new NotImplementedException(context.Source.ToString()); + } + + if (context.CurrentTime == DateTime.MinValue) + return; + + DvbIDataStorage dataStorage = (DvbIDataStorage)context.DataStorage; + long sourceHash = context.GetSourceHash(); + if (dataStorage.TestForServiceListEntryPoints(sourceHash)) + { + DateTime lastChecked = dataStorage.GetLastServiceListEntryPointUpdateDate(sourceHash); + TimeSpan sinceLastChecked = context.CurrentTime - lastChecked; + if (sinceLastChecked.TotalDays < 1.0) + return; + } + else + { + dataStorage.InsertDvbiServiceListEntryPoint(sourceHash); + } + byte[] slepBytes = vfs.GetFile(INDEX_FILENAME); ServiceListEntryPoints serviceListEntryPoints = DvbIUtils.UnpackServiceListEntryPoints(slepBytes); + IEnumerable enumerable = DvbIUtils.FlattenServiceListEntryPoints(serviceListEntryPoints); + foreach(DvbiServiceList serviceList in enumerable) + { + logger.DebugFormat("Found DVB-I Service List: {0}", serviceList.ToString()); + } throw new NotImplementedException(); } diff --git a/skyscraper8/DvbI/DvbIUtils.cs b/skyscraper8/DvbI/DvbIUtils.cs index 006b25b..2c0a544 100644 --- a/skyscraper8/DvbI/DvbIUtils.cs +++ b/skyscraper8/DvbI/DvbIUtils.cs @@ -24,5 +24,42 @@ namespace skyscraper8.DvbI ServiceListEntryPoints serviceListEntryPoint = (ServiceListEntryPoints)slepWrapped; return serviceListEntryPoint; } + + public static IEnumerable FlattenServiceListEntryPoints(ServiceListEntryPoints input) + { + foreach(ServiceListEntryPointsProviderOffering offering in input.ProviderOffering) + { + foreach (ServiceListEntryPointsProviderOfferingServiceListOffering serviceListOffering in offering.ServiceListOffering) + { + DvbiServiceList serviceList = new DvbiServiceList(); + serviceList.Id = serviceListOffering.ServiceListId; + serviceList.Name = serviceListOffering.ServiceListName; + + foreach(ServiceListEntryPointsProviderOfferingServiceListOfferingDelivery delivery in serviceListOffering.Delivery) + { + if (delivery.DASHDelivery != null && string.IsNullOrEmpty(serviceList.DashDelivery)) + serviceList.DashDelivery = delivery.DASHDelivery; + if (delivery.DVBSDelivery != null && string.IsNullOrEmpty(serviceList.DvbSDelivery)) + serviceList.DvbSDelivery = delivery.DVBSDelivery[0].OrbitalPosition; + } + + if (serviceListOffering.RelatedMaterial != null) + { + if (serviceListOffering.RelatedMaterial.MediaLocator != null) + { + serviceList.RelatedMaterial = serviceListOffering.RelatedMaterial.MediaLocator[0].Value; + } + } + + if (serviceListOffering.ServiceListURI != null) + { + serviceList.URI = serviceListOffering.ServiceListURI[0].URI; + } + + yield return serviceList; + } + } + yield break; + } } } diff --git a/skyscraper8/DvbI/DvbiServiceList.cs b/skyscraper8/DvbI/DvbiServiceList.cs new file mode 100644 index 0000000..6d61505 --- /dev/null +++ b/skyscraper8/DvbI/DvbiServiceList.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.DvbI +{ + internal class DvbiServiceList + { + public string Id { get; internal set; } + public string Name { get; internal set; } + public string? DashDelivery { get; internal set; } + public string DvbSDelivery { get; internal set; } + public string RelatedMaterial { get; internal set; } + public string URI { get; internal set; } + + public override string ToString() + { + return Name; + } + } +} diff --git a/skyscraper8/DvbI/ServiceListEntryPointSource.cs b/skyscraper8/DvbI/ServiceListEntryPointSource.cs new file mode 100644 index 0000000..50d7ec3 --- /dev/null +++ b/skyscraper8/DvbI/ServiceListEntryPointSource.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.DvbI +{ + internal enum ServiceListEntryPointSource + { + TransportStream + } +} diff --git a/skyscraper8/Passing.cs b/skyscraper8/Passing.cs index f8c94b2..28ac37e 100644 --- a/skyscraper8/Passing.cs +++ b/skyscraper8/Passing.cs @@ -29,7 +29,6 @@ namespace skyscraper5 private IStreamReader streamReader; private List tuners; private List satellitePositions; - private ISkyscraperEventLogger eventLogger; public Passing() { @@ -284,10 +283,7 @@ namespace skyscraper5 private void ScrapeStream(Stream inStream, bool disk, out int tstype) { - if (eventLogger == null) - eventLogger = new ConsoleEventLogger(); - - SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), eventLogger, ScraperStorage); + SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), ScraperStorage); skyscraperContext.SourceIsDisk = disk; skyscraperContext.InitalizeFilterChain(); skyscraperContext.IngestFromStream(inStream); diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index 3b6a0bd..5967bf3 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -1,13 +1,10 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Net; -using System.Net.Http; using System.Net.NetworkInformation; using System.Net.Sockets; using System.Runtime.InteropServices; -using System.Threading; using skyscraper5.Abertis; using skyscraper5.Hdmv; using skyscraper5.Mpeg2; @@ -27,20 +24,24 @@ using skyscraper5.Skyscraper.Scraper.Storage.InMemory; using skyscraper5.Skyscraper.Webserver; using skyscraper5.src.Aac; using skyscraper5.src.Skyscraper.FrequencyListGenerator; -using skyscraper5.src.Sophtainer; using skyscraper5.T2MI; using skyscraper5.src.Mpeg2.PacketFilter; using skyscraper8.Skyscraper.IO; +using log4net; +[assembly: log4net.Config.XmlConfigurator(ConfigFile = "log4net.config")] namespace skyscraper5 { class Program { - static void Main(string[] args) + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + + static void Main(string[] args) { + logger.Info("Hello!"); ffmpegFrameGrabber.CanStart(); - Console.WriteLine("{1}? {0}", Environment.Is64BitOperatingSystem, "Is 64-bit OS?"); - Console.WriteLine("{1}? {0}", Environment.Is64BitProcess, "Is 64-bit Process?"); + logger.DebugFormat("Found {0}-bit Operating system.", Environment.Is64BitOperatingSystem ? 64 : 32); + logger.DebugFormat("I'm a {0}-bit Process.", Environment.Is64BitProcess ? 64 : 32); PluginManager.GetInstance(); @@ -148,8 +149,7 @@ namespace skyscraper5 { DirectoryInfo doThisDir = new DirectoryInfo(args[1]); IScraperStroage imss = new InMemoryScraperStorage(); - ConsoleEventLogger cel = new ConsoleEventLogger(); - TsFileCollectionImporter importer = new TsFileCollectionImporter(imss, doThisDir, cel); + TsFileCollectionImporter importer = new TsFileCollectionImporter(imss, doThisDir); importer.Run(); return; } @@ -161,8 +161,7 @@ namespace skyscraper5 DirectoryInfo srcDir = new DirectoryInfo(args[1]); DirectoryInfo tgtDir = new DirectoryInfo(args[2]); IScraperStroage fss = new FilesystemScraperStorage(tgtDir); - ConsoleEventLogger cel = new ConsoleEventLogger(); - TsFileCollectionImporter importer = new TsFileCollectionImporter(fss, srcDir, cel); + TsFileCollectionImporter importer = new TsFileCollectionImporter(fss, srcDir); importer.Run(); return; } @@ -248,7 +247,6 @@ namespace skyscraper5 private static void ProcessDirectory(DirectoryInfo di) { - ISkyscraperEventLogger eventLogger = new ConsoleEventLogger(); IScraperStroage scraperStroage = new InMemoryScraperStorage(); //DirectoryInfo di = new DirectoryInfo(@"E:\Skyscraper\Astra 19.2"); @@ -257,7 +255,7 @@ namespace skyscraper5 { Console.WriteLine(new string('_', Console.WindowWidth - 1)); Console.WriteLine("Processing: {0}", fileInfo.Name); - SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), eventLogger, scraperStroage); + SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), scraperStroage); //StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest(); FileStream fileStream = fileInfo.OpenRead(); @@ -294,19 +292,18 @@ namespace skyscraper5 private static void HandleUdpTesting() { - HandleUdpInput(new ConsoleEventLogger(), new InMemoryScraperStorage()); + HandleUdpInput(new InMemoryScraperStorage()); } private static void HandleUdpLive() { - ConsoleEventLogger eventLogger = new ConsoleEventLogger(); ScraperStorageFactoryConnectionManager connectionManager = ScraperStorageFactoryConnectionManager.GetInstance(); IScraperStorageFactory factory = connectionManager.AutoGetDefaultFactory(); IScraperStroage scraperStorage = factory.CreateScraperStroage(); - HandleUdpInput(eventLogger, scraperStorage); + HandleUdpInput(scraperStorage); } - private static void HandleUdpInput(ISkyscraperEventLogger eventLogger, IScraperStroage scraperStorage) + private static void HandleUdpInput(IScraperStroage scraperStorage) { NetworkInterface[] networkInterfaces = NetworkInterface.GetAllNetworkInterfaces(); foreach (NetworkInterface networkInterface in networkInterfaces) @@ -335,10 +332,10 @@ namespace skyscraper5 if (!remote.Equals(oldRemote)) { oldRemote = remote; - skyscraper = new SkyscraperContext(new TsContext(), eventLogger, scraperStorage); + skyscraper = new SkyscraperContext(new TsContext(), scraperStorage); skyscraper.InitalizeFilterChain(); skyscraper.EnableTimeout = true; - Console.WriteLine("Got zapped by {0}", remote); + logger.InfoFormat("Got zapped by {0}", remote); } numPackets = buffer.Length / 188; @@ -369,8 +366,7 @@ namespace skyscraper5 return; } - ISkyscraperEventLogger eventLogger = new ConsoleEventLogger(); - SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), eventLogger, scraperStorage); + SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), scraperStorage); skyscraper.InitalizeFilterChain(); skyscraper.EnableTimeout = false; skyscraper.TcpProxyEnabled = false; @@ -394,17 +390,15 @@ namespace skyscraper5 private static void HandleCrazyScan(string url, IScraperStroage scraperStorage, bool timeout) { - ISkyscraperEventLogger eventLogger = new ConsoleEventLogger(); - if (!url.StartsWith("tcp://")) { - Console.WriteLine("Use TCP, please."); + logger.Fatal("Use TCP, please."); return; } if (url.EndsWith("/")) { - Console.WriteLine("Don't use that trailing slash, please."); + logger.Fatal("Don't use that trailing slash, please."); return; } @@ -415,7 +409,7 @@ namespace skyscraper5 TcpClient tcpClient = new TcpClient(hostname, port); - SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), eventLogger, scraperStorage); + SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), scraperStorage); skyscraper.InitalizeFilterChain(new SkipFilter(1024)); //skyscraper.EnableTimeout = timeout; //skyscraper.TimeoutSeconds = 10; @@ -427,9 +421,8 @@ namespace skyscraper5 { //Environment.CurrentDirectory = fi.Directory.FullName; TsContext tsContext = new TsContext(); - ISkyscraperEventLogger eventLogger = new ConsoleEventLogger(); IScraperStroage scraperStorage = new InMemoryScraperStorage(); - SkyscraperContext skyscraper = new SkyscraperContext(tsContext, eventLogger, scraperStorage); + SkyscraperContext skyscraper = new SkyscraperContext(tsContext, scraperStorage); skyscraper.InitalizeFilterChain(); FileStream fileStream = fi.OpenRead(); skyscraper.IngestFromStream(fileStream); diff --git a/skyscraper8/Scorcher/DummyTsGenerator.cs b/skyscraper8/Scorcher/DummyTsGenerator.cs index cfcad92..6cb92c4 100644 --- a/skyscraper8/Scorcher/DummyTsGenerator.cs +++ b/skyscraper8/Scorcher/DummyTsGenerator.cs @@ -71,7 +71,7 @@ namespace skyscraper5.Scorcher outputStream = fi.OpenRead(); TsContext tsContext = new TsContext(); - SkyscraperContext sc = new SkyscraperContext(tsContext, null, null, 0); + SkyscraperContext sc = new SkyscraperContext(tsContext, null, 0); sc.IngestFromStream(outputStream); outputStream.Close(); diff --git a/skyscraper8/Skyscraper/CableFrequencyListGenerator.cs b/skyscraper8/Skyscraper/CableFrequencyListGenerator.cs index 3cf5a2a..b8b5fda 100644 --- a/skyscraper8/Skyscraper/CableFrequencyListGenerator.cs +++ b/skyscraper8/Skyscraper/CableFrequencyListGenerator.cs @@ -105,7 +105,6 @@ namespace skyscraper5.Skyscraper StreamReaderScraperController scraper = new StreamReaderScraperController(streamReader); scraper.ScraperStroage = new InMemoryScraperStorage(); scraper.Recording = false; - scraper.EventLogger = new ProgressBarLogger(); Console.Write("Running PSI acquisition..."); scraper.Run(); Console.WriteLine(".DONE"); @@ -164,28 +163,5 @@ namespace skyscraper5.Skyscraper Console.WriteLine("Heard something at {0}", searchResult.Freq / 1000); } - private class ProgressBarLogger : ISkyscraperEventLogger - { - public ProgressBarLogger() - { - eventsGoal = 1; - } - private ulong eventsGoal; - private ulong eventsDone; - private char[] animation = new char[] { '\\', '-', '/', '|' }; - public void Log(TimeSpan duration, DateTime eventTimestamp, SkyscraperContextEvent eventType, string name = null) - { - eventsDone++; - Console.Write('\b'); - if (eventsDone == eventsGoal) - { - Console.Write("."); - eventsGoal *= 2; - } - Console.Write(animation[(int)eventsDone % animation.Length]); - } - - public ulong NumEvents => eventsDone; - } } } diff --git a/skyscraper8/Skyscraper/IO/StreamReaderScanController.cs b/skyscraper8/Skyscraper/IO/StreamReaderScanController.cs index 439d0ee..828c6a0 100644 --- a/skyscraper8/Skyscraper/IO/StreamReaderScanController.cs +++ b/skyscraper8/Skyscraper/IO/StreamReaderScanController.cs @@ -401,7 +401,6 @@ namespace skyscraper5.Skyscraper.IO { srsc2 = new StreamReaderScraperController(streamReader); srsc2.RecordingOutputDirectory = new DirectoryInfo("recordings"); - srsc2.EventLogger = new ConsoleEventLogger(); srsc2.Recording = true; srsc2.ScraperStroage = new FilesystemScraperStorage(new DirectoryInfo("srtest_results")); } diff --git a/skyscraper8/Skyscraper/IO/StreamReaderScraperController.cs b/skyscraper8/Skyscraper/IO/StreamReaderScraperController.cs index 1c03a81..1ed5e4a 100644 --- a/skyscraper8/Skyscraper/IO/StreamReaderScraperController.cs +++ b/skyscraper8/Skyscraper/IO/StreamReaderScraperController.cs @@ -23,8 +23,6 @@ namespace skyscraper5.Skyscraper.IO private const string RECORDING_FILENAME_MASK = "skyscraper_{0}.ts"; - public ISkyscraperEventLogger EventLogger { get; set; } - public IScraperStroage ScraperStroage { get; set; } public bool Recording { get; set; } @@ -236,7 +234,7 @@ namespace skyscraper5.Skyscraper.IO } //Use the Filter - skyscraperContext = new SkyscraperContext(new TsContext(), EventLogger, ScraperStroage); + skyscraperContext = new SkyscraperContext(new TsContext(), ScraperStroage); skyscraperContext.TcpProxyEnabled = true; byte[] singlePacketBuffer = new byte[188]; while (!StopConditionMet()) diff --git a/skyscraper8/Skyscraper/Plugins/PluginManager.cs b/skyscraper8/Skyscraper/Plugins/PluginManager.cs index b1b9293..7ae0022 100644 --- a/skyscraper8/Skyscraper/Plugins/PluginManager.cs +++ b/skyscraper8/Skyscraper/Plugins/PluginManager.cs @@ -1,10 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Reflection; +using log4net; using skyscraper5.Docsis; using skyscraper5.DsmCc; using skyscraper5.Dvb; @@ -16,11 +10,20 @@ using skyscraper5.Skyscraper.Scraper.StreamAutodetection; using skyscraper5.src.Skyscraper.Scraper.Dns; using skyscraper5.T2MI; using skyscraper8.yo3explorer; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; namespace skyscraper5.Skyscraper.Plugins { internal class PluginManager { + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private PluginManager() { _mpePlugins = new List(); @@ -71,7 +74,7 @@ namespace skyscraper5.Skyscraper.Plugins } else { - Console.WriteLine(String.Format("{0} was not found. Create it using the UI!", iniFileInfo.FullName)); + logger.WarnFormat("{0} was not found. Create it using the UI!", iniFileInfo.FullName); } } diff --git a/skyscraper8/Skyscraper/RecordingImporter/TsFileCollectionImporter.cs b/skyscraper8/Skyscraper/RecordingImporter/TsFileCollectionImporter.cs index 2d245b0..9e1f82b 100644 --- a/skyscraper8/Skyscraper/RecordingImporter/TsFileCollectionImporter.cs +++ b/skyscraper8/Skyscraper/RecordingImporter/TsFileCollectionImporter.cs @@ -16,13 +16,11 @@ namespace skyscraper5.Skyscraper.RecordingImporter { public IScraperStroage ScraperStorage { get; } public DirectoryInfo RootDirectory { get; } - public ISkyscraperEventLogger EventLogger { get; } - public TsFileCollectionImporter(IScraperStroage scraperStorage, DirectoryInfo rootDirectory, ISkyscraperEventLogger eventLogger) + public TsFileCollectionImporter(IScraperStroage scraperStorage, DirectoryInfo rootDirectory) { ScraperStorage = scraperStorage; RootDirectory = rootDirectory; - EventLogger = eventLogger; } public void Run() @@ -48,7 +46,7 @@ namespace skyscraper5.Skyscraper.RecordingImporter if (ScraperStorage.ImportFileKnown(fi)) return; - SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), EventLogger, ScraperStorage); + SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), ScraperStorage); skyscraper.InitalizeFilterChain(); FileStream fileStream = fi.OpenRead(); diff --git a/skyscraper8/Skyscraper/Scraper/ChildEventLogger.cs b/skyscraper8/Skyscraper/Scraper/ChildEventLogger.cs deleted file mode 100644 index 9ce6ed0..0000000 --- a/skyscraper8/Skyscraper/Scraper/ChildEventLogger.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace skyscraper5.Skyscraper.Scraper -{ - internal class ChildEventLogger : ISkyscraperEventLogger - { - public ISkyscraperEventLogger Wrapped { get; } - public string ChildName { get; } - - public ChildEventLogger(ISkyscraperEventLogger wrapped, string childName) - { - Wrapped = wrapped; - ChildName = childName; - } - - public void Log(TimeSpan duration, DateTime eventTimestamp, SkyscraperContextEvent eventType, string name = null) - { - if (name == null) - name = String.Empty; - Wrapped.Log(duration, eventTimestamp, eventType, String.Format("<{0}> {1}", ChildName, name)); - NumEvents++; - } - - public ulong NumEvents { get; private set; } - } -} diff --git a/skyscraper8/Skyscraper/Scraper/ConsoleEventLogger.cs b/skyscraper8/Skyscraper/Scraper/ConsoleEventLogger.cs deleted file mode 100644 index 3cc47c7..0000000 --- a/skyscraper8/Skyscraper/Scraper/ConsoleEventLogger.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Text; - -namespace skyscraper5.Skyscraper.Scraper -{ - internal class ConsoleEventLogger : ISkyscraperEventLogger - { - public void Log(TimeSpan duration, DateTime eventTimestamp, SkyscraperContextEvent eventType, string name = null) - { - StringBuilder sb = new StringBuilder(); - sb.AppendFormat("[{0} {1}] ", eventTimestamp.ToShortDateString(), eventTimestamp.ToShortTimeString()); - sb.AppendFormat("{0}",eventType.ToString()); - - if (!string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(name)) - { - sb.AppendFormat(" ({0}) ", name); - } - - - if (eventType != SkyscraperContextEvent.StartPacketProcessing && eventType != SkyscraperContextEvent.TdtTime && eventType != SkyscraperContextEvent.TotTime) - { - sb.AppendFormat(" ({0} ms)", duration.TotalMilliseconds); - } - - Console.WriteLine(sb.ToString()); - NumEvents++; - } - - public ulong NumEvents { get; private set; } - } -} diff --git a/skyscraper8/Skyscraper/Scraper/FrameGrabber/ffmpegFrameGrabber.cs b/skyscraper8/Skyscraper/Scraper/FrameGrabber/ffmpegFrameGrabber.cs index 5bc5c78..769d3d1 100644 --- a/skyscraper8/Skyscraper/Scraper/FrameGrabber/ffmpegFrameGrabber.cs +++ b/skyscraper8/Skyscraper/Scraper/FrameGrabber/ffmpegFrameGrabber.cs @@ -1,10 +1,11 @@ -using System; +using log4net; +using skyscraper5.Mpeg2; +using skyscraper5.Mpeg2.Psi.Model; +using System; using System.Diagnostics; using System.IO; using System.Net; using System.Threading; -using skyscraper5.Mpeg2; -using skyscraper5.Mpeg2.Psi.Model; namespace skyscraper5.Skyscraper.Scraper.FrameGrabber { @@ -12,7 +13,9 @@ namespace skyscraper5.Skyscraper.Scraper.FrameGrabber internal class ffmpegFrameGrabber : ITsPacketProcessor { - public int CurrentNetworkId { get; } + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + + public int CurrentNetworkId { get; } public int TransportStreamId { get; } public ProgramMapping Mapping { get; } public ProgramMappingStream MappingStream { get; } @@ -128,7 +131,7 @@ namespace skyscraper5.Skyscraper.Scraper.FrameGrabber if (startable.HasValue) return startable.Value; - Console.WriteLine("Testing for a working ffmpeg..."); + logger.Info("Testing for a working ffmpeg..."); Process ffmpeg = new Process(); ffmpeg.StartInfo.FileName = "ffmpeg"; ffmpeg.StartInfo.UseShellExecute = false; @@ -141,7 +144,7 @@ namespace skyscraper5.Skyscraper.Scraper.FrameGrabber } catch (Exception e) { - Console.WriteLine(e.Message); + logger.Warn("ffmpeg doesn't work.", e); startable = false; return false; } @@ -149,14 +152,14 @@ namespace skyscraper5.Skyscraper.Scraper.FrameGrabber string line = ffmpeg.StandardError.ReadLine(); if (line.StartsWith("ffmpeg version 2") || line.StartsWith("ffmpeg version 5.")) { - Console.WriteLine("Found {0}", line); + logger.Info(String.Format("Found {0}", line)); startable = true; return true; } int exitCode = ffmpeg.ExitCode; - Debug.WriteLine("ffmpeg exit code {0}", exitCode); + logger.DebugFormat("ffmpeg exit code {0}", exitCode); startable = false; - Console.WriteLine(new string('_', Console.WindowWidth)); + return false; } } diff --git a/skyscraper8/Skyscraper/Scraper/ISkyscraperEventLogger.cs b/skyscraper8/Skyscraper/Scraper/ISkyscraperEventLogger.cs deleted file mode 100644 index 5b2e59b..0000000 --- a/skyscraper8/Skyscraper/Scraper/ISkyscraperEventLogger.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; - -namespace skyscraper5.Skyscraper.Scraper -{ - public interface ISkyscraperEventLogger - { - void Log(TimeSpan duration, DateTime eventTimestamp, SkyscraperContextEvent eventType, string name = null); - ulong NumEvents { get; } - } -} diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 22c1242..d0e385a 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -1,12 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.NetworkInformation; -using System.Text; +using log4net; using skyscraper5.Abertis; using skyscraper5.Docsis; using skyscraper5.Docsis.MacManagement; @@ -64,8 +56,18 @@ using skyscraper5.T2MI.Packets.AdressingFunctions; using skyscraper5.Teletext; using skyscraper5.Teletext.Vps; using skyscraper5.Teletext.Wss; +using skyscraper8.DvbI; using skyscraper8.Ses; using skyscraper8.yo3explorer; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.NetworkInformation; +using System.Text; using Tsubasa.IO; using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform; using RntParser = skyscraper5.Dvb.TvAnytime.RntParser; @@ -83,9 +85,9 @@ namespace skyscraper5.Skyscraper.Scraper public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true; public const bool ALLOW_FFMPEG_FRAMEGRABBER = true; public const bool ENABLE_MPE_TO_PCAP = true; - - public TsContext DvbContext { get; } - public ISkyscraperEventLogger EventLogger { get; } + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + + public TsContext DvbContext { get; } public IScraperStroage ScraperStorage { get; } public bool SourceIsDisk { get; set; } @@ -96,8 +98,7 @@ namespace skyscraper5.Skyscraper.Scraper public PsiDecoder Pid0x11Decoder { get; private set; } public PsiDecoder Pid0x12Decoder { get; private set; } public IPsiDecoderTransformer PmtDecoderTransformer { get; set; } - public SkyscraperContext(TsContext dvbContext, ISkyscraperEventLogger eventLogger = null, - IScraperStroage scraperStorage = null, int skipAtStart = 0, int? currentNetworkId = null, int? currentTransportStreamId = null) + public SkyscraperContext(TsContext dvbContext, IScraperStroage scraperStorage = null, int skipAtStart = 0, int? currentNetworkId = null, int? currentTransportStreamId = null) { TsDescriptorUnpacker descriptorUnpacker = TsDescriptorUnpacker.GetInstance(); for (byte i = 0x80; i < 0xfe; i++) @@ -108,9 +109,7 @@ namespace skyscraper5.Skyscraper.Scraper descriptorUnpacker.SetUserDefined(0xfe, true); DvbContext = dvbContext; - EventLogger = eventLogger; - if (EventLogger == null) - EventLogger = new ConsoleEventLogger(); + //PAT PatDecoder = new PsiDecoder(0, new PatParser(this)); @@ -209,7 +208,7 @@ namespace skyscraper5.Skyscraper.Scraper } } stringBuilder.Append(" -> OUT"); - Console.WriteLine("Built filter chain: {0}", stringBuilder.ToString()); + logger.DebugFormat("Built filter chain: {0}", stringBuilder.ToString()); } public void IngestFromStream(Stream stream) @@ -368,15 +367,16 @@ namespace skyscraper5.Skyscraper.Scraper [DebuggerStepThrough] private void LogEvent(SkyscraperContextEvent eventType, string name = null) { - DateTime eventTimestamp = DateTime.Now; - TimeSpan duration = eventTimestamp - lastEventTimestamp; - EventLogger.Log(duration, eventTimestamp, eventType, name); + string loggerName = String.Format("{0}{1}", IsChild ? ChildName + ", " : "", eventType.ToString()); + LogManager.GetLogger(loggerName).Info(name); if (eventType != SkyscraperContextEvent.TdtTime && eventType != SkyscraperContextEvent.TotTime && eventType != SkyscraperContextEvent.Scte35TimeSignal) { - lastEventTimestamp = eventTimestamp; - } + lastEventTimestamp = DateTime.Now; + } + EventsLogged++; } + public ulong EventsLogged { get; private set; } private DateTime lastEventTimestamp; public void NetworkPidFromPat(int networkPid) @@ -1851,15 +1851,18 @@ namespace skyscraper5.Skyscraper.Scraper } if (childSkyscrapers[pid][basebandFramePlpId] == null) { - childSkyscrapers[pid][basebandFramePlpId] = new SkyscraperContext(new TsContext(), new ChildEventLogger(EventLogger, String.Format("PLP {0}", basebandFramePlpId)), ScraperStorage); + childSkyscrapers[pid][basebandFramePlpId] = new SkyscraperContext(new TsContext(), ScraperStorage); childSkyscrapers[pid][basebandFramePlpId].IsChild = true; - childSkyscrapers[pid][basebandFramePlpId].InitalizeFilterChain(); + childSkyscrapers[pid][basebandFramePlpId].ChildName = String.Format("PLP {0}", basebandFramePlpId); + + childSkyscrapers[pid][basebandFramePlpId].InitalizeFilterChain(); LogEvent(SkyscraperContextEvent.T2MiPlpDetected, String.Format("PID {0:X4}, PLP {1}", pid, basebandFramePlpId)); } childSkyscrapers[pid][basebandFramePlpId].IngestSinglePacket(basebandPacket); } + public string ChildName { get; private set; } public bool IsChild { get; set; } public void OnT2MiTimestamp(int pid, _0x20_DvbT2Timestamp t2Timestamp) @@ -1883,7 +1886,7 @@ namespace skyscraper5.Skyscraper.Scraper if (skyscrapers[i] == null) continue; - if (skyscrapers[i].EventLogger.NumEvents >= 2) + if (skyscrapers[i].EventsLogged >= 2) return true; } @@ -2162,7 +2165,11 @@ namespace skyscraper5.Skyscraper.Scraper if (childSkyscrapers[pid] == null) childSkyscrapers[pid] = new SkyscraperContext[1]; if (childSkyscrapers[pid][0] == null) - childSkyscrapers[pid][0] = new SkyscraperContext(new TsContext(), new ChildEventLogger(EventLogger, String.Format("Abertis Tunnel PID {0:X4}",pid)), ScraperStorage); + { + childSkyscrapers[pid][0] = new SkyscraperContext(new TsContext(), ScraperStorage); + childSkyscrapers[pid][0].IsChild = true; + childSkyscrapers[pid][0].ChildName = String.Format("Abertis Tunnel on PID {0:X4}", pid); + } childSkyscrapers[pid][0].IngestSinglePacket(outBuffer); } @@ -2418,9 +2425,17 @@ namespace skyscraper5.Skyscraper.Scraper public void ProcessVirtualFilesystem(IFilesystem vfs, ProgramMappingStream programMappingStream) { - yo3explorerState state = yo3explorerState.GetInstance(); - yo3explorerToSkyscraperContextBridge context = new yo3explorerToSkyscraperContextBridge(); + if (!currentTime.HasValue) + return; + context.CurrentTime = currentTime.Value; + context.Source = ServiceListEntryPointSource.TransportStream; + context.OriginalNetworkId = CurrentNetworkId; + context.TransportStreamId = CurrentTransportStreamId; + context.PID = programMappingStream.ElementaryPid; + context.DataStorage = ScraperStorage; + + yo3explorerState state = yo3explorerState.GetInstance(); PluginManager pluginManager = PluginManager.GetInstance(); IReadOnlyList processors = pluginManager.GetFilesystemProcessors(); diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContextFactory.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContextFactory.cs index 58c1f53..4bdfcf1 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContextFactory.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContextFactory.cs @@ -15,10 +15,10 @@ namespace skyscraper5.Skyscraper.Scraper } - public static ISkyscraperContext CreateSkyscraper(ISkyscraperEventLogger eventLogger, IScraperStroage storage) + public static ISkyscraperContext CreateSkyscraper(IScraperStroage storage) { TsContext tsContext = new TsContext(); - SkyscraperContext result = new SkyscraperContext(tsContext, eventLogger, storage); + SkyscraperContext result = new SkyscraperContext(tsContext, storage); return result; } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs index d7fa446..bd65f0e 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs @@ -1427,5 +1427,20 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem { throw new NotImplementedException(); } + + public DateTime GetLastServiceListEntryPointUpdateDate(long sourceHash) + { + throw new NotImplementedException(); + } + + public void InsertDvbiServiceListEntryPoint(long sourceHash) + { + throw new NotImplementedException(); + } + + public bool TestForServiceListEntryPoints(long sourceHash) + { + throw new NotImplementedException(); + } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs index 3b2dc77..cea4eca 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/IScraperStorage.cs @@ -28,12 +28,13 @@ using skyscraper5.src.InteractionChannel.Model.Descriptors; using skyscraper5.src.Skyscraper.FrequencyListGenerator; using skyscraper5.src.Skyscraper.Scraper.Dns; using skyscraper5.Teletext; +using skyscraper8.DvbI; using skyscraper8.Ses; using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform; namespace skyscraper5.Skyscraper.Scraper.Storage { - public interface IScraperStroage : DataStorage, ObjectStorage, IDbBlindscanJobStorage, IDnsDataSource + public interface IScraperStroage : DataStorage, ObjectStorage, IDbBlindscanJobStorage, IDnsDataSource, DvbIDataStorage { bool TestForNitNetwork(NitNetwork nitNetwork); void StoreNitNetwork(NitNetwork nitNetwork); diff --git a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs index de40da3..b3c5d61 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs @@ -1289,5 +1289,34 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory sgtServices.Add(child); } + + public DateTime GetLastServiceListEntryPointUpdateDate(long sourceHash) + { + throw new NotImplementedException(); + } + + public void InsertDvbiServiceListEntryPoint(long sourceHash) + { + if (_serviceListEntryPointCoordinates == null) + _serviceListEntryPointCoordinates = new HashSet(); + + ServiceListEntryPointCoordinate newCoordinate = new ServiceListEntryPointCoordinate(); + newCoordinate.hash = sourceHash; + _serviceListEntryPointCoordinates.Add(newCoordinate); + } + + public bool TestForServiceListEntryPoints(long sourceHash) + { + if (_serviceListEntryPointCoordinates == null) + return false; + + throw new NotImplementedException(); + } + + private HashSet _serviceListEntryPointCoordinates; + class ServiceListEntryPointCoordinate + { + public long hash; + } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs index 9c3d142..6620f34 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Split/SplitScraperStorage.cs @@ -887,5 +887,20 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Split { dataStorage.InsertSgtService(child); } + + public DateTime GetLastServiceListEntryPointUpdateDate(long sourceHash) + { + throw new NotImplementedException(); + } + + public void InsertDvbiServiceListEntryPoint(long sourceHash) + { + throw new NotImplementedException(); + } + + public bool TestForServiceListEntryPoints(long sourceHash) + { + throw new NotImplementedException(); + } } } diff --git a/skyscraper8/log4net.config b/skyscraper8/log4net.config new file mode 100644 index 0000000..9755f52 --- /dev/null +++ b/skyscraper8/log4net.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/skyscraper8/skyscraper8.csproj b/skyscraper8/skyscraper8.csproj index aa764c0..8b531c1 100644 --- a/skyscraper8/skyscraper8.csproj +++ b/skyscraper8/skyscraper8.csproj @@ -10,6 +10,7 @@ + @@ -17,4 +18,10 @@ + + + PreserveNewest + + + diff --git a/skyscraper8/yo3explorer/yo3explorerToSkyscraperContextBridge.cs b/skyscraper8/yo3explorer/yo3explorerToSkyscraperContextBridge.cs index 02dd7c3..1d3eb58 100644 --- a/skyscraper8/yo3explorer/yo3explorerToSkyscraperContextBridge.cs +++ b/skyscraper8/yo3explorer/yo3explorerToSkyscraperContextBridge.cs @@ -1,4 +1,6 @@ -using System; +using skyscraper5.Skyscraper.Scraper.Storage; +using skyscraper8.DvbI; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -8,5 +10,29 @@ namespace skyscraper8.yo3explorer { internal class yo3explorerToSkyscraperContextBridge { + public DateTime CurrentTime { get; internal set; } + public ServiceListEntryPointSource Source { get; internal set; } + public int? OriginalNetworkId { get; internal set; } + public int? TransportStreamId { get; internal set; } + public int PID { get; internal set; } + public IScraperStroage DataStorage { get; internal set; } + + public long GetSourceHash() + { + long result = (int)Source; + switch (Source) + { + case ServiceListEntryPointSource.TransportStream: + result <<= 4; + result += OriginalNetworkId.Value; + result <<= 16; + result += TransportStreamId.Value; + result <<= 16; + result += PID; + return result; + default: + throw new NotImplementedException(Source.ToString()); + } + } } }