Minor performance improvement on FLUTE.

skyscraper8.sln.DotSettings.user

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/Profiling/Configurations/=1/@EntryIndexedValue">&lt;data&gt;&lt;HostParameters type="LocalHostParameters" /&gt;&lt;Argument type="StandaloneArgument"&gt;&lt;Arguments IsNull="False"&gt;&lt;/Arguments&gt;&lt;FileName IsNull="False"&gt;&lt;/FileName&gt;&lt;WorkingDirectory IsNull="False"&gt;&lt;/WorkingDirectory&gt;&lt;Scope&gt;&lt;ProcessFilters /&gt;&lt;/Scope&gt;&lt;/Argument&gt;&lt;Info type="PerformanceInfo"&gt;&lt;MeasureType&gt;Sampling&lt;/MeasureType&gt;&lt;MeterKind&gt;Rdtsc&lt;/MeterKind&gt;&lt;InjectInfo&gt;&lt;SymbolSearch&gt;&lt;SearchPaths /&gt;&lt;/SymbolSearch&gt;&lt;Scope&gt;&lt;PatternFilters /&gt;&lt;DenyAttributeFilters /&gt;&lt;/Scope&gt;&lt;/InjectInfo&gt;&lt;/Info&gt;&lt;CoreOptions type="CoreOptions"&gt;&lt;CoreTempPath IsNull="False"&gt;&lt;/CoreTempPath&gt;&lt;RemoteEndPoint IsNull="False"&gt;&lt;/RemoteEndPoint&gt;&lt;AdditionalEnvironmentVariables /&gt;&lt;/CoreOptions&gt;&lt;HostOptions type="HostOptions"&gt;&lt;HostTempPath IsNull="False"&gt;&lt;/HostTempPath&gt;&lt;/HostOptions&gt;&lt;/data&gt;</s:String></wpf:ResourceDictionary>

skyscraper8/DvbNip/DvbNipEventHandler.cs

@@ -10,6 +10,8 @@ namespace skyscraper8.DvbNip
 	internal interface IDvbNipEventHandler
 	{
 		void FluteFileArrival(NipActualCarrierInformation currentCarrierInformation, FluteListener fluteListener);
+		void FluteFileDownloadProgress(NipActualCarrierInformation currentCarrierInformation, ulong destinationTsi, ulong destinationToi, double progress, FileType fileAssociation);
 		void OnMulticastGatewayConfiguration(NipActualCarrierInformation currentCarrierInformation, MulticastGatewayConfigurationType multicastGatewayConfiguration);
+		void OnNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation);
 	}
 }

skyscraper8/DvbNip/DvbNipReceiver.cs

@@ -30,10 +30,32 @@ namespace skyscraper8.DvbNip
 		{
 		}
 
+		private bool bootstrapped;
 		private uint fluteHits, fluteMisses;
 		private List<FluteListener> flutes = new List<FluteListener>();
+		private static IPAddress dvbServiceDiscovery = IPAddress.Parse("224.0.23.14");
+
 		public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet)
 		{
+			if (!bootstrapped)
+			{
+				bool isDvbServiceDiscovery = internetHeader.DestinationAddress.Equals(dvbServiceDiscovery);
+				if (isDvbServiceDiscovery)
+				{
+					UserDatagram discoveryUdpPacket = new UserDatagram(ipv4Packet);
+					if (discoveryUdpPacket.DestinationPort == 3937)
+					{
+						LctFrame discoveryLctFrame = new LctFrame(discoveryUdpPacket.Payload);
+						if (discoveryLctFrame.LctHeader.NipActualCarrierInformation != null)
+						{
+							CurrentCarrierInformation = discoveryLctFrame.LctHeader.NipActualCarrierInformation;
+							EventHandler.OnNipCarrierDetected(CurrentCarrierInformation);
+							bootstrapped = true;
+						}
+					}
+				}
+				return;
+			}
 			UserDatagram udpPacket = new UserDatagram(ipv4Packet);
 			LctFrame lctFrame = new LctFrame(udpPacket.Payload);
 
@@ -75,9 +97,15 @@ namespace skyscraper8.DvbNip
 						CurrentCarrierInformation = fluteListener.CarrierInformation;
 						return;
 					}
+
 					Stream fluteStream = fluteListener.ToStream();
-					FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream);
-					SetFileAssociations(fluteListener, fdtAnnouncement);
+					bool isValidXml = FluteUtilities.IsXmlWellFormed(fluteStream);
+					fluteStream.Position = 0;
+					if (isValidXml)
+					{
+						FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream);
+						SetFileAssociations(fluteListener, fdtAnnouncement);
+					}
 					fluteStream.Close();
 					fluteStream.Dispose();
 					flutes.Remove(fluteListener);
@@ -105,12 +133,27 @@ namespace skyscraper8.DvbNip
 				}
 				return;
 			}
+			else
+			{
+				if (fluteListener.DestinationToi != 0)
+				{
+					double progress = fluteListener.DownloadProgress;
+					if (progress > fluteListener.LastReportedProgress)
+					{
+						EventHandler.FluteFileDownloadProgress(CurrentCarrierInformation, fluteListener.DestinationTsi, fluteListener.DestinationToi, progress, fluteListener.FileAssociation);
+						fluteListener.LastReportedProgress = progress;
+					}
+				}
+			}
 		}
 
 		private void SetFileAssociations(FluteListener sourceListener, FDTInstanceType fdtAnnouncement)
 		{
 			foreach(FileType announcedFile in fdtAnnouncement.File)
 			{
+				if (string.IsNullOrEmpty(announcedFile.TOI))
+					continue;
+
 				ulong targetToi = ulong.Parse(announcedFile.TOI);
 				FluteListener? targetListener = flutes.Find(x =>
 					x.DestinationAddress.Equals(sourceListener.DestinationAddress) &&

skyscraper8/DvbNip/DvbNipUtilities.cs

@@ -38,5 +38,18 @@ namespace skyscraper8.DvbNip
 
 			return newFilename;
 		}
+
+		public static bool IsContinuousFileType(string extension)
+		{
+			switch(extension)
+			{
+				case ".mpd":
+				case ".m4s":
+				case ".m3u8":
+					return true;
+				default:
+					return false;
+			}
+		}
 	}
 }

skyscraper8/Ietf/FLUTE/ContentEncodingOfFdtInstance.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Ietf.FLUTE
+{
+	internal class ContentEncodingOfFdtInstance
+	{
+		public ContentEncodingOfFdtInstance(uint fixedHeaderExtension)
+		{
+			fixedHeaderExtension &= 0x00ff0000;
+			fixedHeaderExtension >>= 16;
+			ContentEncodingAlgorithm = fixedHeaderExtension;
+		}
+
+		public uint ContentEncodingAlgorithm { get; }
+	}
+}

skyscraper8/Ietf/FLUTE/FluteListener.cs

@@ -52,28 +52,44 @@ namespace skyscraper8.Ietf.FLUTE
 					CarrierInformation = lctFrame.LctHeader.NipActualCarrierInformation;
 					return;
 				}
-				else
-				{
-					throw new NotImplementedException("non fec");
-				}
 			}
-				
+
 
 			if (transferLength == 0)
+			{
+				if (lctFrame.LctHeader.FecObjectTransmissionInformation == null)
+				{
+					return;
+				}
 				transferLength = lctFrame.LctHeader.FecObjectTransmissionInformation.TransferLength;
+			}
 			if (blocks == null)
 				blocks = new List<FluteBlock>();
 
 			ushort sbn = lctFrame.FecHeader.SourceBlockNumber;
 			ushort esi = lctFrame.FecHeader.EncodingSymbolId;
 
-			FluteBlock? fluteBlock = blocks.Find(x =>
-				x.SourceBlockNumer == sbn &&
-				x.EncodingSymbolId == esi);
+			//FluteBlock? fluteBlock = blocks.Find(x =>
+				//x.SourceBlockNumer == sbn &&
+				//x.EncodingSymbolId == esi);
+
+			FluteBlock fluteBlock = null;
+			foreach (FluteBlock candidateBlock in blocks)
+			{
+				if (candidateBlock.SourceBlockNumer == sbn)
+				{
+					if (candidateBlock.EncodingSymbolId == esi)
+					{
+						fluteBlock = candidateBlock;
+						break;
+					}
+				}
+			}
 			if (fluteBlock == null)
 			{
 				fluteBlock = new FluteBlock(sbn, esi, lctFrame.Payload);
 				blocks.Add(fluteBlock);
+				_dataWritten += (uint)fluteBlock.Payload.Length;
 			}
 			else
 			{
@@ -96,14 +112,18 @@ namespace skyscraper8.Ietf.FLUTE
 			return DataWritten >= transferLength;
 		}
 
+		private ulong _dataWritten;
 		public ulong DataWritten
 		{
 			get
 			{
-				ulong currentAmount = 0;
-				foreach (FluteBlock block in blocks)
-					currentAmount += (uint)block.Payload.Length;
-				return currentAmount;
+				if (blocks == null)
+					return 0;
+
+				if (_disabled)
+					return 0;
+
+				return _dataWritten;
 			}
 		}
 
@@ -128,6 +148,7 @@ namespace skyscraper8.Ietf.FLUTE
 				switch(FileAssociation.ContentEncoding)
 				{
 					case null:
+					case "null":
 						break;
 					case "gzip":
 						GZipStream level2 = new GZipStream(level1, CompressionMode.Decompress, false);
@@ -166,9 +187,9 @@ namespace skyscraper8.Ietf.FLUTE
 				Payload = payload;
 			}
 
-			public ushort SourceBlockNumer { get; }
-			public ushort EncodingSymbolId { get; }
-			public byte[] Payload { get; }
+			public ushort SourceBlockNumer;
+			public ushort EncodingSymbolId;
+			public byte[] Payload;
 
 			public override string ToString()
 			{
@@ -176,7 +197,25 @@ namespace skyscraper8.Ietf.FLUTE
 			}
 		}
 
-		public FileType FileAssociation { get; set; }
+		private FileType _fileAssocitation;
+		public FileType FileAssociation
+		{
+			get
+			{
+				return _fileAssocitation;
+			}
+			set
+			{
+				_fileAssocitation = value;
+				if (_fileAssocitation.ContentLengthSpecified)
+				{
+					if (transferLength == 0)
+					{
+						transferLength = _fileAssocitation.ContentLength;
+					}
+				}
+			}
+		}
 
 		public NipActualCarrierInformation CarrierInformation { get; private set; }
 
@@ -198,5 +237,22 @@ namespace skyscraper8.Ietf.FLUTE
 				_disabled = value;
 			}
 		}
+
+		public double DownloadProgress
+		{
+			get
+			{
+				double w = (double)DataWritten;
+				if (w == 0)
+					return 0;
+
+				double g = (double)transferLength;
+				double result = (w * 100.0) / g;
+				result = Math.Round(result, 1);
+				return result;
+			}
+		}
+
+		public double LastReportedProgress { get; internal set; }
 	}
 }

skyscraper8/Ietf/FLUTE/FluteUtilities.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using System.Xml;
 using System.Xml.Serialization;
 
 namespace skyscraper8.Ietf.FLUTE
@@ -28,5 +29,29 @@ namespace skyscraper8.Ietf.FLUTE
 			FDTInstanceType result = (FDTInstanceType)fdtSerializer.Deserialize(ms);
 			return result;
 		}
+
+		public static bool IsXmlWellFormed(Stream xmlStream)
+		{
+			try
+			{
+				var settings = new XmlReaderSettings
+				{
+					ConformanceLevel = ConformanceLevel.Document,
+					DtdProcessing = DtdProcessing.Ignore,
+					IgnoreComments = true,
+					IgnoreWhitespace = true
+				};
+
+				XmlReader xmlReader = XmlReader.Create(xmlStream, settings);
+
+				while (xmlReader.Read()) { } // Just read through the document
+				return true;
+			}
+			catch (XmlException)
+			{
+				return false;
+			}
+		}
+
 	}
 }

skyscraper8/Ietf/FLUTE/LctHeader.cs

@@ -45,6 +45,9 @@ namespace skyscraper8.Ietf.FLUTE
 						case 192:
 							this.FdtInstanceId = new FdtInstanceHeader(fixedHeaderExtension);
 							break;
+						case 193:
+							this.ContentEncoding = new ContentEncodingOfFdtInstance(fixedHeaderExtension);
+							break;
 						default:
 							throw new NotImplementedException(String.Format("LCT Header Extension {0}", extensionId));
 					}
@@ -60,6 +63,9 @@ namespace skyscraper8.Ietf.FLUTE
 					{
 						case 0:
 							break;
+						case 2:
+							this.TimeExtenstion = new TimeHeaderExtension(extensionBuffer);
+							break;
 						case 64:
 							this.FecObjectTransmissionInformation = new FecObjectTransmissionInformation(extensionBuffer);
 							break;
@@ -96,5 +102,7 @@ namespace skyscraper8.Ietf.FLUTE
 		public FdtInstanceHeader FdtInstanceId { get; }
 		public NipActualCarrierInformation NipActualCarrierInformation { get; }
 		public FecObjectTransmissionInformation FecObjectTransmissionInformation { get; }
+		public TimeHeaderExtension TimeExtenstion { get; }
+		public ContentEncodingOfFdtInstance ContentEncoding { get; }
 	}
 }

skyscraper8/Ietf/FLUTE/TimeHeaderExtension.cs

@@ -0,0 +1,50 @@
+using skyscraper5.Skyscraper.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Ietf.FLUTE
+{
+	internal class TimeHeaderExtension
+	{
+		private const double FACTOR = 1L << 32;
+		private static readonly DateTime epoch = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc) + TimeSpan.FromSeconds(1L << 32);
+
+		public TimeHeaderExtension(byte[] buffer) 
+		{
+			MemoryStream ms = new MemoryStream(buffer, false);
+			byte v = ms.ReadUInt8();
+			bool SctHiFlag = (v & 0x80) != 0;
+			bool SctLowFlag = (v & 0x40) != 0;
+			bool ErtFlag = (v & 0x20) != 0;
+			bool SlcFlag = (v & 0x10) != 0;
+			PiSpecificUse = ms.ReadUInt8();
+
+			uint sctHi = 0;
+			if (SctHiFlag)
+				sctHi = ms.ReadUInt32BE();
+
+			uint sctLow = 0;
+			if (SctLowFlag)
+				sctLow = ms.ReadUInt32BE();
+
+			long sct = sctHi;
+			sct <<= 32;
+			sct += sctLow;
+			this.Sct = epoch + TimeSpan.FromSeconds(sct / FACTOR);
+
+			if (ErtFlag)
+				Ert = ms.ReadUInt32BE();
+
+			if (SlcFlag)
+				Slc = ms.ReadUInt32BE();
+		}
+
+		public byte PiSpecificUse { get; }
+		public DateTime Sct { get; }
+		public uint Ert { get; }
+		public uint Slc { get; }
+	}
+}

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "file-live \"C:\\Temp\\dvbnip-000000.ts\"",
+      "commandLineArgs": "\"E:\\NIP-Research\\nip.m3u8\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -87,7 +87,7 @@ namespace skyscraper5.Skyscraper.Scraper
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
         public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
-        public const bool ENABLE_MPE_TO_PCAP = true;
+        public const bool ENABLE_MPE_TO_PCAP = false;
 		private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
 
 		public TsContext DvbContext { get; }
@@ -2454,7 +2454,9 @@ namespace skyscraper5.Skyscraper.Scraper
             FileInfo fileInfo = new FileInfo(filename);
             if (!fileInfo.Exists)
             {
-				LogEvent(SkyscraperContextEvent.FluteFileArrival, listener.FileAssociation.ContentLocation);
+                string extension = Path.GetExtension(fileInfo.Name).ToLowerInvariant();
+                if (!DvbNipUtilities.IsContinuousFileType(extension))
+				    LogEvent(SkyscraperContextEvent.FluteFileArrival, listener.FileAssociation.ContentLocation);
                 fileInfo.Directory.EnsureExists();
                 listener.WriteToFile(fileInfo.FullName);
 			}
@@ -2464,5 +2466,22 @@ namespace skyscraper5.Skyscraper.Scraper
         {
 
         }
+
+		public void OnNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation)
+		{
+            LogEvent(SkyscraperContextEvent.NipCarrierDetected, String.Format("{0}", currentCarrierInformation.NipStreamProviderName));
+		}
+
+		public void FluteFileDownloadProgress(NipActualCarrierInformation currentCarrierInformation, ulong destinationTsi, ulong destinationToi, double progress, FileType filetype)
+		{
+            if (filetype != null)
+            {
+                string extension = Path.GetExtension(filetype.ContentLocation).ToLowerInvariant();
+                if (!DvbNipUtilities.IsContinuousFileType(extension))
+                {
+                    LogEvent(SkyscraperContextEvent.FluteFileProgress, String.Format("Filename={0}, {1}%", filetype.ContentLocation, progress));
+                }
+            }
+		}
 	}
 }

skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs

@@ -73,6 +73,8 @@
 		TimNetworkLayerInfo,
 		SgtList,
 		SgtService,
-		FluteFileArrival
+		FluteFileArrival,
+		NipCarrierDetected,
+		FluteFileProgress
 	}
 }