diff --git a/skyscraper8.sln.DotSettings.user b/skyscraper8.sln.DotSettings.user new file mode 100644 index 0000000..d3202ca --- /dev/null +++ b/skyscraper8.sln.DotSettings.user @@ -0,0 +1,2 @@ + + <data><HostParameters type="LocalHostParameters" /><Argument type="StandaloneArgument"><Arguments IsNull="False"></Arguments><FileName IsNull="False"></FileName><WorkingDirectory IsNull="False"></WorkingDirectory><Scope><ProcessFilters /></Scope></Argument><Info type="PerformanceInfo"><MeasureType>Sampling</MeasureType><MeterKind>Rdtsc</MeterKind><InjectInfo><SymbolSearch><SearchPaths /></SymbolSearch><Scope><PatternFilters /><DenyAttributeFilters /></Scope></InjectInfo></Info><CoreOptions type="CoreOptions"><CoreTempPath IsNull="False"></CoreTempPath><RemoteEndPoint IsNull="False"></RemoteEndPoint><AdditionalEnvironmentVariables /></CoreOptions><HostOptions type="HostOptions"><HostTempPath IsNull="False"></HostTempPath></HostOptions></data> \ No newline at end of file diff --git a/skyscraper8/DvbNip/DvbNipEventHandler.cs b/skyscraper8/DvbNip/DvbNipEventHandler.cs index 7b200ac..ba8e640 100644 --- a/skyscraper8/DvbNip/DvbNipEventHandler.cs +++ b/skyscraper8/DvbNip/DvbNipEventHandler.cs @@ -10,6 +10,8 @@ namespace skyscraper8.DvbNip internal interface IDvbNipEventHandler { void FluteFileArrival(NipActualCarrierInformation currentCarrierInformation, FluteListener fluteListener); + void FluteFileDownloadProgress(NipActualCarrierInformation currentCarrierInformation, ulong destinationTsi, ulong destinationToi, double progress, FileType fileAssociation); void OnMulticastGatewayConfiguration(NipActualCarrierInformation currentCarrierInformation, MulticastGatewayConfigurationType multicastGatewayConfiguration); + void OnNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation); } } diff --git a/skyscraper8/DvbNip/DvbNipReceiver.cs b/skyscraper8/DvbNip/DvbNipReceiver.cs index 604b724..1cfe628 100644 --- a/skyscraper8/DvbNip/DvbNipReceiver.cs +++ b/skyscraper8/DvbNip/DvbNipReceiver.cs @@ -30,10 +30,32 @@ namespace skyscraper8.DvbNip { } + private bool bootstrapped; private uint fluteHits, fluteMisses; private List flutes = new List(); + private static IPAddress dvbServiceDiscovery = IPAddress.Parse("224.0.23.14"); + public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) { + if (!bootstrapped) + { + bool isDvbServiceDiscovery = internetHeader.DestinationAddress.Equals(dvbServiceDiscovery); + if (isDvbServiceDiscovery) + { + UserDatagram discoveryUdpPacket = new UserDatagram(ipv4Packet); + if (discoveryUdpPacket.DestinationPort == 3937) + { + LctFrame discoveryLctFrame = new LctFrame(discoveryUdpPacket.Payload); + if (discoveryLctFrame.LctHeader.NipActualCarrierInformation != null) + { + CurrentCarrierInformation = discoveryLctFrame.LctHeader.NipActualCarrierInformation; + EventHandler.OnNipCarrierDetected(CurrentCarrierInformation); + bootstrapped = true; + } + } + } + return; + } UserDatagram udpPacket = new UserDatagram(ipv4Packet); LctFrame lctFrame = new LctFrame(udpPacket.Payload); @@ -75,9 +97,15 @@ namespace skyscraper8.DvbNip CurrentCarrierInformation = fluteListener.CarrierInformation; return; } + Stream fluteStream = fluteListener.ToStream(); - FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream); - SetFileAssociations(fluteListener, fdtAnnouncement); + bool isValidXml = FluteUtilities.IsXmlWellFormed(fluteStream); + fluteStream.Position = 0; + if (isValidXml) + { + FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream); + SetFileAssociations(fluteListener, fdtAnnouncement); + } fluteStream.Close(); fluteStream.Dispose(); flutes.Remove(fluteListener); @@ -105,12 +133,27 @@ namespace skyscraper8.DvbNip } return; } + else + { + if (fluteListener.DestinationToi != 0) + { + double progress = fluteListener.DownloadProgress; + if (progress > fluteListener.LastReportedProgress) + { + EventHandler.FluteFileDownloadProgress(CurrentCarrierInformation, fluteListener.DestinationTsi, fluteListener.DestinationToi, progress, fluteListener.FileAssociation); + fluteListener.LastReportedProgress = progress; + } + } + } } private void SetFileAssociations(FluteListener sourceListener, FDTInstanceType fdtAnnouncement) { foreach(FileType announcedFile in fdtAnnouncement.File) { + if (string.IsNullOrEmpty(announcedFile.TOI)) + continue; + ulong targetToi = ulong.Parse(announcedFile.TOI); FluteListener? targetListener = flutes.Find(x => x.DestinationAddress.Equals(sourceListener.DestinationAddress) && diff --git a/skyscraper8/DvbNip/DvbNipUtilities.cs b/skyscraper8/DvbNip/DvbNipUtilities.cs index f11e52d..345ec27 100644 --- a/skyscraper8/DvbNip/DvbNipUtilities.cs +++ b/skyscraper8/DvbNip/DvbNipUtilities.cs @@ -38,5 +38,18 @@ namespace skyscraper8.DvbNip return newFilename; } + + public static bool IsContinuousFileType(string extension) + { + switch(extension) + { + case ".mpd": + case ".m4s": + case ".m3u8": + return true; + default: + return false; + } + } } } diff --git a/skyscraper8/Ietf/FLUTE/ContentEncodingOfFdtInstance.cs b/skyscraper8/Ietf/FLUTE/ContentEncodingOfFdtInstance.cs new file mode 100644 index 0000000..b84a35a --- /dev/null +++ b/skyscraper8/Ietf/FLUTE/ContentEncodingOfFdtInstance.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Ietf.FLUTE +{ + internal class ContentEncodingOfFdtInstance + { + public ContentEncodingOfFdtInstance(uint fixedHeaderExtension) + { + fixedHeaderExtension &= 0x00ff0000; + fixedHeaderExtension >>= 16; + ContentEncodingAlgorithm = fixedHeaderExtension; + } + + public uint ContentEncodingAlgorithm { get; } + } +} diff --git a/skyscraper8/Ietf/FLUTE/FluteListener.cs b/skyscraper8/Ietf/FLUTE/FluteListener.cs index acf90ea..7fb79a5 100644 --- a/skyscraper8/Ietf/FLUTE/FluteListener.cs +++ b/skyscraper8/Ietf/FLUTE/FluteListener.cs @@ -52,28 +52,44 @@ namespace skyscraper8.Ietf.FLUTE CarrierInformation = lctFrame.LctHeader.NipActualCarrierInformation; return; } - else - { - throw new NotImplementedException("non fec"); - } } - + if (transferLength == 0) + { + if (lctFrame.LctHeader.FecObjectTransmissionInformation == null) + { + return; + } transferLength = lctFrame.LctHeader.FecObjectTransmissionInformation.TransferLength; + } if (blocks == null) blocks = new List(); ushort sbn = lctFrame.FecHeader.SourceBlockNumber; ushort esi = lctFrame.FecHeader.EncodingSymbolId; - FluteBlock? fluteBlock = blocks.Find(x => - x.SourceBlockNumer == sbn && - x.EncodingSymbolId == esi); + //FluteBlock? fluteBlock = blocks.Find(x => + //x.SourceBlockNumer == sbn && + //x.EncodingSymbolId == esi); + + FluteBlock fluteBlock = null; + foreach (FluteBlock candidateBlock in blocks) + { + if (candidateBlock.SourceBlockNumer == sbn) + { + if (candidateBlock.EncodingSymbolId == esi) + { + fluteBlock = candidateBlock; + break; + } + } + } if (fluteBlock == null) { fluteBlock = new FluteBlock(sbn, esi, lctFrame.Payload); blocks.Add(fluteBlock); + _dataWritten += (uint)fluteBlock.Payload.Length; } else { @@ -96,14 +112,18 @@ namespace skyscraper8.Ietf.FLUTE return DataWritten >= transferLength; } + private ulong _dataWritten; public ulong DataWritten { get { - ulong currentAmount = 0; - foreach (FluteBlock block in blocks) - currentAmount += (uint)block.Payload.Length; - return currentAmount; + if (blocks == null) + return 0; + + if (_disabled) + return 0; + + return _dataWritten; } } @@ -128,6 +148,7 @@ namespace skyscraper8.Ietf.FLUTE switch(FileAssociation.ContentEncoding) { case null: + case "null": break; case "gzip": GZipStream level2 = new GZipStream(level1, CompressionMode.Decompress, false); @@ -166,9 +187,9 @@ namespace skyscraper8.Ietf.FLUTE Payload = payload; } - public ushort SourceBlockNumer { get; } - public ushort EncodingSymbolId { get; } - public byte[] Payload { get; } + public ushort SourceBlockNumer; + public ushort EncodingSymbolId; + public byte[] Payload; public override string ToString() { @@ -176,7 +197,25 @@ namespace skyscraper8.Ietf.FLUTE } } - public FileType FileAssociation { get; set; } + private FileType _fileAssocitation; + public FileType FileAssociation + { + get + { + return _fileAssocitation; + } + set + { + _fileAssocitation = value; + if (_fileAssocitation.ContentLengthSpecified) + { + if (transferLength == 0) + { + transferLength = _fileAssocitation.ContentLength; + } + } + } + } public NipActualCarrierInformation CarrierInformation { get; private set; } @@ -198,5 +237,22 @@ namespace skyscraper8.Ietf.FLUTE _disabled = value; } } + + public double DownloadProgress + { + get + { + double w = (double)DataWritten; + if (w == 0) + return 0; + + double g = (double)transferLength; + double result = (w * 100.0) / g; + result = Math.Round(result, 1); + return result; + } + } + + public double LastReportedProgress { get; internal set; } } } diff --git a/skyscraper8/Ietf/FLUTE/FluteUtilities.cs b/skyscraper8/Ietf/FLUTE/FluteUtilities.cs index b0cc8ba..b20cc98 100644 --- a/skyscraper8/Ietf/FLUTE/FluteUtilities.cs +++ b/skyscraper8/Ietf/FLUTE/FluteUtilities.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using System.Xml; using System.Xml.Serialization; namespace skyscraper8.Ietf.FLUTE @@ -28,5 +29,29 @@ namespace skyscraper8.Ietf.FLUTE FDTInstanceType result = (FDTInstanceType)fdtSerializer.Deserialize(ms); return result; } + + public static bool IsXmlWellFormed(Stream xmlStream) + { + try + { + var settings = new XmlReaderSettings + { + ConformanceLevel = ConformanceLevel.Document, + DtdProcessing = DtdProcessing.Ignore, + IgnoreComments = true, + IgnoreWhitespace = true + }; + + XmlReader xmlReader = XmlReader.Create(xmlStream, settings); + + while (xmlReader.Read()) { } // Just read through the document + return true; + } + catch (XmlException) + { + return false; + } + } + } } diff --git a/skyscraper8/Ietf/FLUTE/LctHeader.cs b/skyscraper8/Ietf/FLUTE/LctHeader.cs index 0a4a0ad..c43f520 100644 --- a/skyscraper8/Ietf/FLUTE/LctHeader.cs +++ b/skyscraper8/Ietf/FLUTE/LctHeader.cs @@ -45,6 +45,9 @@ namespace skyscraper8.Ietf.FLUTE case 192: this.FdtInstanceId = new FdtInstanceHeader(fixedHeaderExtension); break; + case 193: + this.ContentEncoding = new ContentEncodingOfFdtInstance(fixedHeaderExtension); + break; default: throw new NotImplementedException(String.Format("LCT Header Extension {0}", extensionId)); } @@ -60,6 +63,9 @@ namespace skyscraper8.Ietf.FLUTE { case 0: break; + case 2: + this.TimeExtenstion = new TimeHeaderExtension(extensionBuffer); + break; case 64: this.FecObjectTransmissionInformation = new FecObjectTransmissionInformation(extensionBuffer); break; @@ -96,5 +102,7 @@ namespace skyscraper8.Ietf.FLUTE public FdtInstanceHeader FdtInstanceId { get; } public NipActualCarrierInformation NipActualCarrierInformation { get; } public FecObjectTransmissionInformation FecObjectTransmissionInformation { get; } + public TimeHeaderExtension TimeExtenstion { get; } + public ContentEncodingOfFdtInstance ContentEncoding { get; } } } diff --git a/skyscraper8/Ietf/FLUTE/TimeHeaderExtension.cs b/skyscraper8/Ietf/FLUTE/TimeHeaderExtension.cs new file mode 100644 index 0000000..c8d0180 --- /dev/null +++ b/skyscraper8/Ietf/FLUTE/TimeHeaderExtension.cs @@ -0,0 +1,50 @@ +using skyscraper5.Skyscraper.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Ietf.FLUTE +{ + internal class TimeHeaderExtension + { + private const double FACTOR = 1L << 32; + private static readonly DateTime epoch = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc) + TimeSpan.FromSeconds(1L << 32); + + public TimeHeaderExtension(byte[] buffer) + { + MemoryStream ms = new MemoryStream(buffer, false); + byte v = ms.ReadUInt8(); + bool SctHiFlag = (v & 0x80) != 0; + bool SctLowFlag = (v & 0x40) != 0; + bool ErtFlag = (v & 0x20) != 0; + bool SlcFlag = (v & 0x10) != 0; + PiSpecificUse = ms.ReadUInt8(); + + uint sctHi = 0; + if (SctHiFlag) + sctHi = ms.ReadUInt32BE(); + + uint sctLow = 0; + if (SctLowFlag) + sctLow = ms.ReadUInt32BE(); + + long sct = sctHi; + sct <<= 32; + sct += sctLow; + this.Sct = epoch + TimeSpan.FromSeconds(sct / FACTOR); + + if (ErtFlag) + Ert = ms.ReadUInt32BE(); + + if (SlcFlag) + Slc = ms.ReadUInt32BE(); + } + + public byte PiSpecificUse { get; } + public DateTime Sct { get; } + public uint Ert { get; } + public uint Slc { get; } + } +} diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index ccb8be3..e9429fe 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "file-live \"C:\\Temp\\dvbnip-000000.ts\"", + "commandLineArgs": "\"E:\\NIP-Research\\nip.m3u8\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 8d359f9..0d2c00b 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -87,7 +87,7 @@ namespace skyscraper5.Skyscraper.Scraper { public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true; public const bool ALLOW_FFMPEG_FRAMEGRABBER = true; - public const bool ENABLE_MPE_TO_PCAP = true; + public const bool ENABLE_MPE_TO_PCAP = false; private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); public TsContext DvbContext { get; } @@ -2454,7 +2454,9 @@ namespace skyscraper5.Skyscraper.Scraper FileInfo fileInfo = new FileInfo(filename); if (!fileInfo.Exists) { - LogEvent(SkyscraperContextEvent.FluteFileArrival, listener.FileAssociation.ContentLocation); + string extension = Path.GetExtension(fileInfo.Name).ToLowerInvariant(); + if (!DvbNipUtilities.IsContinuousFileType(extension)) + LogEvent(SkyscraperContextEvent.FluteFileArrival, listener.FileAssociation.ContentLocation); fileInfo.Directory.EnsureExists(); listener.WriteToFile(fileInfo.FullName); } @@ -2464,5 +2466,22 @@ namespace skyscraper5.Skyscraper.Scraper { } + + public void OnNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation) + { + LogEvent(SkyscraperContextEvent.NipCarrierDetected, String.Format("{0}", currentCarrierInformation.NipStreamProviderName)); + } + + public void FluteFileDownloadProgress(NipActualCarrierInformation currentCarrierInformation, ulong destinationTsi, ulong destinationToi, double progress, FileType filetype) + { + if (filetype != null) + { + string extension = Path.GetExtension(filetype.ContentLocation).ToLowerInvariant(); + if (!DvbNipUtilities.IsContinuousFileType(extension)) + { + LogEvent(SkyscraperContextEvent.FluteFileProgress, String.Format("Filename={0}, {1}%", filetype.ContentLocation, progress)); + } + } + } } } diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs index 730ece5..3e727e8 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs @@ -73,6 +73,8 @@ TimNetworkLayerInfo, SgtList, SgtService, - FluteFileArrival + FluteFileArrival, + NipCarrierDetected, + FluteFileProgress } }