Read M3U8 Files from disk.

skyscraper8/Docsis/MacManagement/T5_V1_RangingResponse.cs

@@ -31,6 +31,16 @@ namespace skyscraper5.Docsis.MacManagement
                 positionFor1516 = ms.Position;
                 byte type = ms.ReadUInt8();
                 byte length = ms.ReadUInt8();
+                if (length > ms.GetAvailableBytes())
+                {
+                    Valid = false;
+                    return;
+                }
+                if (length == 0)
+                {
+                    Valid = false;
+                    return;
+                }
                 byte[] v = ms.ReadBytes(length);
                 switch (type)
                 {
@@ -38,6 +48,11 @@ namespace skyscraper5.Docsis.MacManagement
                         //reserved
                         break;
                     case 1:
+                        if (length < 4)
+                        {
+                            Valid = false;
+                            return;
+                        }
                         (v[0], v[1], v[2], v[3]) = (v[3], v[2], v[1], v[0]);
                         TimingAdjust = BitConverter.ToUInt32(v, 0);
                         break;
@@ -45,7 +60,12 @@ namespace skyscraper5.Docsis.MacManagement
                         PowerLevelAdjust = v[0];
                         break;
                     case 3:
-                        (v[0], v[1]) = (v[1], v[0]);
+						if (length < 2)
+						{
+							Valid = false;
+							return;
+						}
+						(v[0], v[1]) = (v[1], v[0]);
                         OffsetFrequencyAdjust = BitConverter.ToUInt16(v, 0);
                         break;
                     case 4:
@@ -55,6 +75,11 @@ namespace skyscraper5.Docsis.MacManagement
                         RangingStatus = (RangingStatusEnum)v[0];
                         break;
                     case 6:
+						if (length < 4)
+						{
+							Valid = false;
+							return;
+						}
 						(v[0], v[1], v[2], v[3]) = (v[3], v[2], v[1], v[0]);
 						DownstreamFrequencyOverride = BitConverter.ToUInt32(v, 0);
 						break;

skyscraper8/Program.cs

@@ -30,6 +30,7 @@ using skyscraper5.src.Skyscraper.FrequencyListGenerator;
 using skyscraper5.src.Sophtainer;
 using skyscraper5.T2MI;
 using skyscraper5.src.Mpeg2.PacketFilter;
+using skyscraper8.Skyscraper.IO;
 
 namespace skyscraper5
 {
@@ -55,6 +56,14 @@ namespace skyscraper5
                     return;
                 }
 
+                if (args[0].ToLowerInvariant().EndsWith(".m3u8"))
+                {
+                    M3U8Stream m3U8Stream = new M3U8Stream(args[0]);
+                    SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext());
+                    skyscraperContext.InitalizeFilterChain();
+                    skyscraperContext.IngestFromStream(m3U8Stream);
+                }
+
                 if (args[0].Equals("aactest"))
                 {
 	                new AacTestProgram().Run();

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "cscan \"tcp://172.20.20.202:6970\"",
+      "commandLineArgs": "\"D:\\DocsisDemo\\docsis.m3u8\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/IO/EmptyStream.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.IO
+{
+	internal class EmptyStream : Stream
+	{
+		public override bool CanRead => true;
+
+		public override bool CanSeek => true;
+
+		public override bool CanWrite => false;
+
+		public override long Length => 0;
+
+		public override long Position { 
+			get => 0; 
+			set
+			{
+				if (value != 0)
+					throw new ArgumentOutOfRangeException(nameof(value));
+			}
+		}
+
+		public override void Flush()
+		{
+		}
+
+		public override int Read(byte[] buffer, int offset, int count)
+		{
+			return 0;
+		}
+
+		public override long Seek(long offset, SeekOrigin origin)
+		{
+			if (offset != 0)
+				throw new ArgumentOutOfRangeException(nameof(offset));
+			return 0;
+		}
+
+		public override void SetLength(long value)
+		{
+			if (value != 0)
+				throw new ArgumentOutOfRangeException(nameof(value));
+		}
+
+		public override void Write(byte[] buffer, int offset, int count)
+		{
+
+		}
+	}
+}

skyscraper8/Skyscraper/IO/M3U8Stream.cs

@@ -0,0 +1,202 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.IO
+{
+	internal class M3U8Stream : Stream
+	{
+		public M3U8Stream(string source) 
+		{
+			this.segmentIndex = 0;
+			this.segmentNames = new string[0];
+			this.source = source;
+			this.finishedSegments = new List<string>();
+			this.innerStream = new EmptyStream();
+			this.sourceRoot = GetRoot(source);
+
+			if (source.StartsWith("http://") || source.StartsWith("https://"))
+			{
+				webClient = new HttpClient();
+				webClient.BaseAddress = new Uri(sourceRoot);
+			}
+			else if (File.Exists(source))
+			{
+				this.rootDirectory = new DirectoryInfo(sourceRoot);
+			}
+			else
+			{
+				throw new NotImplementedException(source);
+			}
+		}
+
+		private int segmentIndex;
+		private string[] segmentNames;
+		private string source;
+		private List<string> finishedSegments;
+		private Stream innerStream;
+		private string sourceRoot;
+
+		private HttpClient webClient;
+		private DirectoryInfo rootDirectory;
+
+		private string GetRoot(string source)
+		{
+			if (source.StartsWith("http://") || source.StartsWith("https://"))
+			{
+				while (!source.EndsWith("/"))
+				{
+					source = source.Substring(0, source.Length - 1);
+				}
+				return source;
+			}
+			else if (File.Exists(source))
+			{
+				FileInfo fi = new FileInfo(source);
+				DirectoryInfo di = fi.Directory;
+				return di.FullName;
+			}
+			else
+			{
+				throw new NotImplementedException(source);
+			}
+		}
+
+		private Stream GetSegmentStream(string segmentName)
+		{
+			Console.WriteLine("open segment {0}", segmentName);
+			if (source.StartsWith("http://") || source.StartsWith("https://"))
+			{
+				return webClient.GetStreamAsync(segmentName).Result;
+			}
+			else if (File.Exists(source))
+			{
+				string partFileName = Path.Combine(rootDirectory.FullName, segmentName);
+				FileInfo fi = new FileInfo(partFileName);
+				return fi.OpenRead();
+			}
+			else
+			{
+				throw new NotImplementedException(source);
+			}
+		}
+
+		private bool ReloadSegmentList()
+		{
+			string[] newLines;
+			if (source.StartsWith("http://") || source.StartsWith("https://"))
+			{
+				byte[] result = webClient.GetByteArrayAsync(source).Result;
+				MemoryStream ms = new MemoryStream(result, false);
+				StreamReader streamReader = new StreamReader(ms);
+				List<string> lines = new List<string>();
+				while (!streamReader.EndOfStream)
+				{
+					lines.Add(streamReader.ReadLine());
+				}
+				streamReader.Close();
+				ms.Close();
+				newLines = lines.ToArray();
+			}
+			else if (File.Exists(source))
+			{
+				newLines = File.ReadAllLines(source);
+			}
+			else
+			{
+				throw new NotImplementedException(source);
+			}
+
+			if (newLines.Length != segmentNames.Length)
+			{
+				segmentNames = newLines;
+				return true;
+			}
+			for (int i = 0; i < Math.Min(newLines.Length, segmentNames.Length); i++)
+			{
+				if (!segmentNames[i].Equals(newLines[i]))
+				{
+					segmentNames = newLines;
+					return true;
+				}
+			}
+
+			return false;
+		}
+
+		private bool GetNextSegment()
+		{
+			for (int i = segmentIndex; i < segmentNames.Length; i++)
+			{
+				if (segmentNames[i].StartsWith("#"))
+					continue;
+				if (finishedSegments.Contains(segmentNames[i]))
+					continue;
+
+				this.segmentIndex = i;
+				this.finishedSegments.Add(segmentNames[i]);
+				this.innerStream = GetSegmentStream(segmentNames[i]);
+				return true;
+			}
+			if (ReloadSegmentList())
+			{
+				return GetNextSegment();
+			}
+			return false;
+		}
+
+		public override bool CanRead => throw new NotImplementedException();
+
+		public override bool CanSeek => throw new NotImplementedException();
+
+		public override bool CanWrite => throw new NotImplementedException();
+
+		public override long Length => throw new NotImplementedException();
+
+		public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
+
+		public override void Flush()
+		{
+			throw new NotImplementedException();
+		}
+
+		public override int Read(byte[] buffer, int offset, int count)
+		{
+			int result = 0;
+			while (count > 0)
+			{
+				int readResult = innerStream.Read(buffer, offset, count);
+				if (readResult == 0)
+				{
+					if (!GetNextSegment())
+					{
+						return result;
+					}
+					continue;
+				}
+				offset += readResult;
+				count -= readResult;
+				result += readResult;
+			}
+			return result;
+		}
+
+		public override long Seek(long offset, SeekOrigin origin)
+		{
+			throw new NotImplementedException();
+		}
+
+		public override void SetLength(long value)
+		{
+			throw new NotImplementedException();
+		}
+
+		public override void Write(byte[] buffer, int offset, int count)
+		{
+			throw new NotImplementedException();
+		}
+	}
+}

skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs

@@ -679,7 +679,7 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
         public int? GetCurrentLocationId()
         {
             if (!File.Exists("docsis_location.txt"))
-                return null;
+                return 0;
             else
             {
                 string readAllText = File.ReadAllText("docsis_location.txt");
@@ -1212,7 +1212,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
 
 		public long DnsCountA()
 		{
-			throw new NotImplementedException();
+            if (dnsRecords == null)
+                return 0;
+
+            return dnsRecords.Count(x => x.RecordType.Item1 == 1);
 		}
 
 		public string DnsIpToName(IPAddress source)
@@ -1225,9 +1228,13 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
 			throw new NotImplementedException();
 		}
 
+        private HashSet<DnsRecord> dnsRecords;
 		public void RememberDnsRecord(DnsRecord record)
 		{
-			throw new NotImplementedException();
+            if (dnsRecords == null)
+                dnsRecords = new HashSet<DnsRecord>();
+
+            dnsRecords.Add(record);
 		}
 	}
 }