From 76386bad33f80bf809e14a68b95899a8481f09af Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Sun, 18 May 2025 22:29:04 +0200 Subject: [PATCH] More Docsis Testing --- .gitignore | 1 + skyscraper8.Tests/DocsisTests.cs | 70 ++++++++++++++++++ .../Properties/Resources.Designer.cs | 40 ++++++++++ skyscraper8.Tests/Properties/Resources.resx | 12 +++ .../ModemCapabilitiesEncodingTest.bin | Bin 0 -> 185 bytes .../Resources/test-1packet-01.ts | 1 + .../Resources/test-2packets-02-03.ts | 1 + .../Resources/test-3packets-04-05-06.ts | 1 + skyscraper8.Tests/TsDuckTestPatterns.cs | 36 +++++++++ skyscraper8.Tests/skyscraper8.Tests.csproj | 1 + .../Docsis/AnnexC/CommonTlvEncodings.cs | 5 ++ .../AnnexC/ModemCapabilitiesEncoding.cs | 28 ++++++- skyscraper8/Docsis/DQoS/AuthorizationBlock.cs | 39 ++++++++++ .../PacketCableAuthorizationBlockEncoding.cs | 45 +++++++++++ skyscraper8/Docsis/DocsisEnvironment.cs | 4 + .../5_AuthReply.cs | 37 ++++++++- .../T16_V2_DynamicServiceAdditionResponse.cs | 1 + .../T19_V2_DynamicServiceChangeResponse.cs | 1 + .../T22_V2_DynamicServiceDeletionResponse.cs | 1 + skyscraper8/Passing.cs | 30 ++++---- skyscraper8/Properties/launchSettings.json | 2 +- skyscraper8/Skyscraper/IO/M3U8Stream.cs | 2 +- .../Filesystem/FilesystemScraperStorage.cs | 3 +- .../FilesystemScraperStorageFactory.cs | 2 +- .../Scraper/Storage/InMemory/KnownTsMemory.cs | 15 +++- 25 files changed, 352 insertions(+), 26 deletions(-) create mode 100644 skyscraper8.Tests/DocsisTests.cs create mode 100644 skyscraper8.Tests/Resources/ModemCapabilitiesEncodingTest.bin create mode 100644 skyscraper8.Tests/Resources/test-1packet-01.ts create mode 100644 skyscraper8.Tests/Resources/test-2packets-02-03.ts create mode 100644 skyscraper8.Tests/Resources/test-3packets-04-05-06.ts create mode 100644 skyscraper8.Tests/TsDuckTestPatterns.cs create mode 100644 skyscraper8/Docsis/DQoS/AuthorizationBlock.cs create mode 100644 skyscraper8/Docsis/DQoS/PacketCableAuthorizationBlockEncoding.cs diff --git a/.gitignore b/.gitignore index 2216d9b..16b61a6 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,4 @@ imgui.ini /skyscraper8.Tests/obj /skyscraper8/bin/Debug/net8.0 /skyscraper8/obj +/GUIs/skyscraper8.UI.ImGui/bin/Debug/net8.0/skyscraper5.ini diff --git a/skyscraper8.Tests/DocsisTests.cs b/skyscraper8.Tests/DocsisTests.cs new file mode 100644 index 0000000..68689a6 --- /dev/null +++ b/skyscraper8.Tests/DocsisTests.cs @@ -0,0 +1,70 @@ +using skyscraper5; +using skyscraper5.Docsis.AnnexC; +using skyscraper5.Mpeg2; +using skyscraper5.Skyscraper.Scraper; +using skyscraper5.Skyscraper.Scraper.Storage; +using skyscraper5.Skyscraper.Scraper.Storage.Filesystem; +using skyscraper5.Skyscraper.Scraper.Storage.InMemory; +using skyscraper8.Skyscraper.IO; +using skyscraper8.Tests.Properties; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit.Sdk; + +namespace skyscraper8.Tests +{ + public class DocsisTests + { + [Fact] + public void TestLongBoiSingleStreams() + { + DirectoryInfo di = new DirectoryInfo("D:\\DocsisDemo"); + Skip.If(!di.Exists, "Files not available."); + + FilesystemScraperStorageFactory storageFactory = new FilesystemScraperStorageFactory(); + storageFactory.Directory = "docsis_longboi"; + + Passing passing = new Passing(); + passing.ScraperStorage = storageFactory.CreateScraperStroage(); + passing.MassImportDirectory(di); + } + + [Fact] + public void TestM3u8Stream() + { + FileInfo docsisM3u8 = new FileInfo("D:\\DocsisDemo\\docsis.m3u8"); + Skip.If(!docsisM3u8.Exists, "Index file not available."); + + FileInfo alreadyTested = new FileInfo("docsis_m3u8_test.complete"); + if (alreadyTested.Exists) + { + Debug.WriteLine("Hello!"); + return; + } + + + M3U8Stream m3u8 = new M3U8Stream(docsisM3u8.FullName); + + TsContext tsContext = new TsContext(); + InMemoryScraperStorageFactory storageFactory = new InMemoryScraperStorageFactory(); + IScraperStroage scraperStorage = storageFactory.CreateScraperStroage(); + SkyscraperContext skyscraperContext = new SkyscraperContext(tsContext, null, scraperStorage); + skyscraperContext.InitalizeFilterChain(); + skyscraperContext.IngestFromStream(m3u8); + + File.WriteAllText("docsis_m3u8_test.complete", "1"); + } + + [Fact] + public void AAA_ModemCapabilitiesEncodingTest() + { + byte[] buffer = Resources.ModemCapabilitiesEncodingTest; + ModemCapabilitiesEncoding modemCapabilitiesEncoding = new ModemCapabilitiesEncoding(buffer); + Assert.True(modemCapabilitiesEncoding.Valid); + } + } +} diff --git a/skyscraper8.Tests/Properties/Resources.Designer.cs b/skyscraper8.Tests/Properties/Resources.Designer.cs index b7614cd..db5952e 100644 --- a/skyscraper8.Tests/Properties/Resources.Designer.cs +++ b/skyscraper8.Tests/Properties/Resources.Designer.cs @@ -60,6 +60,16 @@ namespace skyscraper8.Tests.Properties { } } + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] ModemCapabilitiesEncodingTest { + get { + object obj = ResourceManager.GetObject("ModemCapabilitiesEncodingTest", resourceCulture); + return ((byte[])(obj)); + } + } + /// /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. /// @@ -69,5 +79,35 @@ namespace skyscraper8.Tests.Properties { return ((byte[])(obj)); } } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] test_1packet_01 { + get { + object obj = ResourceManager.GetObject("test-1packet-01", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] test_2packets_02_03 { + get { + object obj = ResourceManager.GetObject("test-2packets-02-03", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] test_3packets_04_05_06 { + get { + object obj = ResourceManager.GetObject("test-3packets-04-05-06", resourceCulture); + return ((byte[])(obj)); + } + } } } diff --git a/skyscraper8.Tests/Properties/Resources.resx b/skyscraper8.Tests/Properties/Resources.resx index 29ae772..6b04fd9 100644 --- a/skyscraper8.Tests/Properties/Resources.resx +++ b/skyscraper8.Tests/Properties/Resources.resx @@ -118,7 +118,19 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\Resources\ModemCapabilitiesEncodingTest.bin;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + ..\Resources\ranging_response_test.bin;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\Resources\test-1packet-01.ts;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Resources\test-2packets-02-03.ts;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Resources\test-3packets-04-05-06.ts;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + \ No newline at end of file diff --git a/skyscraper8.Tests/Resources/ModemCapabilitiesEncodingTest.bin b/skyscraper8.Tests/Resources/ModemCapabilitiesEncodingTest.bin new file mode 100644 index 0000000000000000000000000000000000000000..dfe96e12d70fa7b382e15561caac8d30fa24366d GIT binary patch literal 185 zcmWm8NisuW6oAot?t7BIP^F8$dC>#|714)8yagCqh{aiofyEe***7}%5vi_3Ejo0y zMOkD?S!RXRIwCr2tap8W45pK>4SH-EZ?Vmeva9SV`(_97Q2$8V81=m!%M(sHGaPW9 v(Oz)r=W32tL$2pxlo2=bR^C~-&*u9fo8L#C7|UnzA}0TvQ`IXoy?6B=a?cLQ literal 0 HcmV?d00001 diff --git a/skyscraper8.Tests/Resources/test-1packet-01.ts b/skyscraper8.Tests/Resources/test-1packet-01.ts new file mode 100644 index 0000000..39233ec --- /dev/null +++ b/skyscraper8.Tests/Resources/test-1packet-01.ts @@ -0,0 +1 @@ +G@’ \ No newline at end of file diff --git a/skyscraper8.Tests/Resources/test-2packets-02-03.ts b/skyscraper8.Tests/Resources/test-2packets-02-03.ts new file mode 100644 index 0000000..a501351 --- /dev/null +++ b/skyscraper8.Tests/Resources/test-2packets-02-03.ts @@ -0,0 +1 @@ +G@’G@’ \ No newline at end of file diff --git a/skyscraper8.Tests/Resources/test-3packets-04-05-06.ts b/skyscraper8.Tests/Resources/test-3packets-04-05-06.ts new file mode 100644 index 0000000..730d320 --- /dev/null +++ b/skyscraper8.Tests/Resources/test-3packets-04-05-06.ts @@ -0,0 +1 @@ +G@’G@’G@’ \ No newline at end of file diff --git a/skyscraper8.Tests/TsDuckTestPatterns.cs b/skyscraper8.Tests/TsDuckTestPatterns.cs new file mode 100644 index 0000000..8b27163 --- /dev/null +++ b/skyscraper8.Tests/TsDuckTestPatterns.cs @@ -0,0 +1,36 @@ +using skyscraper5.Mpeg2; +using skyscraper5.Skyscraper.Scraper; +using skyscraper5.Skyscraper.Scraper.Storage.InMemory; +using skyscraper8.Tests.Properties; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Tests +{ + public class TsDuckTestPatterns + { + [Fact] + public void RunTestPatterns() + { + byte[][] buffers = new byte[][] { Resources.test_1packet_01, Resources.test_2packets_02_03, Resources.test_3packets_04_05_06 }; + + InMemoryScraperStorageFactory imssf = new InMemoryScraperStorageFactory(); + + foreach (byte[] buffer in buffers) + { + TsContext mpeg2 = new TsContext(); + SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, null, null); + MemoryStream ms = new MemoryStream(buffer, false); + + skyscraper.InitalizeFilterChain(); + skyscraper.IngestFromStream(ms); + + ms.Close(); + ms.Dispose(); + } + } + } +} diff --git a/skyscraper8.Tests/skyscraper8.Tests.csproj b/skyscraper8.Tests/skyscraper8.Tests.csproj index d1bd80c..f25b1bc 100644 --- a/skyscraper8.Tests/skyscraper8.Tests.csproj +++ b/skyscraper8.Tests/skyscraper8.Tests.csproj @@ -20,6 +20,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/skyscraper8/Docsis/AnnexC/CommonTlvEncodings.cs b/skyscraper8/Docsis/AnnexC/CommonTlvEncodings.cs index e2784fd..51aab93 100644 --- a/skyscraper8/Docsis/AnnexC/CommonTlvEncodings.cs +++ b/skyscraper8/Docsis/AnnexC/CommonTlvEncodings.cs @@ -1,4 +1,5 @@ using skyscraper5.Skyscraper.IO; +using skyscraper8.Docsis.DQoS; using System; using System.Collections.Generic; using System.IO; @@ -42,6 +43,9 @@ namespace skyscraper5.Docsis.AnnexC case 27: HmacDigest = v; break; + case 30: + AuthorizationHint = new AuthorizationBlock(v); + break; case 31: KeySequenceNumber = v[0]; break; @@ -131,5 +135,6 @@ namespace skyscraper5.Docsis.AnnexC public List UpstreamServiceFlows { get; } public RcpIdEncoding RcpId { get; private set; } + public AuthorizationBlock AuthorizationHint { get; } } } diff --git a/skyscraper8/Docsis/AnnexC/ModemCapabilitiesEncoding.cs b/skyscraper8/Docsis/AnnexC/ModemCapabilitiesEncoding.cs index 5378a21..f09e511 100644 --- a/skyscraper8/Docsis/AnnexC/ModemCapabilitiesEncoding.cs +++ b/skyscraper8/Docsis/AnnexC/ModemCapabilitiesEncoding.cs @@ -1,10 +1,11 @@ using System; using System.IO; +using skyscraper5.Skyscraper; using skyscraper5.Skyscraper.IO; namespace skyscraper5.Docsis.AnnexC { - internal class ModemCapabilitiesEncoding + public class ModemCapabilitiesEncoding : Validatable { public ModemCapabilitiesEncoding(byte[] buffer) { @@ -22,11 +23,21 @@ namespace skyscraper5.Docsis.AnnexC case 2: DocsisVersion = DecodeVersion(v[0]); break; - default: + case 3: + FragmentationSupport = ((v[0] & 0x01) != 0); + break; + case 4: + PayloadHeaderSuppressionSupport = ((v[0] & 0x01) != 0); + break; + case 6: + PrivacySupport = (PrivacySupportValue)v[0]; + break; + default: //CM-SP-MULPIv4.0-I01-190815.pdf, page 688 throw new NotFiniteNumberException(String.Format("{0} {1}", nameof(ModemCapabilitiesEncoding), type)); } } + Valid = true; } public Version DocsisVersion { get; private set; } @@ -47,6 +58,15 @@ namespace skyscraper5.Docsis.AnnexC } } - public bool ConcatenationSupport { get; private set; } - } + public bool? ConcatenationSupport { get; private set; } + public bool? FragmentationSupport { get; } + public bool? PayloadHeaderSuppressionSupport { get; } + public PrivacySupportValue? PrivacySupport { get; } + + public enum PrivacySupportValue : byte + { + BpiSupport = 0, + BpiPlusSupport = 1 + } + } } diff --git a/skyscraper8/Docsis/DQoS/AuthorizationBlock.cs b/skyscraper8/Docsis/DQoS/AuthorizationBlock.cs new file mode 100644 index 0000000..97c54e0 --- /dev/null +++ b/skyscraper8/Docsis/DQoS/AuthorizationBlock.cs @@ -0,0 +1,39 @@ +using skyscraper5.Skyscraper; +using skyscraper5.Skyscraper.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Docsis.DQoS +{ + internal class AuthorizationBlock : Validatable + { + public AuthorizationBlock(byte[] buffer) + { + MemoryStream ms = new MemoryStream(buffer, false); + while (ms.GetAvailableBytes() > 3) + { + byte type = ms.ReadUInt8(); + ushort length = ms.ReadUInt8(); + byte[] value = ms.ReadBytes(length); + + switch (type) + { + case 1: + PacketCableAuthorizationBlock = new PacketCableAuthorizationBlockEncoding(value); + break; + default: + //According to https://de.scribd.com/document/482615299/PKT-SP-DQOS-C01-071129 page 43, + //Type 1 is the only valid value. + Valid = false; + return; + } + } + Valid = true; + } + + public PacketCableAuthorizationBlockEncoding PacketCableAuthorizationBlock { get; private set; } + } +} diff --git a/skyscraper8/Docsis/DQoS/PacketCableAuthorizationBlockEncoding.cs b/skyscraper8/Docsis/DQoS/PacketCableAuthorizationBlockEncoding.cs new file mode 100644 index 0000000..2edade3 --- /dev/null +++ b/skyscraper8/Docsis/DQoS/PacketCableAuthorizationBlockEncoding.cs @@ -0,0 +1,45 @@ +using skyscraper5.Skyscraper; +using skyscraper5.Skyscraper.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Docsis.DQoS +{ + internal class PacketCableAuthorizationBlockEncoding : Validatable + { + public PacketCableAuthorizationBlockEncoding(byte[] buffer) + { + MemoryStream ms = new MemoryStream(buffer, false); + while (ms.GetAvailableBytes() > 3) + { + byte type = ms.ReadUInt8(); + ushort length = ms.ReadUInt8(); + byte[] value = ms.ReadBytes(length); + + switch (type) + { + case 1: + (value[0], value[1], value[2], value[3]) = (value[3], value[2], value[1], value[0]); + GateId = BitConverter.ToUInt32(value); + break; + case 2: + (value[0], value[1], value[2], value[3]) = (value[3], value[2], value[1], value[0]); + ResourceId = BitConverter.ToUInt32(value); + break; + default: + //According to https://de.scribd.com/document/482615299/PKT-SP-DQOS-C01-071129 page 43, + //this is complete. + Valid = false; + return; + } + } + Valid = true; + } + + public uint GateId { get; } + public uint ResourceId { get; } + } +} diff --git a/skyscraper8/Docsis/DocsisEnvironment.cs b/skyscraper8/Docsis/DocsisEnvironment.cs index 20d83bc..f9f2cc7 100644 --- a/skyscraper8/Docsis/DocsisEnvironment.cs +++ b/skyscraper8/Docsis/DocsisEnvironment.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using System.Xml.XPath; using Ionic.Crc; using skyscraper5.Docsis.MacManagement; +using skyscraper5.Docsis.MacManagement.BaselinePrivacyKeyManagementMessages; using skyscraper5.Ietf.Rfc2460; using skyscraper5.Ietf.Rfc826; using skyscraper5.Ietf.Rfc971; @@ -340,6 +341,9 @@ namespace skyscraper5.Docsis PrivacyKeyManagementResponse privacyKeyManagementResponse = (PrivacyKeyManagementResponse)mmm; switch (privacyKeyManagementResponse.Code) { + case 5: + //Contains a scrambled authentication key, so not interesting for us + break; case 8: //Since the key is encrypted as well, this isn't interesting. break; diff --git a/skyscraper8/Docsis/MacManagement/BaselinePrivacyKeyManagementMessages/5_AuthReply.cs b/skyscraper8/Docsis/MacManagement/BaselinePrivacyKeyManagementMessages/5_AuthReply.cs index 2c4cef2..355fd3c 100644 --- a/skyscraper8/Docsis/MacManagement/BaselinePrivacyKeyManagementMessages/5_AuthReply.cs +++ b/skyscraper8/Docsis/MacManagement/BaselinePrivacyKeyManagementMessages/5_AuthReply.cs @@ -52,12 +52,47 @@ namespace skyscraper5.Docsis.MacManagement.BaselinePrivacyKeyManagementMessages switch(type) { - default: + case 12: + (value[0], value[1]) = (value[1], value[0]); + SAID = BitConverter.ToUInt16(value); + break; + case 20: + DataEncryptionAlgorithm = (DataEncryptionAlgorithmIdentifiers)value[0]; + DataAuthenticationAlgorithm = (DataAuthenticationIdentifiers)value[1]; + break; + case 24: + SaType = (SaTypeEnum)value[0]; + break; + default: //CM-SP-SECv4.0-I01-190815.pdf, page 65 throw new NotImplementedException(); } } } + + public ushort SAID { get; } + public SaTypeEnum SaType { get; } + public DataEncryptionAlgorithmIdentifiers DataEncryptionAlgorithm { get; } + public DataAuthenticationIdentifiers DataAuthenticationAlgorithm { get; } + + public enum SaTypeEnum + { + Primary, + Static, + Dynamic + } + + public enum DataEncryptionAlgorithmIdentifiers + { + Cbc56Des = 1, + Cbc40Des = 2, + Cbc128Aes = 3, + } + + public enum DataAuthenticationIdentifiers + { + NoDataAuthentication = 0 + } } } } diff --git a/skyscraper8/Docsis/MacManagement/T16_V2_DynamicServiceAdditionResponse.cs b/skyscraper8/Docsis/MacManagement/T16_V2_DynamicServiceAdditionResponse.cs index e37214a..3c838db 100644 --- a/skyscraper8/Docsis/MacManagement/T16_V2_DynamicServiceAdditionResponse.cs +++ b/skyscraper8/Docsis/MacManagement/T16_V2_DynamicServiceAdditionResponse.cs @@ -21,6 +21,7 @@ namespace skyscraper5.Docsis.MacManagement TransactionId = ms.ReadUInt16BE(); ConfirmationCode = ms.ReadUInt8(); TlvEncodedInformation = new CommonTlvEncodingObject(ms); + Valid = true; } public CommonTlvEncodingObject TlvEncodedInformation { get; set; } diff --git a/skyscraper8/Docsis/MacManagement/T19_V2_DynamicServiceChangeResponse.cs b/skyscraper8/Docsis/MacManagement/T19_V2_DynamicServiceChangeResponse.cs index 76f507f..247bb12 100644 --- a/skyscraper8/Docsis/MacManagement/T19_V2_DynamicServiceChangeResponse.cs +++ b/skyscraper8/Docsis/MacManagement/T19_V2_DynamicServiceChangeResponse.cs @@ -22,6 +22,7 @@ namespace skyscraper5.Docsis.MacManagement ConfirmationCode = ms.ReadUInt8(); CommonEncodings = new CommonTlvEncodingObject(ms); + Valid = true; } public CommonTlvEncodingObject CommonEncodings { get; private set; } diff --git a/skyscraper8/Docsis/MacManagement/T22_V2_DynamicServiceDeletionResponse.cs b/skyscraper8/Docsis/MacManagement/T22_V2_DynamicServiceDeletionResponse.cs index edfde32..4535923 100644 --- a/skyscraper8/Docsis/MacManagement/T22_V2_DynamicServiceDeletionResponse.cs +++ b/skyscraper8/Docsis/MacManagement/T22_V2_DynamicServiceDeletionResponse.cs @@ -17,6 +17,7 @@ namespace skyscraper5.Docsis.MacManagement (buffer[0], buffer[1]) = (buffer[1], buffer[0]); TransactionId = BitConverter.ToUInt16(buffer, 0); ConfirmationCode = buffer[2]; + Valid = true; } public byte ConfirmationCode { get; set; } diff --git a/skyscraper8/Passing.cs b/skyscraper8/Passing.cs index a4fb50d..f8c94b2 100644 --- a/skyscraper8/Passing.cs +++ b/skyscraper8/Passing.cs @@ -23,9 +23,9 @@ using skyscraper5.Skyscraper.Scraper.Storage; namespace skyscraper5 { - internal class Passing + public class Passing { - private IScraperStroage scraperStroage; + public IScraperStroage ScraperStorage { get; set; } private IStreamReader streamReader; private List tuners; private List satellitePositions; @@ -76,8 +76,8 @@ namespace skyscraper5 PluginManager.GetInstance().AutoconfigureObject(factoryCname, valuePair.Value); Console.WriteLine("Booting {0}...", valuePair.Key.DisplayName); - scraperStroage = valuePair.Value.CreateScraperStroage(); - if (scraperStroage == null) + ScraperStorage = valuePair.Value.CreateScraperStroage(); + if (ScraperStorage == null) { Console.WriteLine("The storage factory didn't create a storage."); return false; @@ -153,9 +153,9 @@ namespace skyscraper5 continue; } ; - if (scraperStroage.UiTunerTestFor(foundTuner)) + if (ScraperStorage.UiTunerTestFor(foundTuner)) { - scraperStroage.UiTunerGetConfiguration(foundTuner); + ScraperStorage.UiTunerGetConfiguration(foundTuner); } if (tuners == null) @@ -163,14 +163,14 @@ namespace skyscraper5 tuners.Add(foundTuner); } - satellitePositions = scraperStroage.UiSatellitesListAll(); + satellitePositions = ScraperStorage.UiSatellitesListAll(); return true; } private HeadlessJob GetNextJob() { - HeadlessJob headlessJob = scraperStroage.GetQueuedJob(); + HeadlessJob headlessJob = ScraperStorage.GetQueuedJob(); if (headlessJob != null) { return headlessJob; @@ -187,7 +187,7 @@ namespace skyscraper5 Run(headlessJob); if (!headlessJob.isSynthetic) { - scraperStroage.SetQueuedJobComplete(headlessJob); + ScraperStorage.SetQueuedJobComplete(headlessJob); } } } @@ -226,7 +226,7 @@ namespace skyscraper5 private void ReimportTag(int tag1) { - IReadOnlyList queue = scraperStroage.ListImportFileByTag1(tag1); + IReadOnlyList queue = ScraperStorage.ListImportFileByTag1(tag1); foreach(string filename in queue) { FileInfo fi = new FileInfo(filename); @@ -236,7 +236,7 @@ namespace skyscraper5 } } - private void MassImportDirectory(DirectoryInfo sourceDir) + public void MassImportDirectory(DirectoryInfo sourceDir) { foreach (FileSystemInfo fileSystemInfo in sourceDir.GetFileSystemInfos()) { @@ -254,7 +254,7 @@ namespace skyscraper5 if (!fi.Extension.ToLowerInvariant().Equals(".ts")) continue; - if (scraperStroage.ImportFileKnown(fi)) + if (ScraperStorage.ImportFileKnown(fi)) continue; FileStream fileStream = fi.OpenRead(); @@ -263,8 +263,8 @@ namespace skyscraper5 ScrapeStream(fileStream,true, out tstype); stopwatch.Stop(); Console.WriteLine("Importing {0} took {1}",fi.Name, stopwatch.Elapsed); - scraperStroage.WaitForCompletion(); - scraperStroage.ImportMarkFileAsKnown(fi, stopwatch.Elapsed, tstype); + ScraperStorage.WaitForCompletion(); + ScraperStorage.ImportMarkFileAsKnown(fi, stopwatch.Elapsed, tstype); fileStream.Close(); } else @@ -287,7 +287,7 @@ namespace skyscraper5 if (eventLogger == null) eventLogger = new ConsoleEventLogger(); - SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), eventLogger, scraperStroage); + SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), eventLogger, ScraperStorage); skyscraperContext.SourceIsDisk = disk; skyscraperContext.InitalizeFilterChain(); skyscraperContext.IngestFromStream(inStream); diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index a1bb1e7..09f259f 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"D:\\DocsisDemo\\docsis.m3u8\"", + "commandLineArgs": "\"D:\\\\DocsisDemo\\\\docsis-000575.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/IO/M3U8Stream.cs b/skyscraper8/Skyscraper/IO/M3U8Stream.cs index 5d9b45b..d72b71e 100644 --- a/skyscraper8/Skyscraper/IO/M3U8Stream.cs +++ b/skyscraper8/Skyscraper/IO/M3U8Stream.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; namespace skyscraper8.Skyscraper.IO { - internal class M3U8Stream : Stream + public class M3U8Stream : Stream { public M3U8Stream(string source) { diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs index 791b4e7..e9856d5 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorage.cs @@ -45,7 +45,8 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public FilesystemScraperStorage(DirectoryInfo rootDirectory) { - this.rootDirectory = rootDirectory; + EnsureDirectoryExists(rootDirectory); + this.rootDirectory = rootDirectory; this.importFilesKnownFilename = Path.Combine(rootDirectory.FullName, "import_files_known.json"); this.jsonSerializerSettings = new JsonSerializerSettings() { diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorageFactory.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorageFactory.cs index ce8c5ec..ee1e82c 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorageFactory.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemScraperStorageFactory.cs @@ -10,7 +10,7 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem { [SkyscraperPlugin] [ScrapeStorageFactoryId(1,"Filesystem",false)] - internal class FilesystemScraperStorageFactory : IScraperStorageFactory + public class FilesystemScraperStorageFactory : IScraperStorageFactory { public string Directory { get; set; } public IScraperStroage CreateScraperStroage() diff --git a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/KnownTsMemory.cs b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/KnownTsMemory.cs index 7b18cac..8cc5e32 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/KnownTsMemory.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/KnownTsMemory.cs @@ -19,6 +19,11 @@ namespace skyscraper5.src.Skyscraper.Scraper.Storage.InMemory public string name; public TimeSpan timespan; public int tsType; + + public override string ToString() + { + return name; + } } private List importFilesKnown; private string importFilesKnownFilename; @@ -30,7 +35,7 @@ namespace skyscraper5.src.Skyscraper.Scraper.Storage.InMemory if (File.Exists(importFilesKnownFilename)) { - importFilesKnown = JsonConvert.DeserializeObject>(File.ReadAllText("importFilesKnown.json")); + importFilesKnown = JsonConvert.DeserializeObject>(File.ReadAllText(importFilesKnownFilename)); } else { @@ -52,7 +57,13 @@ namespace skyscraper5.src.Skyscraper.Scraper.Storage.InMemory public bool ImportFileKnown(FileInfo fi) { LoadImportFilesKnown(); - return importFilesKnown.Any(x => x.name.Equals(fi)); + + foreach(KnownTs filename in importFilesKnown) + { + if (filename.name.Equals(fi.FullName)) + return true; + } + return false; } } }