More Docsis Testing

This commit is contained in:
feyris-tan 2025-05-18 22:29:04 +02:00
parent 0f5ad93c24
commit 76386bad33
25 changed files with 352 additions and 26 deletions

1
.gitignore vendored
View File

@ -106,3 +106,4 @@ imgui.ini
/skyscraper8.Tests/obj
/skyscraper8/bin/Debug/net8.0
/skyscraper8/obj
/GUIs/skyscraper8.UI.ImGui/bin/Debug/net8.0/skyscraper5.ini

View File

@ -0,0 +1,70 @@
using skyscraper5;
using skyscraper5.Docsis.AnnexC;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.Scraper;
using skyscraper5.Skyscraper.Scraper.Storage;
using skyscraper5.Skyscraper.Scraper.Storage.Filesystem;
using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
using skyscraper8.Skyscraper.IO;
using skyscraper8.Tests.Properties;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Xunit.Sdk;
namespace skyscraper8.Tests
{
public class DocsisTests
{
[Fact]
public void TestLongBoiSingleStreams()
{
DirectoryInfo di = new DirectoryInfo("D:\\DocsisDemo");
Skip.If(!di.Exists, "Files not available.");
FilesystemScraperStorageFactory storageFactory = new FilesystemScraperStorageFactory();
storageFactory.Directory = "docsis_longboi";
Passing passing = new Passing();
passing.ScraperStorage = storageFactory.CreateScraperStroage();
passing.MassImportDirectory(di);
}
[Fact]
public void TestM3u8Stream()
{
FileInfo docsisM3u8 = new FileInfo("D:\\DocsisDemo\\docsis.m3u8");
Skip.If(!docsisM3u8.Exists, "Index file not available.");
FileInfo alreadyTested = new FileInfo("docsis_m3u8_test.complete");
if (alreadyTested.Exists)
{
Debug.WriteLine("Hello!");
return;
}
M3U8Stream m3u8 = new M3U8Stream(docsisM3u8.FullName);
TsContext tsContext = new TsContext();
InMemoryScraperStorageFactory storageFactory = new InMemoryScraperStorageFactory();
IScraperStroage scraperStorage = storageFactory.CreateScraperStroage();
SkyscraperContext skyscraperContext = new SkyscraperContext(tsContext, null, scraperStorage);
skyscraperContext.InitalizeFilterChain();
skyscraperContext.IngestFromStream(m3u8);
File.WriteAllText("docsis_m3u8_test.complete", "1");
}
[Fact]
public void AAA_ModemCapabilitiesEncodingTest()
{
byte[] buffer = Resources.ModemCapabilitiesEncodingTest;
ModemCapabilitiesEncoding modemCapabilitiesEncoding = new ModemCapabilitiesEncoding(buffer);
Assert.True(modemCapabilitiesEncoding.Valid);
}
}
}

View File

@ -60,6 +60,16 @@ namespace skyscraper8.Tests.Properties {
}
}
/// <summary>
/// Sucht eine lokalisierte Ressource vom Typ System.Byte[].
/// </summary>
internal static byte[] ModemCapabilitiesEncodingTest {
get {
object obj = ResourceManager.GetObject("ModemCapabilitiesEncodingTest", resourceCulture);
return ((byte[])(obj));
}
}
/// <summary>
/// Sucht eine lokalisierte Ressource vom Typ System.Byte[].
/// </summary>
@ -69,5 +79,35 @@ namespace skyscraper8.Tests.Properties {
return ((byte[])(obj));
}
}
/// <summary>
/// Sucht eine lokalisierte Ressource vom Typ System.Byte[].
/// </summary>
internal static byte[] test_1packet_01 {
get {
object obj = ResourceManager.GetObject("test-1packet-01", resourceCulture);
return ((byte[])(obj));
}
}
/// <summary>
/// Sucht eine lokalisierte Ressource vom Typ System.Byte[].
/// </summary>
internal static byte[] test_2packets_02_03 {
get {
object obj = ResourceManager.GetObject("test-2packets-02-03", resourceCulture);
return ((byte[])(obj));
}
}
/// <summary>
/// Sucht eine lokalisierte Ressource vom Typ System.Byte[].
/// </summary>
internal static byte[] test_3packets_04_05_06 {
get {
object obj = ResourceManager.GetObject("test-3packets-04-05-06", resourceCulture);
return ((byte[])(obj));
}
}
}
}

View File

@ -118,7 +118,19 @@
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<assembly alias="System.Windows.Forms" name="System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" />
<data name="ModemCapabilitiesEncodingTest" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\ModemCapabilitiesEncodingTest.bin;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</data>
<data name="ranging_response_test" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\ranging_response_test.bin;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</data>
<data name="test-1packet-01" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\test-1packet-01.ts;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</data>
<data name="test-2packets-02-03" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\test-2packets-02-03.ts;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</data>
<data name="test-3packets-04-05-06" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\test-3packets-04-05-06.ts;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</data>
</root>

View File

@ -0,0 +1 @@
G@<EFBFBD>

View File

@ -0,0 +1 @@
G@<EFBFBD>G@<EFBFBD>

View File

@ -0,0 +1 @@
G@<EFBFBD>G@<EFBFBD>G@<EFBFBD>

View File

@ -0,0 +1,36 @@
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.Scraper;
using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
using skyscraper8.Tests.Properties;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.Tests
{
public class TsDuckTestPatterns
{
[Fact]
public void RunTestPatterns()
{
byte[][] buffers = new byte[][] { Resources.test_1packet_01, Resources.test_2packets_02_03, Resources.test_3packets_04_05_06 };
InMemoryScraperStorageFactory imssf = new InMemoryScraperStorageFactory();
foreach (byte[] buffer in buffers)
{
TsContext mpeg2 = new TsContext();
SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, null, null);
MemoryStream ms = new MemoryStream(buffer, false);
skyscraper.InitalizeFilterChain();
skyscraper.IngestFromStream(ms);
ms.Close();
ms.Dispose();
}
}
}
}

View File

@ -20,6 +20,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Xunit.SkippableFact" Version="1.5.23" />
</ItemGroup>
<ItemGroup>

View File

@ -1,4 +1,5 @@
using skyscraper5.Skyscraper.IO;
using skyscraper8.Docsis.DQoS;
using System;
using System.Collections.Generic;
using System.IO;
@ -42,6 +43,9 @@ namespace skyscraper5.Docsis.AnnexC
case 27:
HmacDigest = v;
break;
case 30:
AuthorizationHint = new AuthorizationBlock(v);
break;
case 31:
KeySequenceNumber = v[0];
break;
@ -131,5 +135,6 @@ namespace skyscraper5.Docsis.AnnexC
public List<GeneralServiceFlowEncoding> UpstreamServiceFlows { get; }
public RcpIdEncoding RcpId { get; private set; }
public AuthorizationBlock AuthorizationHint { get; }
}
}

View File

@ -1,10 +1,11 @@
using System;
using System.IO;
using skyscraper5.Skyscraper;
using skyscraper5.Skyscraper.IO;
namespace skyscraper5.Docsis.AnnexC
{
internal class ModemCapabilitiesEncoding
public class ModemCapabilitiesEncoding : Validatable
{
public ModemCapabilitiesEncoding(byte[] buffer)
{
@ -22,11 +23,21 @@ namespace skyscraper5.Docsis.AnnexC
case 2:
DocsisVersion = DecodeVersion(v[0]);
break;
default:
case 3:
FragmentationSupport = ((v[0] & 0x01) != 0);
break;
case 4:
PayloadHeaderSuppressionSupport = ((v[0] & 0x01) != 0);
break;
case 6:
PrivacySupport = (PrivacySupportValue)v[0];
break;
default:
//CM-SP-MULPIv4.0-I01-190815.pdf, page 688
throw new NotFiniteNumberException(String.Format("{0} {1}", nameof(ModemCapabilitiesEncoding), type));
}
}
Valid = true;
}
public Version DocsisVersion { get; private set; }
@ -47,6 +58,15 @@ namespace skyscraper5.Docsis.AnnexC
}
}
public bool ConcatenationSupport { get; private set; }
}
public bool? ConcatenationSupport { get; private set; }
public bool? FragmentationSupport { get; }
public bool? PayloadHeaderSuppressionSupport { get; }
public PrivacySupportValue? PrivacySupport { get; }
public enum PrivacySupportValue : byte
{
BpiSupport = 0,
BpiPlusSupport = 1
}
}
}

View File

@ -0,0 +1,39 @@
using skyscraper5.Skyscraper;
using skyscraper5.Skyscraper.IO;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.Docsis.DQoS
{
internal class AuthorizationBlock : Validatable
{
public AuthorizationBlock(byte[] buffer)
{
MemoryStream ms = new MemoryStream(buffer, false);
while (ms.GetAvailableBytes() > 3)
{
byte type = ms.ReadUInt8();
ushort length = ms.ReadUInt8();
byte[] value = ms.ReadBytes(length);
switch (type)
{
case 1:
PacketCableAuthorizationBlock = new PacketCableAuthorizationBlockEncoding(value);
break;
default:
//According to https://de.scribd.com/document/482615299/PKT-SP-DQOS-C01-071129 page 43,
//Type 1 is the only valid value.
Valid = false;
return;
}
}
Valid = true;
}
public PacketCableAuthorizationBlockEncoding PacketCableAuthorizationBlock { get; private set; }
}
}

View File

@ -0,0 +1,45 @@
using skyscraper5.Skyscraper;
using skyscraper5.Skyscraper.IO;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.Docsis.DQoS
{
internal class PacketCableAuthorizationBlockEncoding : Validatable
{
public PacketCableAuthorizationBlockEncoding(byte[] buffer)
{
MemoryStream ms = new MemoryStream(buffer, false);
while (ms.GetAvailableBytes() > 3)
{
byte type = ms.ReadUInt8();
ushort length = ms.ReadUInt8();
byte[] value = ms.ReadBytes(length);
switch (type)
{
case 1:
(value[0], value[1], value[2], value[3]) = (value[3], value[2], value[1], value[0]);
GateId = BitConverter.ToUInt32(value);
break;
case 2:
(value[0], value[1], value[2], value[3]) = (value[3], value[2], value[1], value[0]);
ResourceId = BitConverter.ToUInt32(value);
break;
default:
//According to https://de.scribd.com/document/482615299/PKT-SP-DQOS-C01-071129 page 43,
//this is complete.
Valid = false;
return;
}
}
Valid = true;
}
public uint GateId { get; }
public uint ResourceId { get; }
}
}

View File

@ -12,6 +12,7 @@ using System.Threading.Tasks;
using System.Xml.XPath;
using Ionic.Crc;
using skyscraper5.Docsis.MacManagement;
using skyscraper5.Docsis.MacManagement.BaselinePrivacyKeyManagementMessages;
using skyscraper5.Ietf.Rfc2460;
using skyscraper5.Ietf.Rfc826;
using skyscraper5.Ietf.Rfc971;
@ -340,6 +341,9 @@ namespace skyscraper5.Docsis
PrivacyKeyManagementResponse privacyKeyManagementResponse = (PrivacyKeyManagementResponse)mmm;
switch (privacyKeyManagementResponse.Code)
{
case 5:
//Contains a scrambled authentication key, so not interesting for us
break;
case 8:
//Since the key is encrypted as well, this isn't interesting.
break;

View File

@ -52,12 +52,47 @@ namespace skyscraper5.Docsis.MacManagement.BaselinePrivacyKeyManagementMessages
switch(type)
{
default:
case 12:
(value[0], value[1]) = (value[1], value[0]);
SAID = BitConverter.ToUInt16(value);
break;
case 20:
DataEncryptionAlgorithm = (DataEncryptionAlgorithmIdentifiers)value[0];
DataAuthenticationAlgorithm = (DataAuthenticationIdentifiers)value[1];
break;
case 24:
SaType = (SaTypeEnum)value[0];
break;
default:
//CM-SP-SECv4.0-I01-190815.pdf, page 65
throw new NotImplementedException();
}
}
}
public ushort SAID { get; }
public SaTypeEnum SaType { get; }
public DataEncryptionAlgorithmIdentifiers DataEncryptionAlgorithm { get; }
public DataAuthenticationIdentifiers DataAuthenticationAlgorithm { get; }
public enum SaTypeEnum
{
Primary,
Static,
Dynamic
}
public enum DataEncryptionAlgorithmIdentifiers
{
Cbc56Des = 1,
Cbc40Des = 2,
Cbc128Aes = 3,
}
public enum DataAuthenticationIdentifiers
{
NoDataAuthentication = 0
}
}
}
}

View File

@ -21,6 +21,7 @@ namespace skyscraper5.Docsis.MacManagement
TransactionId = ms.ReadUInt16BE();
ConfirmationCode = ms.ReadUInt8();
TlvEncodedInformation = new CommonTlvEncodingObject(ms);
Valid = true;
}
public CommonTlvEncodingObject TlvEncodedInformation { get; set; }

View File

@ -22,6 +22,7 @@ namespace skyscraper5.Docsis.MacManagement
ConfirmationCode = ms.ReadUInt8();
CommonEncodings = new CommonTlvEncodingObject(ms);
Valid = true;
}
public CommonTlvEncodingObject CommonEncodings { get; private set; }

View File

@ -17,6 +17,7 @@ namespace skyscraper5.Docsis.MacManagement
(buffer[0], buffer[1]) = (buffer[1], buffer[0]);
TransactionId = BitConverter.ToUInt16(buffer, 0);
ConfirmationCode = buffer[2];
Valid = true;
}
public byte ConfirmationCode { get; set; }

View File

@ -23,9 +23,9 @@ using skyscraper5.Skyscraper.Scraper.Storage;
namespace skyscraper5
{
internal class Passing
public class Passing
{
private IScraperStroage scraperStroage;
public IScraperStroage ScraperStorage { get; set; }
private IStreamReader streamReader;
private List<TunerMetadata> tuners;
private List<SatellitePosition> satellitePositions;
@ -76,8 +76,8 @@ namespace skyscraper5
PluginManager.GetInstance().AutoconfigureObject(factoryCname, valuePair.Value);
Console.WriteLine("Booting {0}...", valuePair.Key.DisplayName);
scraperStroage = valuePair.Value.CreateScraperStroage();
if (scraperStroage == null)
ScraperStorage = valuePair.Value.CreateScraperStroage();
if (ScraperStorage == null)
{
Console.WriteLine("The storage factory didn't create a storage.");
return false;
@ -153,9 +153,9 @@ namespace skyscraper5
continue;
}
;
if (scraperStroage.UiTunerTestFor(foundTuner))
if (ScraperStorage.UiTunerTestFor(foundTuner))
{
scraperStroage.UiTunerGetConfiguration(foundTuner);
ScraperStorage.UiTunerGetConfiguration(foundTuner);
}
if (tuners == null)
@ -163,14 +163,14 @@ namespace skyscraper5
tuners.Add(foundTuner);
}
satellitePositions = scraperStroage.UiSatellitesListAll();
satellitePositions = ScraperStorage.UiSatellitesListAll();
return true;
}
private HeadlessJob GetNextJob()
{
HeadlessJob headlessJob = scraperStroage.GetQueuedJob();
HeadlessJob headlessJob = ScraperStorage.GetQueuedJob();
if (headlessJob != null)
{
return headlessJob;
@ -187,7 +187,7 @@ namespace skyscraper5
Run(headlessJob);
if (!headlessJob.isSynthetic)
{
scraperStroage.SetQueuedJobComplete(headlessJob);
ScraperStorage.SetQueuedJobComplete(headlessJob);
}
}
}
@ -226,7 +226,7 @@ namespace skyscraper5
private void ReimportTag(int tag1)
{
IReadOnlyList<string> queue = scraperStroage.ListImportFileByTag1(tag1);
IReadOnlyList<string> queue = ScraperStorage.ListImportFileByTag1(tag1);
foreach(string filename in queue)
{
FileInfo fi = new FileInfo(filename);
@ -236,7 +236,7 @@ namespace skyscraper5
}
}
private void MassImportDirectory(DirectoryInfo sourceDir)
public void MassImportDirectory(DirectoryInfo sourceDir)
{
foreach (FileSystemInfo fileSystemInfo in sourceDir.GetFileSystemInfos())
{
@ -254,7 +254,7 @@ namespace skyscraper5
if (!fi.Extension.ToLowerInvariant().Equals(".ts"))
continue;
if (scraperStroage.ImportFileKnown(fi))
if (ScraperStorage.ImportFileKnown(fi))
continue;
FileStream fileStream = fi.OpenRead();
@ -263,8 +263,8 @@ namespace skyscraper5
ScrapeStream(fileStream,true, out tstype);
stopwatch.Stop();
Console.WriteLine("Importing {0} took {1}",fi.Name, stopwatch.Elapsed);
scraperStroage.WaitForCompletion();
scraperStroage.ImportMarkFileAsKnown(fi, stopwatch.Elapsed, tstype);
ScraperStorage.WaitForCompletion();
ScraperStorage.ImportMarkFileAsKnown(fi, stopwatch.Elapsed, tstype);
fileStream.Close();
}
else
@ -287,7 +287,7 @@ namespace skyscraper5
if (eventLogger == null)
eventLogger = new ConsoleEventLogger();
SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), eventLogger, scraperStroage);
SkyscraperContext skyscraperContext = new SkyscraperContext(new TsContext(), eventLogger, ScraperStorage);
skyscraperContext.SourceIsDisk = disk;
skyscraperContext.InitalizeFilterChain();
skyscraperContext.IngestFromStream(inStream);

View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "\"D:\\DocsisDemo\\docsis.m3u8\"",
"commandLineArgs": "\"D:\\\\DocsisDemo\\\\docsis-000575.ts\"",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

View File

@ -7,7 +7,7 @@ using System.Threading.Tasks;
namespace skyscraper8.Skyscraper.IO
{
internal class M3U8Stream : Stream
public class M3U8Stream : Stream
{
public M3U8Stream(string source)
{

View File

@ -45,7 +45,8 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
public FilesystemScraperStorage(DirectoryInfo rootDirectory)
{
this.rootDirectory = rootDirectory;
EnsureDirectoryExists(rootDirectory);
this.rootDirectory = rootDirectory;
this.importFilesKnownFilename = Path.Combine(rootDirectory.FullName, "import_files_known.json");
this.jsonSerializerSettings = new JsonSerializerSettings()
{

View File

@ -10,7 +10,7 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
{
[SkyscraperPlugin]
[ScrapeStorageFactoryId(1,"Filesystem",false)]
internal class FilesystemScraperStorageFactory : IScraperStorageFactory
public class FilesystemScraperStorageFactory : IScraperStorageFactory
{
public string Directory { get; set; }
public IScraperStroage CreateScraperStroage()

View File

@ -19,6 +19,11 @@ namespace skyscraper5.src.Skyscraper.Scraper.Storage.InMemory
public string name;
public TimeSpan timespan;
public int tsType;
public override string ToString()
{
return name;
}
}
private List<KnownTs> importFilesKnown;
private string importFilesKnownFilename;
@ -30,7 +35,7 @@ namespace skyscraper5.src.Skyscraper.Scraper.Storage.InMemory
if (File.Exists(importFilesKnownFilename))
{
importFilesKnown = JsonConvert.DeserializeObject<List<KnownTs>>(File.ReadAllText("importFilesKnown.json"));
importFilesKnown = JsonConvert.DeserializeObject<List<KnownTs>>(File.ReadAllText(importFilesKnownFilename));
}
else
{
@ -52,7 +57,13 @@ namespace skyscraper5.src.Skyscraper.Scraper.Storage.InMemory
public bool ImportFileKnown(FileInfo fi)
{
LoadImportFilesKnown();
return importFilesKnown.Any(x => x.name.Equals(fi));
foreach(KnownTs filename in importFilesKnown)
{
if (filename.name.Equals(fi.FullName))
return true;
}
return false;
}
}
}