From 92056f4ce3dcb3ec0f51996b214cbeacf56114b8 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Sat, 16 May 2026 20:59:46 +0200 Subject: [PATCH] Began working on the salvager. --- .../ResourceTests/NtbuDoabiliyTest.cs | 36 --- skyscraper8.Tests/Resources1.Designer.cs | 255 +++++++++++------- skyscraper8.Tests/Resources1.resx | 3 - .../skyscraper8.Tests.csproj.user | 9 + skyscraper8/Program.cs | 14 + skyscraper8/Properties/launchSettings.json | 2 +- .../Skyscraper/IO/UnseekableInputStream.cs | 116 ++++++++ skyscraper8/Skyscraper/Salvager.cs | 100 +++++++ 8 files changed, 394 insertions(+), 141 deletions(-) delete mode 100644 skyscraper8.Tests/ResourceTests/NtbuDoabiliyTest.cs create mode 100644 skyscraper8.Tests/skyscraper8.Tests.csproj.user create mode 100644 skyscraper8/Skyscraper/IO/UnseekableInputStream.cs create mode 100644 skyscraper8/Skyscraper/Salvager.cs diff --git a/skyscraper8.Tests/ResourceTests/NtbuDoabiliyTest.cs b/skyscraper8.Tests/ResourceTests/NtbuDoabiliyTest.cs deleted file mode 100644 index b7806eb..0000000 --- a/skyscraper8.Tests/ResourceTests/NtbuDoabiliyTest.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.IO; -using System.Net.NetworkInformation; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using skyscraper5.Skyscraper.IO; - -namespace skyscraper8.Tests.ResourceTests; - -[TestClass] -public class NtbuDoability -{ - [TestMethod] - public void DecodeUnifiInform() - { - byte[] bytes = Resources1.ubnt; - MemoryStream stream = new MemoryStream(bytes,false); - if (stream.ReadUInt32BE() != 1414414933u) - Assert.Fail(); - - if (stream.ReadUInt32BE() != 0) - Assert.Fail(); - - PhysicalAddress apMac = new PhysicalAddress(stream.ReadBytes(6)); - ushort flags = stream.ReadUInt16BE(); - - bool encrypted = (flags & 0x0001) != 0; - bool zlib = (flags & 0x0002) != 0; - bool snappy = (flags & 0x0004) != 0; - bool aesGcm = (flags & 0x0008) != 0; - - byte[] iv = stream.ReadBytes(16); - - uint dataVersion = stream.ReadUInt32BE(); - uint payloadLength = stream.ReadUInt32BE(); - byte[] payload = stream.ReadBytes(payloadLength); - } -} diff --git a/skyscraper8.Tests/Resources1.Designer.cs b/skyscraper8.Tests/Resources1.Designer.cs index 750dc61..e60c27e 100644 --- a/skyscraper8.Tests/Resources1.Designer.cs +++ b/skyscraper8.Tests/Resources1.Designer.cs @@ -1,9 +1,10 @@ //------------------------------------------------------------------------------ // -// This code was generated by a tool. +// Dieser Code wurde von einem Tool generiert. +// Laufzeitversion:4.0.30319.42000 // -// Changes to this file may cause incorrect behavior and will be lost if -// the code is regenerated. +// Änderungen an dieser Datei können falsches Verhalten verursachen und gehen verloren, wenn +// der Code erneut generiert wird. // //------------------------------------------------------------------------------ @@ -11,32 +12,46 @@ namespace skyscraper8.Tests { using System; - [System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] - [System.Diagnostics.DebuggerNonUserCodeAttribute()] - [System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + /// + /// Eine stark typisierte Ressourcenklasse zum Suchen von lokalisierten Zeichenfolgen usw. + /// + // Diese Klasse wurde von der StronglyTypedResourceBuilder automatisch generiert + // -Klasse über ein Tool wie ResGen oder Visual Studio automatisch generiert. + // Um einen Member hinzuzufügen oder zu entfernen, bearbeiten Sie die .ResX-Datei und führen dann ResGen + // mit der /str-Option erneut aus, oder Sie erstellen Ihr VS-Projekt neu. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources1 { - private static System.Resources.ResourceManager resourceMan; + private static global::System.Resources.ResourceManager resourceMan; - private static System.Globalization.CultureInfo resourceCulture; + private static global::System.Globalization.CultureInfo resourceCulture; - [System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal Resources1() { } - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Advanced)] - internal static System.Resources.ResourceManager ResourceManager { + /// + /// Gibt die zwischengespeicherte ResourceManager-Instanz zurück, die von dieser Klasse verwendet wird. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { get { - if (object.Equals(null, resourceMan)) { - System.Resources.ResourceManager temp = new System.Resources.ResourceManager("skyscraper8.Tests.Resources1", typeof(Resources1).Assembly); + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("skyscraper8.Tests.Resources1", typeof(Resources1).Assembly); resourceMan = temp; } return resourceMan; } } - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Advanced)] - internal static System.Globalization.CultureInfo Culture { + /// + /// Überschreibt die CurrentUICulture-Eigenschaft des aktuellen Threads für alle + /// Ressourcenzuordnungen, die diese stark typisierte Ressourcenklasse verwenden. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { get { return resourceCulture; } @@ -45,83 +60,9 @@ namespace skyscraper8.Tests { } } - internal static byte[] ModemCapabilitiesEncodingTest { - get { - object obj = ResourceManager.GetObject("ModemCapabilitiesEncodingTest", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] MultipartRegistrationResponseTest { - get { - object obj = ResourceManager.GetObject("MultipartRegistrationResponseTest", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] PushMacManagementMessage_Version4_Type45 { - get { - object obj = ResourceManager.GetObject("PushMacManagementMessage_Version4_Type45", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] ranging_response_test { - get { - object obj = ResourceManager.GetObject("ranging_response_test", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] MultipartRegistrationResponseTest2 { - get { - object obj = ResourceManager.GetObject("MultipartRegistrationResponseTest2", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] test_1packet_01 { - get { - object obj = ResourceManager.GetObject("test-1packet-01", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] test_2packets_02_03 { - get { - object obj = ResourceManager.GetObject("test-2packets-02-03", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] test_3packets_04_05_06 { - get { - object obj = ResourceManager.GetObject("test-3packets-04-05-06", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] TransmitChannelConfigurationObject { - get { - object obj = ResourceManager.GetObject("TransmitChannelConfigurationObject", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] UpstreamChannelDescriptorTest { - get { - object obj = ResourceManager.GetObject("UpstreamChannelDescriptorTest", resourceCulture); - return ((byte[])(obj)); - } - } - - internal static byte[] Frame00001343_TSGS1_MIS000_SYNC001 { - get { - object obj = ResourceManager.GetObject("Frame00001343_TSGS1_MIS000_SYNC001", resourceCulture); - return ((byte[])(obj)); - } - } - + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// internal static byte[] Frame00000008_TSGS1_MIS000_SYNC001 { get { object obj = ResourceManager.GetObject("Frame00000008_TSGS1_MIS000_SYNC001", resourceCulture); @@ -129,6 +70,9 @@ namespace skyscraper8.Tests { } } + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// internal static byte[] Frame00000012_TSGS1_MIS000_SYNC001 { get { object obj = ResourceManager.GetObject("Frame00000012_TSGS1_MIS000_SYNC001", resourceCulture); @@ -136,13 +80,9 @@ namespace skyscraper8.Tests { } } - internal static byte[] sdpTest { - get { - object obj = ResourceManager.GetObject("sdpTest", resourceCulture); - return ((byte[])(obj)); - } - } - + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// internal static byte[] Frame00000357_TSGS1_MIS000_SYNC184 { get { object obj = ResourceManager.GetObject("Frame00000357_TSGS1_MIS000_SYNC184", resourceCulture); @@ -150,9 +90,122 @@ namespace skyscraper8.Tests { } } - internal static byte[] ubnt { + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] Frame00001343_TSGS1_MIS000_SYNC001 { get { - object obj = ResourceManager.GetObject("ubnt", resourceCulture); + object obj = ResourceManager.GetObject("Frame00001343_TSGS1_MIS000_SYNC001", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] ModemCapabilitiesEncodingTest { + get { + object obj = ResourceManager.GetObject("ModemCapabilitiesEncodingTest", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] MultipartRegistrationResponseTest { + get { + object obj = ResourceManager.GetObject("MultipartRegistrationResponseTest", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] MultipartRegistrationResponseTest2 { + get { + object obj = ResourceManager.GetObject("MultipartRegistrationResponseTest2", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] PushMacManagementMessage_Version4_Type45 { + get { + object obj = ResourceManager.GetObject("PushMacManagementMessage_Version4_Type45", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] ranging_response_test { + get { + object obj = ResourceManager.GetObject("ranging_response_test", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] sdpTest { + get { + object obj = ResourceManager.GetObject("sdpTest", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] test_1packet_01 { + get { + object obj = ResourceManager.GetObject("test-1packet-01", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] test_2packets_02_03 { + get { + object obj = ResourceManager.GetObject("test-2packets-02-03", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] test_3packets_04_05_06 { + get { + object obj = ResourceManager.GetObject("test-3packets-04-05-06", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] TransmitChannelConfigurationObject { + get { + object obj = ResourceManager.GetObject("TransmitChannelConfigurationObject", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Sucht eine lokalisierte Ressource vom Typ System.Byte[]. + /// + internal static byte[] UpstreamChannelDescriptorTest { + get { + object obj = ResourceManager.GetObject("UpstreamChannelDescriptorTest", resourceCulture); return ((byte[])(obj)); } } diff --git a/skyscraper8.Tests/Resources1.resx b/skyscraper8.Tests/Resources1.resx index 2a9ee3f..7aca4aa 100644 --- a/skyscraper8.Tests/Resources1.resx +++ b/skyscraper8.Tests/Resources1.resx @@ -163,7 +163,4 @@ Resources\Frame00000357_TSGS1_MIS000_SYNC184.bbframe;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - Resources\ubnt.bin;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - diff --git a/skyscraper8.Tests/skyscraper8.Tests.csproj.user b/skyscraper8.Tests/skyscraper8.Tests.csproj.user new file mode 100644 index 0000000..d30f2ad --- /dev/null +++ b/skyscraper8.Tests/skyscraper8.Tests.csproj.user @@ -0,0 +1,9 @@ + + + + + + Designer + + + \ No newline at end of file diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index 168f42b..587accc 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -406,6 +406,13 @@ namespace skyscraper5 rotator.Run(); return; } + + if (args[0].ToLowerInvariant().Equals("salvage")) + { + Salvager salvager = new Salvager(); + salvager.Run(args); + return; + } } /*Passing passing = new Passing(); @@ -435,6 +442,13 @@ namespace skyscraper5 Console.WriteLine(".\\skyscraper8.exe shannon \"C:\\some\\file.bmp\" - calculates the Shannon entropy value for any given file."); Console.WriteLine(".\\skyscraper8.exe make-catalogue \"C:\\path\\to\\ts\\collection\\\" \"C:\\outputted_index.csv\" - generates a catalogue with core information about your TS files."); Console.WriteLine(".\\skyscraper8.exe pts2bbf2 \"C:\\path\\to\\file.ts\\\" - extracts every single BBFrame from a GS to into a small file for each. (be careful, might generate many small files!)"); + Console.WriteLine(".\\skyscraper8.exe salvage strategyName \"C:\\path\\to\\file.ts\\\" - attempts to salvage a corrupted TS.)"); + Console.WriteLine(); + Console.WriteLine("Salvaging strategy names:"); + Console.WriteLine(" s1 - Stay aligned with 188-bytes, but skip blocks which do not feature the 0x49 sync byte."); + Console.WriteLine(" s2 - Stay aligned with 188-bytes, but pretends the sync byte is valid even if it isn't."); + Console.WriteLine(" s3 - Ignore 188-byte alignment. When the sync-byte is not found after a packet, seek the stream forward until something is found that *might* sync."); + Console.WriteLine(" Note that the salvaging feature needs the first packet to be valid."); } private static void ToggleSubTsDumpConfiguration(bool enabled) diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index ddf05f4..29e21ca 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "F:\\alpha\\2026-03\\reuters-telstar-session3\\reuters3.m3u8", + "commandLineArgs": "salvage s1 \"C:\\devel\\skyscraper8-testsuite\\105.5E_4169.263_H_5237_(2026-04-24 12.36.13)_dump.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/IO/UnseekableInputStream.cs b/skyscraper8/Skyscraper/IO/UnseekableInputStream.cs new file mode 100644 index 0000000..a1f399a --- /dev/null +++ b/skyscraper8/Skyscraper/IO/UnseekableInputStream.cs @@ -0,0 +1,116 @@ +using log4net; +using skyscraper5.src.InteractionChannel.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.IO +{ + internal class UnseekableInputStream : Stream + { + private readonly Stream _wrappedStream; + private bool _lengthSupported; + private bool _positionSupported; + private readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private long emulatedPosition; + + public UnseekableInputStream(Stream wrappedStream) + { + _wrappedStream = wrappedStream; + + try + { + long length = wrappedStream.Length; + logger.DebugFormat("Wrapping a Stream of {0} bytes in an unseekable container.", length); + _lengthSupported = true; + } + catch (Exception e) + { + _lengthSupported = false; + } + + try + { + long position = wrappedStream.Position; + logger.DebugFormat("Current stream position: {0}", position); + _positionSupported = true; + } + catch (Exception e) + { + _positionSupported = false; + } + } + + + public override void Flush() + { + throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + int result = _wrappedStream.Read(buffer, offset, count); + if (!_positionSupported) + { + emulatedPosition += result; + } + + return result; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override bool CanRead => _wrappedStream.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + + public override long Length + { + get + { + if (_lengthSupported) + { + return _wrappedStream.Length; + } + else + { + throw new NotSupportedException(); + } + } + } + + public override long Position + { + get + { + if (_positionSupported) + { + return _wrappedStream.Position; + } + else + { + return emulatedPosition; + } + } + set + { + throw new NotImplementedException(); + } + } + } +} diff --git a/skyscraper8/Skyscraper/Salvager.cs b/skyscraper8/Skyscraper/Salvager.cs new file mode 100644 index 0000000..f95bcc8 --- /dev/null +++ b/skyscraper8/Skyscraper/Salvager.cs @@ -0,0 +1,100 @@ +using log4net; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Sockets; +using System.Text; +using System.Threading.Tasks; +using skyscraper5.Mpeg2; +using skyscraper5.Skyscraper.Scraper; +using skyscraper5.Skyscraper.Scraper.Storage.Filesystem; +using skyscraper5.Skyscraper.Scraper.Storage.InMemory; +using skyscraper8.Skyscraper.Scraper.Storage; + +namespace skyscraper8.Skyscraper +{ + internal class Salvager + { + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + + public void Run(string[] args) + { + if (args.Length != 3) + { + Console.WriteLine("specify the salvaging strategy and the source."); + return; + } + + switch (args[1]) + { + case "s1": Strategy = SalvagingStrategy.IgnorePacketsWithWrongSyncByte; break; + case "s2": Strategy = SalvagingStrategy.PatchSyncByte; break; + case "s3": Strategy = SalvagingStrategy.SeekForSyncByte; break; + default: + { + logger.Fatal("\"{0}\" is not a known salvaging strategy."); + return; + } + } + + FileInfo fi = new FileInfo(args[2]); + if (fi.Exists) + { + InputStream = fi.OpenRead(); + } + else + { + var uri = new Uri("tcp://177.20.20.69:6969", UriKind.Absolute); + + if (uri.Scheme.Equals("tcp")) + { + string host = uri.Host; // "177.20.20.69" + int port = uri.Port; // 6969 + + TcpClient tcpClient = new TcpClient(host, port); + InputStream = tcpClient.GetStream(); + } + else + { + logger.FatalFormat("URI Scheme \"{0}\" is not supported for salvaging.", uri.Scheme); + return; + } + + + } + + if (InputStream == null) + { + logger.FatalFormat("{0} is not a file that exist, and doesn't fit to any known URL scheme."); + return; + } + + Run(); + } + + private void Run() + { + TsContext mpeg2 = new TsContext(); + InMemoryScraperStorage dataStorage = new InMemoryScraperStorage(); + FilesystemStorage objectStorage = new FilesystemStorage(new DirectoryInfo(".")); + Context = new SkyscraperContext(mpeg2, dataStorage, objectStorage); + switch (Strategy) + { + default: + logger.ErrorFormat("I'm sorry, but the strategy \"{0}\" is not yet fully implemented.",Strategy); + return; + } + } + + public SalvagingStrategy Strategy { get; set; } + public Stream InputStream { get; set; } + public SkyscraperContext Context { get; set; } + } + + public enum SalvagingStrategy + { + IgnorePacketsWithWrongSyncByte, + PatchSyncByte, + SeekForSyncByte + } +}