ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented some missing storage methods.

Browse Source
This commit is contained in:
feyris-tan 2025-09-05 23:00:13 +02:00
parent cee4a4ad44
commit 3595b51fcc
8 changed files with 168 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -123,3 +123,4 @@ imgui.ini
/GUIs/skyscraper8.UI.ImGui.MonoGame/obj/Debug/net8.0
/GUIs/skyscraper8.UI.ImGui.MonoGame/obj
/.vs/skyscraper8/CopilotIndices/17.14.995.13737
/.vs/skyscraper8/config

2
GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs
View File

@ -1742,6 +1742,8 @@ namespace SDL2Demo.Jobs
jobContext.Renderables.Add(this);
});
foundFrequenciesWindow.allowZapNow = true;
foundFrequenciesWindow.statusPacketsInQueue = 0;
foundFrequenciesWindow.statusPacketsInTotal = 0;
}
public void OnBlindscanScrapeStopCondition()

84
skyscraper8/Program.cs
View File

@ -24,7 +24,9 @@ using System.Diagnostics;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Reflection;
using System.Runtime.InteropServices;
using skyscraper5.Mpeg2.Descriptors;
using skyscraper5.Skyscraper.Scraper.StreamAutodetection;
using skyscraper8;
using skyscraper8.SatIp;
@ -37,6 +39,7 @@ namespace skyscraper5
class Program
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
private const int PUBLIC_RELEASE = 4;
private static void IntegrationTest()
{
/*List<SsdpDevice> ssdpDevices = SsdpClient.GetSsdpDevices(1000).ToList();
@ -44,8 +47,7 @@ namespace skyscraper5
{
Console.WriteLine("SSDP device: {0}", ssdpDevice.Server);
}*/
Console.WriteLine("yeet!");
/*RtspClient rtspClient = new RtspClient("172.20.20.121", 554);
rtspClient.AutoReconnect = true;
RtspOptionsResponse options = rtspClient.GetOptions("/");
@ -100,8 +102,8 @@ namespace skyscraper5
{
IntegrationTest();
logger.Info("Hello!");
ffmpegFrameGrabber.CanStart();
logger.InfoFormat(String.Format("Hello! This is skyscraper8, public release #{0}, code version {1}", PUBLIC_RELEASE, GetCurrentAssemblyDisplayVersion()));
logger.DebugFormat("Found {0}-bit Operating system.", Environment.Is64BitOperatingSystem ? 64 : 32);
logger.DebugFormat("I'm a {0}-bit Process.", Environment.Is64BitProcess ? 64 : 32);
@ -286,6 +288,7 @@ namespace skyscraper5
Console.WriteLine("You're not supposed to run me directly. Run me from the commandline using the following:");
Console.WriteLine("for example: .\\skyscraper8.exe cscan tcp://127.0.0.1:6969");
Console.WriteLine(" or: .\\skyscraper8.exe \"C:\\path\\to\\file.ts\\");
Console.WriteLine(" or: .\\skyscraper8.exe \"C:\\path\\to\\my\\folder\\with\\ts\\files\\\" (for batch extraction)");
Console.WriteLine();
Console.WriteLine("Bonus feature:");
Console.WriteLine(".\\skyscraper8.exe hlsproxy \"C:\\path\\to\\hls\\files\\\" - to pipe a HLS stream from a local directory into VLC.");
@ -295,30 +298,39 @@ namespace skyscraper5
{
DataStorage dataStorage = new InMemoryScraperStorage();
FilesystemStorage filesystemStorage = new FilesystemStorage(new DirectoryInfo("."));
//DirectoryInfo di = new DirectoryInfo(@"E:\Skyscraper\Astra 19.2");
FileInfo[] fileInfos = di.GetFiles("*.ts");
foreach (FileInfo fileInfo in fileInfos)
{
Console.WriteLine(new string('_', Console.WindowWidth - 1));
Console.WriteLine("Processing: {0}", fileInfo.Name);
SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), dataStorage,filesystemStorage);
//StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest();
FileStream fileStream = fileInfo.OpenRead();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
skyscraper.IngestFromStream(fileStream);
//streamTypeAutodetectionTest.IngestFromStream(fileStream);
stopwatch.Stop();
Console.WriteLine("Time to process: " + stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("File was: {0}", fileInfo.Name);
PrintPidStatistics(skyscraper);
}
ProcessDirectory(di, dataStorage, filesystemStorage);
}
private static void ProcessDirectory(DirectoryInfo di, DataStorage dataStorage, ObjectStorage objectStorage)
{
DirectoryInfo[] directoryInfos = di.GetDirectories();
foreach (DirectoryInfo subdir in directoryInfos)
{
ProcessDirectory(subdir, dataStorage, objectStorage);
}
FileInfo[] fileInfos = di.GetFiles("*.ts");
foreach (FileInfo fileInfo in fileInfos)
{
Console.WriteLine(new string('_', Console.WindowWidth - 1));
Console.WriteLine("Processing: {0}", fileInfo.Name);
SkyscraperContext skyscraper = new SkyscraperContext(new TsContext(), dataStorage, objectStorage);
skyscraper.InitalizeFilterChain();
//StreamTypeAutodetectionTest streamTypeAutodetectionTest = new StreamTypeAutodetectionTest();
FileStream fileStream = fileInfo.OpenRead();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
skyscraper.IngestFromStream(fileStream);
//streamTypeAutodetectionTest.IngestFromStream(fileStream);
stopwatch.Stop();
Console.WriteLine("Time to process: " + stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("File was: {0}", fileInfo.Name);
PrintPidStatistics(skyscraper);
}
}
private static void PrintPidStatistics(SkyscraperContext skyscraper)
{
Console.WriteLine("Unattached PIDs:");
@ -440,8 +452,6 @@ namespace skyscraper5
skyscraper.Dispose();
}
private static void HandleCrazyScanToLiveSystem(string url, bool withTimeout = true)
{
StorageConnectionManager connectionManager = StorageConnectionManager.GetInstance();
@ -798,5 +808,23 @@ namespace skyscraper5
}
}
private static string GetCurrentAssemblyDisplayVersion()
{
try
{
Assembly executingAssembly = System.Reflection.Assembly.GetExecutingAssembly();
AssemblyName assemblyName = executingAssembly.GetName();
Version version = assemblyName.Version;
DateTime buildDate = new DateTime(2000, 1, 1)
.AddDays(version.Build).AddSeconds(version.Revision * 2);
string displayableVersion = $"{version} ({buildDate})";
return displayableVersion;
}
catch (Exception e)
{
return "???";
}
}
}
}

2
skyscraper8/Properties/launchSettings.json
View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "\"C:\\Users\\Sascha Schiemann\\Downloads\\36E-12226L(1)\\36E-12226L.ts\"",
"commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\incoming\"",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

3
skyscraper8/Skyscraper/Scraper/CaSystemNames.cs
View File

@ -32,7 +32,8 @@ namespace skyscraper5.Skyscraper.Scraper
0x18fe,
0x1813, 0x1817, 0x1818, 0x1819, 0x1863, 0x1801, 0x1810, 0x1702, 0x1722, 0x1762,
0x1834,
0x1879
0x1879,
0x185f
};

82
skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs
View File

@ -1460,7 +1460,11 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
public void InsertSgtList(SgtList list)
{
throw new NotImplementedException();
string path = Path.Combine(rootDirectory.FullName, "Astra-SGT", list.ServiceListId.ToString() + ".json");
FileInfo fi = new FileInfo(path);
fi.Directory.EnsureExists();
string json = JsonConvert.SerializeObject(list, Formatting.Indented);
File.WriteAllText(fi.FullName, json);
}
public bool TestForSgtService(SgtService child)
@ -1610,7 +1614,8 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
}
}
string newJson = JsonConvert.SerializeObject(incomingObject);
fileInfo.Directory.EnsureExists();
string newJson = JsonConvert.SerializeObject(incomingObject);
File.WriteAllText(path, newJson);
return true;
}
@ -1651,26 +1656,62 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
File.WriteAllText(path, json);
}
private static string CleanFilename(string input)
{
char[] forbiddenOnes = Path.GetInvalidPathChars();
char[] alsoForbbiden = Path.GetInvalidFileNameChars();
char[] charArray = input.ToCharArray();
for (int i = 0; i < charArray.Length; i++)
{
if (forbiddenOnes.Contains(charArray[i]))
{
charArray[i] = '_';
}
if (alsoForbbiden.Contains(charArray[i]))
{
charArray[i] = '_';
}
}
return new string(charArray);
}
public bool DvbNipTestForMulticastSession(MulticastSessionType multicastSession)
{
throw new NotImplementedException();
string jsonFileName = CleanFilename(multicastSession.serviceIdentifier) + ".json";
string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_sessions", jsonFileName);
FileInfo fi = new FileInfo(path);
return fi.Exists;
}
public void DvbNipInsertMulticastSession(MulticastSessionType multicastSession)
{
throw new NotImplementedException();
string jsonFileName = CleanFilename(multicastSession.serviceIdentifier) + ".json";
string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_sessions", jsonFileName);
FileInfo fi = new FileInfo(path);
fi.Directory.EnsureExists();
string json = JsonConvert.SerializeObject(multicastSession, Formatting.Indented);
File.WriteAllText(fi.FullName, json);
}
public bool DvbNipTestForMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier,
MulticastEndpointAddressType multicastEndpointAddressType)
{
throw new NotImplementedException();
string jsonFileName = String.Format("{0}_{1}_{2}_{3}_{4}.json", carrier.NipNetworkId, carrier.NipCarrierId, carrier.NipLinkId, carrier.NipServiceId, carrier.NipStreamProviderName);
string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_gateway_configuration_transport_sessions", jsonFileName);
FileInfo fi = new FileInfo(path);
return fi.Exists;
}
public void DvbNipInsertMulticastGatewayConfigurationTransportSession(NipActualCarrierInformation carrier,
MulticastEndpointAddressType multicastGatewayConfigurationTransportSession)
{
throw new NotImplementedException();
string jsonFileName = String.Format("{0}_{1}_{2}_{3}_{4}.json", carrier.NipNetworkId, carrier.NipCarrierId, carrier.NipLinkId, carrier.NipServiceId, carrier.NipStreamProviderName);
string path = Path.Combine(rootDirectory.FullName, "DVB-NIP", "multicast_gateway_configuration_transport_sessions", jsonFileName);
FileInfo fi = new FileInfo(path);
fi.Directory.EnsureExists();
string json =
JsonConvert.SerializeObject(multicastGatewayConfigurationTransportSession, Formatting.Indented);
File.WriteAllText(fi.FullName, json);
}
public bool DvbNipTestForCarrier(NipActualCarrierInformation currentCarrierInformation)
@ -1690,12 +1731,21 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
public DateTime GetLastDvbiServiceListEntryPointUpdateDate(long sourceHash)
{
throw new NotImplementedException();
string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag");
FileInfo fi = new FileInfo(path);
if (!fi.Exists)
{
return DateTime.MinValue;
}
return fi.LastWriteTime;
}
public void InsertDvbiServiceListEntryPoint(long sourceHash)
{
throw new NotImplementedException();
string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag");
FileInfo fi = new FileInfo(path);
fi.Directory.EnsureExists();
File.WriteAllText(fi.FullName, "");
}
public bool TestForServiceListEntryPoints(long sourceHash)
@ -1710,7 +1760,12 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
public void InsertDvbiServiceList(DvbiServiceList serviceList)
{
throw new NotImplementedException();
string jsonFileName = CleanFilename(serviceList.Name) + ".json";
string path = Path.Combine(rootDirectory.FullName, "DVB-I", "services", jsonFileName);
FileInfo fi = new FileInfo(path);
fi.Directory.EnsureExists();
string json = JsonConvert.SerializeObject(serviceList, Formatting.Indented);
File.WriteAllText(fi.FullName, json);
}
public bool TestForDvbiService(string id)
@ -1720,7 +1775,9 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
public bool TestForDvbiServiceListEntryPoints(long sourceHash)
{
throw new NotImplementedException();
string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_list_entry_points", sourceHash.ToString() + ".flag");
FileInfo fi = new FileInfo(path);
return fi.Exists;
}
public void UpdateDvbiServiceListLastCheckedDate(string id, DateTime currentTime)
@ -1760,7 +1817,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
public bool TestForDvbiServiceList(string id)
{
throw new NotImplementedException();
string jsonFileName = CleanFilename(id) + ".json";
string path = Path.Combine(rootDirectory.FullName, "DVB-I", "service_lists", id);
FileInfo fi = new FileInfo(path);
return fi.Exists;
}
}
}

28
skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs
View File

@ -861,22 +861,44 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
//Since we're working solely in memory, we won't need to remember this.
}
class T2MiTransmitterData
{
public ushort TimeOffset { get; set; }
}
private Dictionary<Tuple<int?, int?, int, ushort>,T2MiTransmitterData> t2miTransmitters;
public bool T2MiTestForTransmitter(int? currentNetworkId, int? currentTransportStreamId, int relatedPid,
ushort txIdentifier)
{
throw new NotImplementedException();
if (t2miTransmitters == null)
return false;
Tuple<int?, int?, int, ushort> t2miTransmitterId = new Tuple<int?, int?, int, ushort>(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier);
return t2miTransmitters.ContainsKey(t2miTransmitterId);
}
public void T2MiRememberTransmitter(int? currentNetworkId, int? currentTransportStreamId, int relatedPid,
ushort txIdentifier)
{
throw new NotImplementedException();
Tuple<int?, int?, int, ushort> t2miTransmitterId =
new Tuple<int?, int?, int, ushort>(currentNetworkId, currentTransportStreamId, relatedPid,
txIdentifier);
if (t2miTransmitters == null)
t2miTransmitters = new Dictionary<Tuple<int?, int?, int, ushort>, T2MiTransmitterData>();
if (t2miTransmitters.ContainsKey(t2miTransmitterId))
return;
t2miTransmitters.Add(t2miTransmitterId, new T2MiTransmitterData());
}
public void T2MiSetTransmitterTimeOffset(int? currentNetworkId, int? currentTransportStreamId, int relatedPid,
ushort txIdentifier, ushort timeOffset)
{
throw new NotImplementedException();
T2MiRememberTransmitter(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier);
Tuple<int?, int?, int, ushort> t2miTransmitterId = new Tuple<int?, int?, int, ushort>(currentNetworkId, currentTransportStreamId, relatedPid, txIdentifier);
t2miTransmitters[t2miTransmitterId].TimeOffset = timeOffset;
}
private List<LnbType> _lnbTypes;

10
skyscraper8/skyscraper8.csproj
View File

@ -6,6 +6,16 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<AssemblyVersion>8.0.*</AssemblyVersion>
<FileVersion>5.6.7.8</FileVersion>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<Deterministic>False</Deterministic>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<Deterministic>False</Deterministic>
</PropertyGroup>
<ItemGroup>