From 9be1ed8fee9901c5863b5d083a5f3d0be894dd97 Mon Sep 17 00:00:00 2001 From: ft Date: Thu, 9 Oct 2025 12:07:15 +0000 Subject: [PATCH 01/10] .gitea/workflows/demo.yaml aktualisiert --- .gitea/workflows/demo.yaml | 39 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/.gitea/workflows/demo.yaml b/.gitea/workflows/demo.yaml index b600ec1..8a0a686 100644 --- a/.gitea/workflows/demo.yaml +++ b/.gitea/workflows/demo.yaml @@ -1,45 +1,36 @@ -name: Gitea Actions Demo -run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀 +name: 🚀 Pack skyscraper8 +run-name: 👷 ${{ gitea.actor }} is building skyscraper8 on: [push] jobs: - Explore-Gitea-Actions: + make-zip: runs-on: self-hosted steps: - - run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!" - - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}." - - name: Check out repository code + - name: 📥 Check out repository code uses: actions/checkout@v4 - - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner." - - run: echo "🖥️ The workflow is now ready to test your code on the runner." - - name: List files in the repository - run: | - ls ${{ gitea.workspace }} - - name: Show current working directory - working-directory: ${{ gitea.workspace }} - run: | - pwd - ls -d $(pwd -P)/* - - name: Building the main assembly... + + - name: 🛠️ Building the main assembly... working-directory: ${{ gitea.workspace }} run: | dotnet build /p:EnableWindowsTargeting=true - - name: Typesetting the readme file... + + - name: 📄 Typesetting the readme file... working-directory: ${{ gitea.workspace }}/skyscraper8.Manual run: | pdflatex skyscraper8.Manual.tex - - name: Packing the main assembly... + + - name: 📦 Packing the main assembly... working-directory: ${{ gitea.workspace }}/skyscraper8/bin/Debug/net8.0 run: | 7zz a ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip * - - name: Packing the manual... + + - name: 📚 Packing the manual... working-directory: ${{ gitea.workspace }}/skyscraper8.Manual run: | 7zz a ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip skyscraper8.Manual.pdf - - name: Uploading the Package... + + - name: ☁️ Uploading the Package... uses: akkuman/gitea-release-action@v1 with: files: |- - ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip - - run: echo "🍏 This job's status is ${{ job.status }}." + ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip \ No newline at end of file -- 2.49.1 From f4a3ba73033ae293c54bb3c0fcdb766d653740ad Mon Sep 17 00:00:00 2001 From: ft Date: Thu, 9 Oct 2025 12:22:26 +0000 Subject: [PATCH 02/10] .gitea/workflows/demo.yaml aktualisiert --- .gitea/workflows/demo.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitea/workflows/demo.yaml b/.gitea/workflows/demo.yaml index 8a0a686..5ba17c5 100644 --- a/.gitea/workflows/demo.yaml +++ b/.gitea/workflows/demo.yaml @@ -32,5 +32,8 @@ jobs: - name: ☁️ Uploading the Package... uses: akkuman/gitea-release-action@v1 with: + name: ${{ gitea.sha }} + md5sum: true + sha256sum: true files: |- ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip \ No newline at end of file -- 2.49.1 From cffcd5b9e3c5f84c55d042232ea0821fe534f1f3 Mon Sep 17 00:00:00 2001 From: ft Date: Thu, 9 Oct 2025 14:01:18 +0000 Subject: [PATCH 03/10] skyscraper8.Manual/skyscraper8.Manual.tex aktualisiert --- skyscraper8.Manual/skyscraper8.Manual.tex | 42 +++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/skyscraper8.Manual/skyscraper8.Manual.tex b/skyscraper8.Manual/skyscraper8.Manual.tex index 9711f74..44abe85 100644 --- a/skyscraper8.Manual/skyscraper8.Manual.tex +++ b/skyscraper8.Manual/skyscraper8.Manual.tex @@ -60,7 +60,45 @@ Use TSDuck's \codeword{tsp} and use the \codeword{dvb} input plugin and the \cod See the paragraph about BDA tuners. The same applies to video4linux compatible tuners. TSDuck's \codeword{tsp} can be used on Windows and Linux in the same way. \subsection{Using a SAT\textgreater IP capable tuner} -This feature is intended for devices like the Digital Devices Octopus NET or the Kathrein EXIP 418. If you're using a PCI/PCIe tuner card or an USB Tuner with your PC, these passages about SAT\textgreater IP do not apply to you. +This feature is intended for SAT\textgreater IP server devices like the Digital Devices Octopus NET or the Kathrein EXIP 418. If you're using a PCI/PCIe tuner card or an USB Tuner with your PC, these passages about SAT\textgreater IP do not apply to you. + +To use skyscraper8 with a SAT\textgreater IP device, open up a shell and use the following command scheme: + +\begin{verbatim} +.\skyscraper8.exe satip IP_ADDRESS DISEQC POLARITY FREQUENCY SYSTEM SYMBOL_RATE +\end{verbatim} + +Replace these variables as follows: +\\ +\begin{tabular}{llp{5cm}} +\hline +IP\_ADDRESS & \parbox{13.5cm}{The IP address (for example 172.20.20.122) of your SAT\textgreater IP server, or "auto" for autodetection of a SAT\textgreater IP server.} \\ +\hline +DISEQC & \parbox{13.5cm}{The DiSEqC 1.0 Port you want to use. Say "1" here for A/A, "2" for B/A, "3" for A/B, 4 for B/B} \\ +\hline +POLARITY & \parbox{13.5cm}{Either "H" or "V". The SAT\textgreater IP specification also allows "L" and "R", but this is not yet implemented in skyscraper8.} \\ +\hline +FREQUENCY & \parbox{13.5cm}{The frequency you want to tune to in MHz. (for example 12226)} \\ +\hline +SYSTEM & \parbox{13.5cm}{SYSTEM is either "S" or "S2". The SAT\textgreater IP specification does not consider S2X, but in my testings, an S2X capable device like the Octopus NET also accepts "S2" here for S2X transponders. } \\ +\hline +SYMBOL\_RATE & \parbox{13.5cm}{The symbol rate of the transponder you want to tune to in Ks. (for example 27500)} \\ +\hline +\end{tabular}\\ + +So a valid command would be for example: + +\begin{verbatim} +.\skyscraper8.exe satip 172.20.20.122 2 V 12226 S2 27500 +\end{verbatim} +to tune to the Hotbird DVB-NIP demo transponder, assuming Hotbird is at DiSEqC B/A. \\ + +If you don't know your SAT\textgreater IP server address, the following command would accomplish the same: +\begin{verbatim} +.\skyscraper8.exe satip auto 2 V 12226 S2 27500" +\end{verbatim} + +If your SAT\textgreater IP server features a STiD135 chipset, you can also catch GS/GSE with it. \subsection{Using prerecorded TS files} This is probably the simplest way to get a TS into skyscraper8. Just run the skyscraper8.exe from the commandline passing the path to your TS file as an argument. @@ -81,7 +119,7 @@ If you are using Microsoft Windows, you can also Drag’n’Drop a TS file onto \section{What skyscraper8 can not do} \begin{itemize} - \item Watch TV or Listen to Radio. Extracting files and playing back audio/video are very different things, and skyscraper8 is not designed to do the latter. If this is something you're looking for, you probably want something like ProgDVB or DVBViewer. + \item Watch TV or Listen to Radio. Extracting files and playing back audio/video are two very different things, and skyscraper8 is not designed to do the latter. If this is something you're looking for, you probably want something like ProgDVB or DVBViewer. \item Descramble encrypted channels. This is illegal in most, if not all, jurisdictions, and I want nothing to with that. \item Circumvent copy protections. TS contained on Blu-Ray disc are usually scrambled using AACS. Although not too difficult, bypassing it is also a complicated legal matter, which is why I prefer not doing it. \item Descramble encrypted DOCSIS\footnote{Data-Over-Cable Service Interface Specifications} Traffic. While skyscraper8 does understand DOCSIS traffic and is perfectly capable of sniffing packets from it using tuners like the TBS-6281 SE, it will drop any scrambled packets - meaning those that have the DOCSIS privacy extension enabled. -- 2.49.1 From 5bbe8e9dc07556838cdf5cbc124c242cc98eeb1a Mon Sep 17 00:00:00 2001 From: ft Date: Thu, 9 Oct 2025 14:30:02 +0000 Subject: [PATCH 04/10] skyscraper8.Manual/skyscraper8.Manual.tex aktualisiert --- skyscraper8.Manual/skyscraper8.Manual.tex | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/skyscraper8.Manual/skyscraper8.Manual.tex b/skyscraper8.Manual/skyscraper8.Manual.tex index 44abe85..26fd9f9 100644 --- a/skyscraper8.Manual/skyscraper8.Manual.tex +++ b/skyscraper8.Manual/skyscraper8.Manual.tex @@ -5,6 +5,7 @@ \usepackage{amsfonts} \usepackage{amssymb} \usepackage{xcolor} + \usepackage{geometry} \geometry{ a4paper, @@ -13,8 +14,19 @@ top=20mm, } +\usepackage{hyperref} +\hypersetup{ + colorlinks=true, + linkcolor=blue, + filecolor=magenta, + urlcolor=cyan, + pdftitle={skyscraper8 Manual}, + pdfpagemode=FullScreen, + } + + \author{Fey} -\title{skyscraper8} +\title{skyscraper8 \\ A hobbyist project to learn about MPEG-II} \date{October 2025} \NewDocumentCommand{\codeword}{v}{% @@ -130,8 +142,8 @@ If you are using Microsoft Windows, you can also Drag’n’Drop a TS file onto \section{How skyscraper8 was made} skyscraper8 is developed in the C\# programming language, using Microsoft Visual Studio 2022. Therefore it requires a Microsoft .NET runtime, which is included in Microsoft Windows and freely available for most Linux distributions and Apple's macOS. -The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This is on purpose. If you want to study how skyscraper8 works under the hood, I hereby allow you to use tools like RedGate's Reflector or JetBrains' dotPeek to inspect the skyscraper8.dll file. I plan to release the clear source code at a later date. +The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This is on purpose. If you want to study how skyscraper8 works under the hood, I hereby allow you to use tools like RedGate's Reflector or JetBrains' dotPeek to inspect the skyscraper8.dll file. Of course you can also acquire the source code from my Gitea instance.\footnote{\url{https://gitea.yo3explorer.moe/ft/skyscraper8}} -This document was written in \LaTeX{}, and while I wrote it myself, I did use GPT-5 for proofreading. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM. +This document was typeset in \LaTeX{}, using the TeX Live distribution, and while I wrote it myself, I did use GPT-5 for proofreading. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM. \end{document} \ No newline at end of file -- 2.49.1 From b3f92bf407e479836aff61b13d6a0f0bb6d9061e Mon Sep 17 00:00:00 2001 From: ft Date: Fri, 10 Oct 2025 11:25:10 +0000 Subject: [PATCH 05/10] Added notice about GS to the manual. --- skyscraper8.Manual/skyscraper8.Manual.tex | 31 +++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/skyscraper8.Manual/skyscraper8.Manual.tex b/skyscraper8.Manual/skyscraper8.Manual.tex index 26fd9f9..f47441a 100644 --- a/skyscraper8.Manual/skyscraper8.Manual.tex +++ b/skyscraper8.Manual/skyscraper8.Manual.tex @@ -5,6 +5,8 @@ \usepackage{amsfonts} \usepackage{amssymb} \usepackage{xcolor} +\usepackage{pifont,mdframed} + \usepackage{geometry} \geometry{ @@ -33,6 +35,13 @@ \texttt{\textcolor{blue}{#1}}% } +\newenvironment{warning} + {\par\begin{mdframed}[linewidth=2pt,linecolor=red]% + \begin{list}{}{\leftmargin=1cm + \labelwidth=\leftmargin}\item[\Large\ding{43}]\textbf{IMPORTANT:}} + {\end{list}\end{mdframed}\par} + + \begin{document} \maketitle @@ -55,6 +64,7 @@ This is the niche gap skyscraper8 intents to cover: Extracting files transmitted \section{How to get TS into skyscraper8} skyscraper8 is a command-line operated tool. A graphical user interface is planned, but not yet fully developed. \\ This means that you have to use the command-line interpreter of your operating system to use skyscraper8. On Windows, both cmd.exe and PowerShell are known to work well. +On Linux and macOS, any halfway recent shell should be fine. \subsection{Using a StreamReader compatible tuner} \textbf{This is the preferred way} and applies to tuners like the popular TBS5927 or TBS6903x. @@ -64,6 +74,22 @@ This means that you have to use the command-line interpreter of your operating s .\skyscraper8.exe cscan tcp://127.0.0.1:6969 \end{verbatim} +\label{sec:gswarning} +\begin{warning} +If you're planning to work with GS, make sure to disable StreamReader.dll's internal deencapsulator. You can do this by keeping the semicolon in StreamReader.ini like this +\begin{verbatim} +;FrameMode=1 +\end{verbatim} +or set it to 0 like this: +\begin{verbatim} +FrameMode=0 +\end{verbatim} + +Why is that? While StreamReader.dll's Deencapsulation works correctly in theory, it does not seem to check whether an encapsulated packet actually contains valid MPEG-II packets and will happily try to deencapsulate all other packets as well, causing a lot of Packets with invalid data to appear in the outputted TS. +\end{warning} + + + \subsection{Using a BDA compatible tuner} Use TSDuck's \codeword{tsp} and use the \codeword{dvb} input plugin and the \codeword{ip} output plugin. \\ TODO: Talk about TSDuck here. @@ -137,13 +163,14 @@ If you are using Microsoft Windows, you can also Drag’n’Drop a TS file onto \item Descramble encrypted DOCSIS\footnote{Data-Over-Cable Service Interface Specifications} Traffic. While skyscraper8 does understand DOCSIS traffic and is perfectly capable of sniffing packets from it using tuners like the TBS-6281 SE, it will drop any scrambled packets - meaning those that have the DOCSIS privacy extension enabled. \item Do a professional grade analysis of a TS. There are many intricacies in broadcasting systems which are out of scope for a simple file extractor developed by a mere hobbyist. If you're looking for a professional grade analyzing tool, I'd recommend you to check out the tools included in VMA Video Analyzer, Dektec's StreamXpert, GkWare's StreamGuru, or the \codeword{analyze} plugin of TSDuck. \item Handle MPEG-DASH streams. Currently, there are two known video formats used in DVB-NIP: HLS and MPEG-DASH. While skyscraper8 is able to extract segments of both types from a TS, I do not know of an easy way to playback MPEG-DASH formatted streams. HLS on the other hand is trivial. + \item Repair broken GS. When using a non STiD135-based tuner to tune to a GS, the framing gets broken in extremely weird ways. Also, when using StreamReader.dll to tune to a GS with a STiD135 - it is imperative to disable to StreamReader.dll's deencapsulator. It often breaks the framing as well. See \hyperref[sec:gswarning]{the notice about GS in Section 2.1} \end{itemize} \section{How skyscraper8 was made} skyscraper8 is developed in the C\# programming language, using Microsoft Visual Studio 2022. Therefore it requires a Microsoft .NET runtime, which is included in Microsoft Windows and freely available for most Linux distributions and Apple's macOS. -The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This is on purpose. If you want to study how skyscraper8 works under the hood, I hereby allow you to use tools like RedGate's Reflector or JetBrains' dotPeek to inspect the skyscraper8.dll file. Of course you can also acquire the source code from my Gitea instance.\footnote{\url{https://gitea.yo3explorer.moe/ft/skyscraper8}} +The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This is on purpose. If you want to study how skyscraper8 works under the hood, I hereby allow you to use tools like RedGate's Reflector or JetBrains' dotPeek to inspect the skyscraper8.dll file. Of course you can also acquire the plain source code from my Gitea instance.\footnote{\url{https://gitea.yo3explorer.moe/ft/skyscraper8}} -This document was typeset in \LaTeX{}, using the TeX Live distribution, and while I wrote it myself, I did use GPT-5 for proofreading. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM. +This document was typeset in \LaTeX{}, using the TeX Live distribution, and while I wrote it myself, I did use GPT-5 for proofreading the initial version of this document. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM. \end{document} \ No newline at end of file -- 2.49.1 From 39f8e936f35a6b88d860bb4664f2f49384f3c7ba Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Sun, 12 Oct 2025 20:29:50 +0200 Subject: [PATCH 06/10] Added partial support for continuous GS. --- skyscraper8/DvbNip/DvbNipReceiver.cs | 2 +- .../BbframeDeencapsulator.cs | 86 ++++++++-- skyscraper8/GSE/GseFragmentation.cs | 26 +++ skyscraper8/GSE/GsePacket.cs | 156 ++++++++++++++++++ skyscraper8/Properties/launchSettings.json | 2 +- .../Skyscraper/DigitalDevicesBbFrameReader.cs | 2 +- 6 files changed, 259 insertions(+), 15 deletions(-) rename skyscraper8/{Skyscraper => GSE}/BbframeDeencapsulator.cs (80%) create mode 100644 skyscraper8/GSE/GseFragmentation.cs create mode 100644 skyscraper8/GSE/GsePacket.cs diff --git a/skyscraper8/DvbNip/DvbNipReceiver.cs b/skyscraper8/DvbNip/DvbNipReceiver.cs index b10ca4f..ed89f9d 100644 --- a/skyscraper8/DvbNip/DvbNipReceiver.cs +++ b/skyscraper8/DvbNip/DvbNipReceiver.cs @@ -108,9 +108,9 @@ namespace skyscraper8.DvbNip if (isValidXml) { FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream); - EventHandler.FluteFileAnnouncement(fluteCoordinate.Item1, fluteCoordinate.Item2, fdtAnnouncement); if (fdtAnnouncement != null) { + EventHandler.FluteFileAnnouncement(fluteCoordinate.Item1, fluteCoordinate.Item2, fdtAnnouncement); SetFileAssociations(fluteListener, fdtAnnouncement); } } diff --git a/skyscraper8/Skyscraper/BbframeDeencapsulator.cs b/skyscraper8/GSE/BbframeDeencapsulator.cs similarity index 80% rename from skyscraper8/Skyscraper/BbframeDeencapsulator.cs rename to skyscraper8/GSE/BbframeDeencapsulator.cs index 83a24cb..3251415 100644 --- a/skyscraper8/Skyscraper/BbframeDeencapsulator.cs +++ b/skyscraper8/GSE/BbframeDeencapsulator.cs @@ -11,7 +11,7 @@ using System.Text; using System.Threading.Tasks; using log4net; -namespace skyscraper8.Skyscraper +namespace skyscraper8.GSE { internal class BbframeDeencapsulator { @@ -22,6 +22,8 @@ namespace skyscraper8.Skyscraper private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private bool shownNonGseWarning; + public void PushPacket(byte[] bbframe) { MemoryStream ms = new MemoryStream(bbframe, false); @@ -32,7 +34,7 @@ namespace skyscraper8.Skyscraper bool ccmAcmField = (matype1 & 0x10) != 0; bool issyi = (matype1 & 0x08) != 0; bool npd = (matype1 & 0x04) != 0; - int ro = (matype1 & 0x03); + int ro = matype1 & 0x03; byte matype2 = ms.ReadUInt8(); @@ -42,25 +44,84 @@ namespace skyscraper8.Skyscraper ushort syncd = ms.ReadUInt16BE(); byte crc8 = ms.ReadUInt8(); + if (userPacketLength == 0 && dataFieldLength == 0) + return; + switch (tsGsField) { - case 2: + case 1: + if (sync != 0) + { + if (!shownNonGseWarning) + { + logger.WarnFormat("This stream is a valid GS, but also contains packets which are not GSE (type 0), but of type {0} . Please share a sample of this stream!", sync); + shownNonGseWarning = true; + } + return; + } int bytes = dataFieldLength / 8; if (ms.GetAvailableBytes() < bytes) return; - HandleGse(ms); + HandleContinous(ms); + break; + case 2: + int hemBytes = dataFieldLength / 8; + if (ms.GetAvailableBytes() < hemBytes) + return; + HandleGseHem(ms); break; default: //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts - logger.Warn(String.Format("Unsupported: TS/GS field says 0x{0:X2}", tsGsField)); + logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}", tsGsField)); break; } } - private bool gseNeedMore; + private GseFragmentation[] gseFragmentations; + private void HandleContinous(MemoryStream ms) + { + ms.Position = 10; + + + while (ms.GetAvailableBytes() > 0) + { + GsePacket packet = GsePacket.Read(ms); + if (packet.IsPadding) + { + break; + } + + if (packet.IsCompletePacket) + { + if (ValidateEthertype(packet.ProtocolType.Value)) + { + MpeEventHandler.OnIpDatagram(PID, packet.GseData); + } + else + { + logger.WarnFormat("Unknown EtherType: 0x{0:X4}", packet.ProtocolType.Value); + } + continue; + } + + if (packet.StartIndicator && !packet.EndIndicator) + { + if (gseFragmentations == null) + gseFragmentations = new GseFragmentation[256]; + + gseFragmentations[packet.FragmentId.Value] = new GseFragmentation(); + gseFragmentations[packet.FragmentId.Value].AddPacket(packet); + continue; + } + + throw new NotImplementedException(); + } + } + + private bool gseNeedMore; private MemoryStream gseAssembler; public int PID; - private void HandleGse(MemoryStream ms) + private void HandleGseHem(MemoryStream ms) { ms.Position = 10; byte syncByte = ms.ReadUInt8(); @@ -101,24 +162,24 @@ namespace skyscraper8.Skyscraper } gseAssembler = null; - HandleGse(ms); + HandleGseHem(ms); return; } else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE) { gseAssembler = null; - HandleGse(ms); + HandleGseHem(ms); return; } else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH) { gseAssembler = null; - HandleGse(ms); + HandleGseHem(ms); return; } else { - throw new NotImplementedException(String.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte)); + throw new NotImplementedException(string.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte)); } } @@ -130,7 +191,8 @@ namespace skyscraper8.Skyscraper BROKEN_NEGATIVE_LENGTH, NEED_MORE_DATA, BROKEN_SUSPECT_LENGTH, - BROKEN_SUSPECT_ETHERTYPE + BROKEN_SUSPECT_ETHERTYPE, + VALID_NULL_PACKET, //somehow these only show up on TBS6903x, not DD Max SX8? } private GseAssemblyState ValidateGse(MemoryStream ms) diff --git a/skyscraper8/GSE/GseFragmentation.cs b/skyscraper8/GSE/GseFragmentation.cs new file mode 100644 index 0000000..5d6d700 --- /dev/null +++ b/skyscraper8/GSE/GseFragmentation.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.GSE +{ + internal class GseFragmentation + { + public void AddPacket(GsePacket packet) + { + if (spanningGsePackets == null) + { + spanningGsePackets = new List(); + TotalLength = packet.TotalLength.Value; + } + + spanningGsePackets.Add(packet); + } + + public ushort TotalLength { get; private set; } + + private List spanningGsePackets; + } +} diff --git a/skyscraper8/GSE/GsePacket.cs b/skyscraper8/GSE/GsePacket.cs new file mode 100644 index 0000000..da4f130 --- /dev/null +++ b/skyscraper8/GSE/GsePacket.cs @@ -0,0 +1,156 @@ +using skyscraper5.Skyscraper.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.NetworkInformation; +using System.Text; +using System.Threading.Tasks; +using skyscraper5.Skyscraper; + +namespace skyscraper8.GSE +{ + internal class GsePacket + { + public static GsePacket Read(MemoryStream ms) + { + GsePacket packet = new GsePacket(); + + byte byteA = ms.ReadUInt8(); + packet.StartIndicator = (byteA & 0x80) != 0; + packet.EndIndicator = (byteA & 0x40) != 0; + packet.LabelIndicator = (byteA & 0x30) >> 4; + if (!packet.StartIndicator && !packet.EndIndicator && packet.LabelIndicator == 0) + { + //End of baseband frame + return packet; + } + else + { + byte byteB = ms.ReadUInt8(); + int gseLength = byteA & 0x0f; + gseLength <<= 8; + gseLength += byteB; + + if (!packet.StartIndicator || !packet.EndIndicator) + { + packet.FragmentId = ms.ReadUInt8(); + gseLength--; + } + + if (packet.StartIndicator && !packet.EndIndicator) + { + packet.TotalLength = ms.ReadUInt16BE(); + gseLength -= 2; + } + + if (packet.StartIndicator) + { + packet.ProtocolType = ms.ReadUInt16BE(); + gseLength -= 2; + if (packet.LabelIndicator == 0) + { + packet.Label = new SixByteGseLabel(ms.ReadBytes(6)); + gseLength -= 6; + } + else if (packet.LabelIndicator == 1) + { + packet.Label = new ThreeByteGseLabel(ms.ReadBytes(3)); + gseLength -= 3; + } + } + + if (!packet.StartIndicator && packet.EndIndicator) + gseLength -= 4; + + packet.GseData = ms.ReadBytes(gseLength); + + if (!packet.StartIndicator && packet.EndIndicator) + packet.Crc32 = ms.ReadUInt32BE(); + } + + return packet; + } + + public uint? Crc32 { get; set; } + + public byte[] GseData { get; set; } + + public GseLabel Label { get; set; } + public ushort? ProtocolType { get; set; } + + public ushort? TotalLength { get; set; } + public byte? FragmentId { get; set; } + public int LabelIndicator { get; private set; } + public bool EndIndicator { get; private set; } + public bool StartIndicator { get; private set; } + public bool IsPadding + { + get + { + return !StartIndicator && !EndIndicator && LabelIndicator == 0; + } + } + + public bool IsCompletePacket + { + get + { + return StartIndicator && EndIndicator; + } + } + } + + public abstract class GseLabel : Validatable + { + public abstract int LabelLength { get; } + public abstract string ToString(); + } + + public class ThreeByteGseLabel : GseLabel + { + public ThreeByteGseLabel(byte[] buffer) + { + if (buffer.Length != 3) + { + Valid = false; + return; + } + + LabelBytes = buffer; + Valid = true; + } + + public byte[] LabelBytes { get; private set; } + + public override string ToString() + { + return BitConverter.ToString(LabelBytes); + } + + public override int LabelLength => 3; + } + + public class SixByteGseLabel : GseLabel + { + public SixByteGseLabel(byte[] buffer) + { + if (buffer.Length != 6) + { + Valid = false; + return; + } + + MacAddress = new PhysicalAddress(buffer); + Valid = true; + } + + public PhysicalAddress MacAddress { get; private set; } + + public override string ToString() + { + return MacAddress.ToString(); + } + + public override int LabelLength => 6; + } +} diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index df6935c..d7ff207 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "satip auto 1 H 11141 S2 23500", + "commandLineArgs": "\"\\\\utena\\mergerfs\\Skyscraper\\ipProto253_eutelsat7_10803h-max-sx8.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs b/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs index 7787b96..54ff52c 100644 --- a/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs +++ b/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs @@ -6,7 +6,7 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; -using skyscraper8.Skyscraper; +using skyscraper8.GSE; namespace skyscraper5.src.Skyscraper { -- 2.49.1 From 3da10ae897f8ad8d511c0619f64a45e5f80153cf Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Sun, 12 Oct 2025 22:03:52 +0200 Subject: [PATCH 07/10] Removed the discrimination between "TBS-style GS encapsulation" and "Digital Devices style GS encapsulation" as it was factually wrong. --- skyscraper8/GSE/BbframeDeencapsulator.cs | 2 +- .../Stid135BbFrameReader.cs} | 11 ++++---- ...GsReader.cs => OldStreamReaderDetector.cs} | 28 ++++++++++--------- .../Skyscraper/Scraper/SkyscraperContext.cs | 18 +++++------- 4 files changed, 28 insertions(+), 31 deletions(-) rename skyscraper8/{Skyscraper/DigitalDevicesBbFrameReader.cs => GSE/Stid135BbFrameReader.cs} (82%) rename skyscraper8/Skyscraper/{Tbs6903xGsReader.cs => OldStreamReaderDetector.cs} (68%) diff --git a/skyscraper8/GSE/BbframeDeencapsulator.cs b/skyscraper8/GSE/BbframeDeencapsulator.cs index 3251415..e933f9e 100644 --- a/skyscraper8/GSE/BbframeDeencapsulator.cs +++ b/skyscraper8/GSE/BbframeDeencapsulator.cs @@ -71,7 +71,7 @@ namespace skyscraper8.GSE HandleGseHem(ms); break; default: //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts - logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}", tsGsField)); + logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}, please share a sample of this stream.", tsGsField)); break; } } diff --git a/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs b/skyscraper8/GSE/Stid135BbFrameReader.cs similarity index 82% rename from skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs rename to skyscraper8/GSE/Stid135BbFrameReader.cs index 54ff52c..c56264e 100644 --- a/skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs +++ b/skyscraper8/GSE/Stid135BbFrameReader.cs @@ -6,16 +6,15 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; -using skyscraper8.GSE; -namespace skyscraper5.src.Skyscraper +namespace skyscraper8.GSE { - internal class DigitalDevicesBbFrameReader : ITsPacketProcessor + internal class Stid135BbFrameReader : ITsPacketProcessor { private IGsEventHandler mpeEventHandler; private BbframeDeencapsulator deencapsulator; - public DigitalDevicesBbFrameReader(IGsEventHandler mpeEventHandler) + public Stid135BbFrameReader(IGsEventHandler mpeEventHandler) { this.mpeEventHandler = mpeEventHandler; } @@ -27,10 +26,10 @@ namespace skyscraper5.src.Skyscraper public void PushPacket(TsPacket packet) { byte[] packets = packet.RawPacket; - int pid = (packets[1]); + int pid = packets[1]; pid &= 0x01f; pid <<= 8; - pid |= (packets[2]); + pid |= packets[2]; if (pid != 0x010e) return; diff --git a/skyscraper8/Skyscraper/Tbs6903xGsReader.cs b/skyscraper8/Skyscraper/OldStreamReaderDetector.cs similarity index 68% rename from skyscraper8/Skyscraper/Tbs6903xGsReader.cs rename to skyscraper8/Skyscraper/OldStreamReaderDetector.cs index 5fa0782..54cc6a7 100644 --- a/skyscraper8/Skyscraper/Tbs6903xGsReader.cs +++ b/skyscraper8/Skyscraper/OldStreamReaderDetector.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using log4net; using skyscraper5.Dvb; using skyscraper5.Dvb.DataBroadcasting; using skyscraper5.Mpeg2; @@ -12,13 +13,12 @@ using skyscraper5.Skyscraper.IO; namespace skyscraper5.Skyscraper { - internal class Tbs6903xGsReader : ITsPacketProcessor + internal class OldStreamReaderDetector : ITsPacketProcessor { - private readonly IGsEventHandler mpeEventHandler; + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); - public Tbs6903xGsReader(IGsEventHandler mpeEventHandler) + public OldStreamReaderDetector() { - this.mpeEventHandler = mpeEventHandler; } private MemoryStream outbuf; @@ -27,13 +27,16 @@ namespace skyscraper5.Skyscraper public void PushPacket(TsPacket packet) { - byte[] packets = packet.RawPacket; - int pid = (packets[1]); - pid &= 0x01f; - pid <<= 8; - pid |= (packets[2]); - if (pid != 0x0118) - return; + if (annoncementDone) + return; + + byte[] packets = packet.RawPacket; + int pid = (packets[1]); + pid &= 0x01f; + pid <<= 8; + pid |= (packets[2]); + if (pid != 0x0118) + return; if ((packets[8] & 0xff) == 0xd0) { @@ -45,10 +48,9 @@ namespace skyscraper5.Skyscraper { if (!annoncementDone) { - mpeEventHandler.GsIpTrafficDetected(); + logger.WarnFormat("It seems like you're using a StreamReader.dll version which interferes with the STiD135's BBFrame encapsulation. Try updating to 1.2.5.208, this is known to work."); annoncementDone = true; } - mpeEventHandler.OnIpDatagram(0x0118, ipPacket); } } outbuf = new MemoryStream(); diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index bc3e32f..296c47d 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -78,6 +78,7 @@ using System.Security.Policy; using System.Text; using skyscraper8.Experimentals.NdsSsu; using skyscraper8.Experimentals.OtvSsu; +using skyscraper8.GSE; using skyscraper8.Skyscraper.Net; using skyscraper8.Skyscraper.Scraper; using Tsubasa.IO; @@ -293,16 +294,14 @@ namespace skyscraper5.Skyscraper.Scraper LogEvent(SkyscraperContextEvent.StartPacketProcessing); if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x18 && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00) { - DvbContext.RegisterPacketProcessor(0x0118, new Tbs6903xGsReader(this)); - UiJunction?.SetGseMode(); - LogEvent(SkyscraperContextEvent.SpecialTsMode, "TBS 6903-X GSE Pseudo TS detected."); + DvbContext.RegisterPacketProcessor(0x0118, new OldStreamReaderDetector()); SpecialTsType = 2; } if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x0e && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00) { - DvbContext.RegisterPacketProcessor(0x010e, new DigitalDevicesBbFrameReader(this)); + DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this)); UiJunction?.SetGseMode(); - LogEvent(SkyscraperContextEvent.SpecialTsMode, "Digital Devices BBFrame Pseudo TS detected."); + LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); SpecialTsType = 3; } firstPacketDone = true; @@ -339,10 +338,7 @@ namespace skyscraper5.Skyscraper.Scraper { if (!DvbContext.IsPidProcessorPresent(0x0118)) { - DvbContext.RegisterPacketProcessor(0x0118, new Tbs6903xGsReader(this)); - UiJunction?.SetGseMode(); - LogEvent(SkyscraperContextEvent.SpecialTsMode, "TBS 6903-X GSE Pseudo TS detected."); - SpecialTsType = 2; + DvbContext.RegisterPacketProcessor(0x0118, new OldStreamReaderDetector()); return; } } @@ -357,9 +353,9 @@ namespace skyscraper5.Skyscraper.Scraper { if (!DvbContext.IsPidProcessorPresent(0x010e)) { - DvbContext.RegisterPacketProcessor(0x010e, new DigitalDevicesBbFrameReader(this)); + DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this)); UiJunction?.SetGseMode(); - LogEvent(SkyscraperContextEvent.SpecialTsMode, "Digital Devices BBFRAME data TS detected."); + LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); SpecialTsType = 3; return; } -- 2.49.1 From fe66c643f8cf2f965abebea4330fb6b67f108132 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Sun, 12 Oct 2025 23:29:51 +0200 Subject: [PATCH 08/10] Added a warning about older StreamReader versions to the manual. --- .gitignore | 5 ++++ skyscraper8.Manual/skyscraper8.Manual.tex | 31 ++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 749f173..7196d81 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,8 @@ imgui.ini /GUIs/skyscraper8.UI.ImGui.MonoGame/obj /.vs/skyscraper8/CopilotIndices/17.14.995.13737 /.vs/skyscraper8/config +/skyscraper8.Manual/skyscraper8.Manual.aux +/skyscraper8.Manual/skyscraper8.Manual.log +/skyscraper8.Manual/skyscraper8.Manual.out +/skyscraper8.Manual/skyscraper8.Manual.pdf +/skyscraper8.Manual/skyscraper8.Manual.synctex.gz diff --git a/skyscraper8.Manual/skyscraper8.Manual.tex b/skyscraper8.Manual/skyscraper8.Manual.tex index f47441a..f27f2a4 100644 --- a/skyscraper8.Manual/skyscraper8.Manual.tex +++ b/skyscraper8.Manual/skyscraper8.Manual.tex @@ -74,9 +74,12 @@ On Linux and macOS, any halfway recent shell should be fine. .\skyscraper8.exe cscan tcp://127.0.0.1:6969 \end{verbatim} -\label{sec:gswarning} -\begin{warning} -If you're planning to work with GS, make sure to disable StreamReader.dll's internal deencapsulator. You can do this by keeping the semicolon in StreamReader.ini like this +\subsubsection{Working with GS using StreamReader} +There are a few caveats when working with StreamReader.dll to capture GS. + +Some older versions of StreamReader.dll are known to interfere with the STiD135's BBFrame encapsulation feature. Version 1.2.4.101 is a prominent example of doing this. It changes the encapsulation PID and also seems to frame GSE frames differently. Therefore, please make sure to use a recent version of StreamReader.dll. Version 1.2.5.208 is known to work correctly. + +Also, make sure to disable StreamReader.dll's internal deencapsulator. You can do this by keeping the semicolon in StreamReader.ini like this \begin{verbatim} ;FrameMode=1 \end{verbatim} @@ -86,7 +89,6 @@ FrameMode=0 \end{verbatim} Why is that? While StreamReader.dll's Deencapsulation works correctly in theory, it does not seem to check whether an encapsulated packet actually contains valid MPEG-II packets and will happily try to deencapsulate all other packets as well, causing a lot of Packets with invalid data to appear in the outputted TS. -\end{warning} @@ -173,4 +175,25 @@ The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This This document was typeset in \LaTeX{}, using the TeX Live distribution, and while I wrote it myself, I did use GPT-5 for proofreading the initial version of this document. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM. +\subsection{Personal remarks and some useless bonus information} + +\subsubsection{Music!} +Like a lot of programmers, I do enjoy listening to music while working. Some programmers even put song references in their software. Like how MKVToolnix' version names are actually song names, or how BSD developer fk even put \href{https://www.fabiankeil.de/nutzloseinfos.html}{a list of albums on his website} listing what albums he listened to while making it. Although this is absolutely useless, I'd liketo do this as well. Therefore, here follows a list of musical albums I listened to while developing skyscraper8 - no claim to completeness. + +\begin{itemize} + \item Bel Canto - White-Out Conditions + \item Beautiful World - In Existance + \item Kitaro - Millennia + \item Macross 82-99 - A Million Miles Away + \item Megazone 23 Vocal Collection + \item Moodswings - Moodfood + \item Opus III - Guru Mother + \item Satsuki Shibano / Yoshio Ojima - Caresse + \item SHIFT UP - Stellar Blade Original Soundtrack + \item Small Affairs - Small Affairs + \item Yoshio Ojima - Hands-Some + \item Youssou N’Dour - The Guide (Wommat) + \item Yu-Gi-Oh! Sound Duel Quarter Century Collection +\end{itemize} + \end{document} \ No newline at end of file -- 2.49.1 From a46e4ff5568b025e98bdb73a538a452630cc555e Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Tue, 14 Oct 2025 07:08:27 +0200 Subject: [PATCH 09/10] Detect broken GS encapsulation. --- .gitignore | 1 + skyscraper8/GSE/BbframeDeencapsulator.cs | 60 ++++++++++++++++++++-- skyscraper8/GSE/GsePacket.cs | 13 ++++- skyscraper8/Mpeg2/TsContext.cs | 50 ++++++++++++++++-- skyscraper8/Properties/launchSettings.json | 2 +- 5 files changed, 117 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 7196d81..833cb57 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,4 @@ imgui.ini /skyscraper8.Manual/skyscraper8.Manual.out /skyscraper8.Manual/skyscraper8.Manual.pdf /skyscraper8.Manual/skyscraper8.Manual.synctex.gz +/.vs/skyscraper8/CopilotIndices/17.14.1231.31060 diff --git a/skyscraper8/GSE/BbframeDeencapsulator.cs b/skyscraper8/GSE/BbframeDeencapsulator.cs index e933f9e..ab2f788 100644 --- a/skyscraper8/GSE/BbframeDeencapsulator.cs +++ b/skyscraper8/GSE/BbframeDeencapsulator.cs @@ -90,6 +90,11 @@ namespace skyscraper8.GSE break; } + if (!packet.Valid) + { + break; + } + if (packet.IsCompletePacket) { if (ValidateEthertype(packet.ProtocolType.Value)) @@ -113,6 +118,25 @@ namespace skyscraper8.GSE continue; } + if (!packet.StartIndicator && packet.EndIndicator) + { + if (gseFragmentations == null) + continue; + if (gseFragmentations[packet.FragmentId.Value] == null) + continue; + + gseFragmentations[packet.FragmentId.Value].AddPacket(packet); + continue; + } + + if (!packet.StartIndicator && !packet.EndIndicator) + { + if (gseFragmentations == null) + continue; + + throw new NotImplementedException(); + } + throw new NotImplementedException(); } } @@ -129,8 +153,8 @@ namespace skyscraper8.GSE long availableBytes = ms.GetAvailableBytes(); byte[] readBytes = ms.ReadBytes(availableBytes); - - GseAssemblyState assemblyState = ValidateGse(gseAssembler); + + GseAssemblyState assemblyState = ValidateGse(gseAssembler, true); if (assemblyState == GseAssemblyState.NOT_YET_BEGUN) { if ((syncByte & 0xc0) == 0xc0) @@ -195,7 +219,7 @@ namespace skyscraper8.GSE VALID_NULL_PACKET, //somehow these only show up on TBS6903x, not DD Max SX8? } - private GseAssemblyState ValidateGse(MemoryStream ms) + private GseAssemblyState ValidateGse(MemoryStream ms, bool tryFix = false) { if (ms == null) return GseAssemblyState.NOT_YET_BEGUN; @@ -204,6 +228,8 @@ namespace skyscraper8.GSE ms.Position = 0; while (ms.GetAvailableBytes() > 2) { + long fixPosition = ms.Position; + //GSE-Header byte byteA = ms.ReadUInt8(); bool startIndicator = (byteA & 0x80) != 0; @@ -224,6 +250,12 @@ namespace skyscraper8.GSE //Console.WriteLine("GSE Length: {0}", gseLength); if (gseLength > 9000) { + if (tryFix) + { + ms.Position = fixPosition; + ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes()); + break; + } ms.Position = backupPosition; return GseAssemblyState.BROKEN_SUSPECT_LENGTH; } @@ -236,6 +268,11 @@ namespace skyscraper8.GSE if (startIndicator && !endIndicator) { + if (2 > ms.GetAvailableBytes()) + { + ms.Position = backupPosition; + return GseAssemblyState.NEED_MORE_DATA; + } ushort totalLength = ms.ReadUInt16BE(); gseLength -= 2; } @@ -250,6 +287,12 @@ namespace skyscraper8.GSE ushort protocolType = ms.ReadUInt16BE(); if (!ValidateEthertype(protocolType)) { + if (tryFix) + { + ms.Position = fixPosition; + ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes()); + break; + } ms.Position = backupPosition; return GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE; } @@ -278,6 +321,12 @@ namespace skyscraper8.GSE if (gseLength < 0) { + if (tryFix) + { + ms.Position = fixPosition; + ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes()); + break; + } ms.Position = backupPosition; return GseAssemblyState.BROKEN_NEGATIVE_LENGTH; } @@ -291,6 +340,11 @@ namespace skyscraper8.GSE ms.Position += gseLength; if (!startIndicator && endIndicator) { + if (4 > ms.GetAvailableBytes()) + { + ms.Position = backupPosition; + return GseAssemblyState.NEED_MORE_DATA; + } uint crc32 = ms.ReadUInt32BE(); int endCrc32 = (int)ms.Position; DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32); diff --git a/skyscraper8/GSE/GsePacket.cs b/skyscraper8/GSE/GsePacket.cs index da4f130..d3ec9d3 100644 --- a/skyscraper8/GSE/GsePacket.cs +++ b/skyscraper8/GSE/GsePacket.cs @@ -9,7 +9,7 @@ using skyscraper5.Skyscraper; namespace skyscraper8.GSE { - internal class GsePacket + internal class GsePacket : Validatable { public static GsePacket Read(MemoryStream ms) { @@ -62,12 +62,23 @@ namespace skyscraper8.GSE if (!packet.StartIndicator && packet.EndIndicator) gseLength -= 4; + if (gseLength < 0) + { + packet.Valid = false; + return packet; + } + if (gseLength > ms.GetAvailableBytes()) + { + packet.Valid = false; + return packet; + } packet.GseData = ms.ReadBytes(gseLength); if (!packet.StartIndicator && packet.EndIndicator) packet.Crc32 = ms.ReadUInt32BE(); } + packet.Valid = true; return packet; } diff --git a/skyscraper8/Mpeg2/TsContext.cs b/skyscraper8/Mpeg2/TsContext.cs index 7bbaa52..f94245e 100644 --- a/skyscraper8/Mpeg2/TsContext.cs +++ b/skyscraper8/Mpeg2/TsContext.cs @@ -1,15 +1,17 @@ -using System; +using log4net; +using skyscraper5.Skyscraper; +using skyscraper5.src.Mpeg2.PacketFilter; +using System; using System.Collections.Generic; using System.Net; using System.Runtime.CompilerServices; -using skyscraper5.Skyscraper; -using skyscraper5.src.Mpeg2.PacketFilter; namespace skyscraper5.Mpeg2 { public class TsContext : ITsPacketProcessor { - private ITsPacketProcessor[] processors; + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + private ITsPacketProcessor[] processors; private uint[] continuities; private ulong[] pidPackets; @@ -42,6 +44,8 @@ namespace skyscraper5.Mpeg2 if (FilterChain == null || FilterChain.Count == 0) throw new InvalidOperationException("The filter chain has not been initialized."); + CheckBrokenEncapsulation(packet); + for (int i = 0; i < FilterChain.Count; i++) { if (FilterChain[i] == null) @@ -61,6 +65,44 @@ namespace skyscraper5.Mpeg2 processors[packet.PID].PushPacket(packet); } + private bool encapsulationBrokenConfirmed; + private bool CheckBrokenEncapsulation(TsPacket packet) + { + if (encapsulationBrokenConfirmed) + return true; + + switch (packet.PID) + { + case 0x0000: + case 0x0001: + break; + default: + return false; + } + + int occupiedPids = CountOccupiedPids(); + if (occupiedPids < 1000) + return false; + + if (packet.TSC == 0) + return false; + + logger.WarnFormat("It looks like you're either having bad reception or this is a GS with broken encapsulation. This means that you either have FrameMode enabled, or are not using a STiD135."); + encapsulationBrokenConfirmed = true; + return true; + } + + private int CountOccupiedPids() + { + int result = 0; + for (int i = 0; i < pidPackets.Length; i++) + { + if (pidPackets[i] > 0) + result++; + } + return result; + } + private bool EnsureContinuity(TsPacket packet) { //Found in ISO 13818-1.pdf, page 38 diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index d7ff207..4075ac8 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"\\\\utena\\mergerfs\\Skyscraper\\ipProto253_eutelsat7_10803h-max-sx8.ts\"", + "commandLineArgs": "cscan tcp://172.20.20.203:6969", "remoteDebugEnabled": false }, "Container (Dockerfile)": { -- 2.49.1 From 7630775d555e03cb2e93465220b4e18e2ce5490b Mon Sep 17 00:00:00 2001 From: feyris-tan Date: Tue, 14 Oct 2025 11:51:50 +0200 Subject: [PATCH 10/10] As a small exercise, made the output of Stid135BbFrameReader bit-identical to the one of pts2bbf.cpp --- skyscraper8.sln.DotSettings.user | 1 + skyscraper8/GSE/BbframeDeencapsulator.cs | 2 +- skyscraper8/GSE/BbframeDumper.cs | 26 +++++++++++++ .../{Skyscraper => GSE}/GsEventHandler.cs | 0 skyscraper8/GSE/IBbframeDeencapsulator.cs | 6 +++ skyscraper8/GSE/NullGsEventHandler.cs | 16 ++++++++ skyscraper8/GSE/Pts2Bbf.cs | 37 +++++++++++++++++++ skyscraper8/GSE/Stid135BbFrameReader.cs | 36 +++++++++++++----- skyscraper8/Program.cs | 8 ++++ .../Skyscraper/Scraper/SkyscraperContext.cs | 13 ++++--- 10 files changed, 129 insertions(+), 16 deletions(-) create mode 100644 skyscraper8/GSE/BbframeDumper.cs rename skyscraper8/{Skyscraper => GSE}/GsEventHandler.cs (100%) create mode 100644 skyscraper8/GSE/IBbframeDeencapsulator.cs create mode 100644 skyscraper8/GSE/NullGsEventHandler.cs create mode 100644 skyscraper8/GSE/Pts2Bbf.cs diff --git a/skyscraper8.sln.DotSettings.user b/skyscraper8.sln.DotSettings.user index f031d72..24e0a36 100644 --- a/skyscraper8.sln.DotSettings.user +++ b/skyscraper8.sln.DotSettings.user @@ -1,2 +1,3 @@  + ForceIncluded <data><HostParameters type="LocalHostParameters" /><Argument type="StandaloneArgument"><Arguments IsNull="False"></Arguments><FileName IsNull="False"></FileName><WorkingDirectory IsNull="False"></WorkingDirectory><Scope><ProcessFilters /></Scope></Argument><Info type="TimelineInfo" /><CoreOptions type="CoreOptions"><CoreTempPath IsNull="False"></CoreTempPath><RemoteEndPoint IsNull="False"></RemoteEndPoint><AdditionalEnvironmentVariables /></CoreOptions><HostOptions type="HostOptions"><HostTempPath IsNull="False"></HostTempPath></HostOptions></data> \ No newline at end of file diff --git a/skyscraper8/GSE/BbframeDeencapsulator.cs b/skyscraper8/GSE/BbframeDeencapsulator.cs index ab2f788..523545b 100644 --- a/skyscraper8/GSE/BbframeDeencapsulator.cs +++ b/skyscraper8/GSE/BbframeDeencapsulator.cs @@ -13,7 +13,7 @@ using log4net; namespace skyscraper8.GSE { - internal class BbframeDeencapsulator + internal class BbframeDeencapsulator : IBbframeDeencapsulator { private bool interruptedGseHem; private MemoryStream interruptedGseHemBuffer; diff --git a/skyscraper8/GSE/BbframeDumper.cs b/skyscraper8/GSE/BbframeDumper.cs new file mode 100644 index 0000000..37caf26 --- /dev/null +++ b/skyscraper8/GSE/BbframeDumper.cs @@ -0,0 +1,26 @@ +using skyscraper5.Skyscraper.IO; + +namespace skyscraper8.GSE; + +public class BbframeDumper : IBbframeDeencapsulator, IDisposable +{ + public BbframeDumper(FileInfo file) + { + file.Directory.EnsureExists(); + ourStream = file.OpenWrite(); + } + + private FileStream ourStream; + + public void PushPacket(byte[] bbframe) + { + ourStream.Write(bbframe, 0, bbframe.Length); + } + + public void Dispose() + { + ourStream.Flush(); + ourStream.Close(); + ourStream.Dispose(); + } +} \ No newline at end of file diff --git a/skyscraper8/Skyscraper/GsEventHandler.cs b/skyscraper8/GSE/GsEventHandler.cs similarity index 100% rename from skyscraper8/Skyscraper/GsEventHandler.cs rename to skyscraper8/GSE/GsEventHandler.cs diff --git a/skyscraper8/GSE/IBbframeDeencapsulator.cs b/skyscraper8/GSE/IBbframeDeencapsulator.cs new file mode 100644 index 0000000..b4a3b4e --- /dev/null +++ b/skyscraper8/GSE/IBbframeDeencapsulator.cs @@ -0,0 +1,6 @@ +namespace skyscraper8.GSE; + +public interface IBbframeDeencapsulator +{ + public void PushPacket(byte[] bbframe); +} \ No newline at end of file diff --git a/skyscraper8/GSE/NullGsEventHandler.cs b/skyscraper8/GSE/NullGsEventHandler.cs new file mode 100644 index 0000000..3d29335 --- /dev/null +++ b/skyscraper8/GSE/NullGsEventHandler.cs @@ -0,0 +1,16 @@ +using skyscraper5.Skyscraper; + +namespace skyscraper8.GSE; + +public class NullGsEventHandler : IGsEventHandler +{ + public void OnIpDatagram(int pid, byte[] payload) + { + + } + + public void GsIpTrafficDetected() + { + + } +} \ No newline at end of file diff --git a/skyscraper8/GSE/Pts2Bbf.cs b/skyscraper8/GSE/Pts2Bbf.cs new file mode 100644 index 0000000..60dd0a3 --- /dev/null +++ b/skyscraper8/GSE/Pts2Bbf.cs @@ -0,0 +1,37 @@ +using log4net; +using skyscraper5.Mpeg2; +using skyscraper5.Skyscraper.Scraper; +using skyscraper5.Skyscraper.Scraper.Storage.InMemory; +using skyscraper8.Skyscraper.Scraper.Storage; + +namespace skyscraper8.GSE; + +public class Pts2Bbf +{ + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + + public static void Run(FileInfo file) + { + if (!file.Exists) + { + logger.Error("File not found: " + file.FullName); + return; + } + + string changeExtension = Path.ChangeExtension(file.FullName, ".sbbf"); + BbframeDumper dumper = new BbframeDumper(new FileInfo(changeExtension)); + + FileStream fileStream = file.OpenRead(); + + TsContext mpeg2 = new TsContext(); + mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),dumper)); + DataStorage dataStorage = new InMemoryScraperStorage(); + ObjectStorage objectStorage = new NullObjectStorage(); + SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage); + skyscraper.InitalizeFilterChain(); + skyscraper.IngestFromStream(fileStream); + + fileStream.Close(); + logger.Info("Pts2Bbf finished"); + } +} \ No newline at end of file diff --git a/skyscraper8/GSE/Stid135BbFrameReader.cs b/skyscraper8/GSE/Stid135BbFrameReader.cs index c56264e..e42f903 100644 --- a/skyscraper8/GSE/Stid135BbFrameReader.cs +++ b/skyscraper8/GSE/Stid135BbFrameReader.cs @@ -12,11 +12,15 @@ namespace skyscraper8.GSE internal class Stid135BbFrameReader : ITsPacketProcessor { private IGsEventHandler mpeEventHandler; - private BbframeDeencapsulator deencapsulator; + private IBbframeDeencapsulator deencapsulator; - public Stid135BbFrameReader(IGsEventHandler mpeEventHandler) + public Stid135BbFrameReader(IGsEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null) { + if (deencapsulator == null) + deencapsulator = new BbframeDeencapsulator(); + this.mpeEventHandler = mpeEventHandler; + this.deencapsulator = deencapsulator; } private long packetsRecovered; @@ -33,18 +37,30 @@ namespace skyscraper8.GSE if (pid != 0x010e) return; + if (outbuf == null) + outbuf = new MemoryStream(); + if ((packets[8] & 0xff) == 0xb8) + { + if (outbuf.Length > 0) + { + byte[] completeBbframe = outbuf.ToArray(); + deencapsulator.PushPacket(completeBbframe); + outbuf = new MemoryStream(); + } + + //Start indicaator + outbuf.Write(packets, 8, packets[7]); + } + else + { + outbuf.Write(packets, 9, packets[7] - 1); + } + /*if ((packets[8] & 0xff) == 0xb8) { if (outbuf != null) { byte[] chi = outbuf.ToArray(); - if (deencapsulator == null) - { - deencapsulator = new BbframeDeencapsulator(); - deencapsulator.MpeEventHandler = mpeEventHandler; - deencapsulator.PID = pid; - } - deencapsulator.PushPacket(chi); } outbuf = new MemoryStream(); @@ -54,7 +70,7 @@ namespace skyscraper8.GSE { if (outbuf != null) outbuf.Write(packets, 9, packets[7] - 1); - } + }*/ } } } diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index e827496..0b42bd6 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -29,6 +29,7 @@ using System.Runtime.InteropServices; using skyscraper5.Mpeg2.Descriptors; using skyscraper5.Skyscraper.Scraper.StreamAutodetection; using skyscraper8; +using skyscraper8.GSE; using skyscraper8.SatIp; using skyscraper8.SatIp.RtspResponses; using skyscraper8.SimpleServiceDiscoveryProtocol; @@ -306,6 +307,13 @@ namespace skyscraper5 TogglePcapConfiguration(2); return; } + + if (args[0].ToLowerInvariant().Equals("pts2bbf")) + { + FileInfo fi = new FileInfo(args[1]); + Pts2Bbf.Run(fi); + return; + } } /*Passing passing = new Passing(); diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 296c47d..82b878e 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -299,11 +299,14 @@ namespace skyscraper5.Skyscraper.Scraper } if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x0e && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00) { - DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this)); - UiJunction?.SetGseMode(); - LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); - SpecialTsType = 3; - } + if (!DvbContext.IsPidProcessorPresent(0x010e)) + { + DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this)); + UiJunction?.SetGseMode(); + LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); + SpecialTsType = 3; + } + } firstPacketDone = true; } -- 2.49.1