From 29ebce8a3cac3706019b9b56db309e6d2fd6cfc1 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Wed, 15 Apr 2026 19:03:40 +0200 Subject: [PATCH] Began work on Reuters' WNE format. --- .../AI Food/WNE-related/WNE_Packet_stats.csv | 86 +++++++++++++++++++ .../AI Food/WNE-related/hexdump1.txt | 24 ++++++ .../AI Food/WNE-related/hexdump2.txt | 24 ++++++ .../AI Food/WNE-related/hexdump3.txt | 25 ++++++ .../AI Food/WNE-related/hexdump4.txt | 24 ++++++ .../AI Food/WNE-related/hexdump5.txt | 22 +++++ .../AI Food/WNE-related/hexdump6.txt | 21 +++++ .../AI Food/WNE-related/hexdump7-short.txt | 13 +++ Documentation/Reuters WNE Packet Format.md | 31 +++++++ skyscraper8/ReutersWne/ReutersWneExtractor.cs | 22 +++-- skyscraper8/Skyscraper/SpanByteExtensions.cs | 28 ++++++ 11 files changed, 313 insertions(+), 7 deletions(-) create mode 100644 Documentation/AI Food/WNE-related/WNE_Packet_stats.csv create mode 100644 Documentation/AI Food/WNE-related/hexdump1.txt create mode 100644 Documentation/AI Food/WNE-related/hexdump2.txt create mode 100644 Documentation/AI Food/WNE-related/hexdump3.txt create mode 100644 Documentation/AI Food/WNE-related/hexdump4.txt create mode 100644 Documentation/AI Food/WNE-related/hexdump5.txt create mode 100644 Documentation/AI Food/WNE-related/hexdump6.txt create mode 100644 Documentation/AI Food/WNE-related/hexdump7-short.txt create mode 100644 Documentation/Reuters WNE Packet Format.md diff --git a/Documentation/AI Food/WNE-related/WNE_Packet_stats.csv b/Documentation/AI Food/WNE-related/WNE_Packet_stats.csv new file mode 100644 index 0000000..cf931ca --- /dev/null +++ b/Documentation/AI Food/WNE-related/WNE_Packet_stats.csv @@ -0,0 +1,86 @@ +byte1 in Hex;unkn1 in hex;unkn1 in Binary;byte2 in Hex;Times occurred;smallest seen packet;largest seen packet +0x01;0x10;0b00010000;0x00;1941;18;18 +0x01;0x18;0b00011000;0x00;2898;24;24 +0x01;0xC8;0b11001000;0x00;268;200;200 +0x01;0xD4;0b11010100;0x00;286;212;212 +0x01;0x38;0b00111000;0x01;24;312;312 +0x01;0x3A;0b00111010;0x01;84;314;314 +0x01;0x42;0b01000010;0x01;9;322;322 +0x01;0x44;0b01000100;0x01;163;324;324 +0x01;0x46;0b01000110;0x01;295;326;326 +0x01;0x48;0b01001000;0x01;480;328;328 +0x01;0x4A;0b01001010;0x01;143;330;330 +0x01;0x4C;0b01001100;0x01;129;332;332 +0x01;0x4E;0b01001110;0x01;86;334;334 +0x01;0x50;0b01010000;0x01;102;336;336 +0x01;0x52;0b01010010;0x01;145;338;338 +0x01;0x54;0b01010100;0x01;149;340;340 +0x01;0x56;0b01010110;0x01;444;342;342 +0x01;0x58;0b01011000;0x01;93;344;344 +0x01;0x5A;0b01011010;0x01;5;346;346 +0x01;0x5C;0b01011100;0x01;69;348;348 +0x01;0x5E;0b01011110;0x01;238;350;350 +0x01;0x60;0b01100000;0x01;191;352;352 +0x01;0x62;0b01100010;0x01;395;354;354 +0x01;0x64;0b01100100;0x01;145;356;356 +0x01;0x66;0b01100110;0x01;170;358;358 +0x01;0x68;0b01101000;0x01;308;360;360 +0x01;0x6A;0b01101010;0x01;262;362;362 +0x01;0x6C;0b01101100;0x01;81;364;364 +0x01;0x6E;0b01101110;0x01;153;366;366 +0x01;0x70;0b01110000;0x01;177;368;368 +0x01;0x72;0b01110010;0x01;207;370;370 +0x01;0x74;0b01110100;0x01;158;372;372 +0x01;0x76;0b01110110;0x01;285;374;374 +0x01;0x78;0b01111000;0x01;421;376;376 +0x01;0x7A;0b01111010;0x01;107;378;378 +0x01;0x7C;0b01111100;0x01;59;380;380 +0x01;0x7E;0b01111110;0x01;54;382;382 +0x01;0x80;0b10000000;0x01;33;384;384 +0x01;0x82;0b10000010;0x01;45;386;386 +0x01;0x84;0b10000100;0x01;5;388;388 +0x01;0x86;0b10000110;0x01;10;390;390 +0x01;0x88;0b10001000;0x01;15;392;392 +0x01;0x8A;0b10001010;0x01;15;394;394 +0x01;0x8E;0b10001110;0x01;10;398;398 +0x01;0x90;0b10010000;0x01;10;400;400 +0x01;0x92;0b10010010;0x01;5;402;402 +0x01;0x96;0b10010110;0x01;5;406;406 +0x01;0x9A;0b10011010;0x01;10;410;410 +0x01;0x9C;0b10011100;0x01;10;412;412 +0x01;0x9E;0b10011110;0x01;5;414;414 +0x01;0xA2;0b10100010;0x01;10;418;418 +0x01;0xAE;0b10101110;0x01;10;430;430 +0x01;0xB6;0b10110110;0x01;5;438;438 +0x01;0x90;0b10010000;0x05;17020001;1424;1424 +0x01;0xA0;0b10100000;0x05;257300;1440;1440 +0x03;0x42;0b01000010;0x00;789;66;66 +0x03;0x3E;0b00111110;0x01;2494;318;318 +0x03;0x40;0b01000000;0x01;6732;320;320 +0x03;0x42;0b01000010;0x01;674;322;322 +0x03;0x44;0b01000100;0x01;654;324;324 +0x03;0x46;0b01000110;0x01;1182;326;326 +0x03;0x48;0b01001000;0x01;696;328;328 +0x03;0x4A;0b01001010;0x01;2046;330;330 +0x03;0x4C;0b01001100;0x01;843;332;332 +0x03;0x4E;0b01001110;0x01;4130;334;334 +0x03;0x50;0b01010000;0x01;2032;336;336 +0x03;0x54;0b01010100;0x01;1121;340;340 +0x03;0x56;0b01010110;0x01;2686;342;342 +0x03;0x58;0b01011000;0x01;6191;344;344 +0x03;0x5A;0b01011010;0x01;533;346;346 +0x03;0x5C;0b01011100;0x01;1102;348;348 +0x03;0x5E;0b01011110;0x01;2777;350;350 +0x03;0x60;0b01100000;0x01;3290;352;352 +0x03;0x62;0b01100010;0x01;4831;354;354 +0x03;0x64;0b01100100;0x01;140;356;356 +0x03;0x66;0b01100110;0x01;2485;358;358 +0x03;0x68;0b01101000;0x01;4622;360;360 +0x03;0x6A;0b01101010;0x01;3865;362;362 +0x03;0x6C;0b01101100;0x01;5000;364;364 +0x03;0x70;0b01110000;0x01;305;368;368 +0x03;0x72;0b01110010;0x01;372;370;370 +0x03;0x74;0b01110100;0x01;1106;372;372 +0x03;0x78;0b01111000;0x01;1969;376;376 +0x03;0xBA;0b10111010;0x05;4037;1466;1466 +0xFF;0x08;0b00001000;0x00;421;18;18 diff --git a/Documentation/AI Food/WNE-related/hexdump1.txt b/Documentation/AI Food/WNE-related/hexdump1.txt new file mode 100644 index 0000000..fe8578a --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump1.txt @@ -0,0 +1,24 @@ +0000 00 01 80 01 e1 91 85 92 eb 7c 00 00 00 20 71 01 ・・・・・・・・ ・|・・・ q・ +0010 00 03 70 01 a9 96 24 a7 68 01 00 00 58 01 00 00 ・・p・・・$・ h・・・X・・・ +0020 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0030 97 12 0a 00 00 00 00 00 02 00 00 00 44 01 00 00 ・・・・・・・・ ・・・・D・・・ +0040 17 00 00 00 00 00 00 00 80 05 00 00 8b 47 d9 69 ・・・・・・・・ ・・・・・G・i +0050 24 00 00 00 00 00 00 00 24 00 00 00 00 00 00 00 $・・・・・・・ $・・・・・・・ +0060 01 00 00 00 32 30 32 36 2d 30 34 2d 31 30 54 31 ・・・・2026 -04-10T1 +0070 38 35 31 32 35 5a 5f 33 5f 52 57 31 35 34 38 31 85125Z_3 _RW15481 +0080 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 4d 41 0042026R P1_RTRMA +0090 44 43 5f 30 5f 48 55 4e 47 41 52 59 2d 45 4c 45 DC_0_HUN GARY-ELE +00A0 43 54 49 4f 4e 2d 4f 50 50 4f 53 49 54 49 4f 4e CTION-OP POSITION +00B0 2d 43 4f 4e 43 45 52 54 2d 4d 4f 52 45 2e 58 4d -CONCERT -MORE.XM +00C0 4c 00 83 01 00 00 00 c1 10 65 79 84 53 b5 87 72 L・・・・・・・ ・ey・S・・r +00D0 48 9a c0 a7 5d 0b 6d 6a 58 a7 00 00 00 00 00 00 H・・・]・mj X・・・・・・・ +00E0 00 0c 00 00 00 24 00 00 00 00 00 00 00 02 00 00 ・・・・・$・・ ・・・・・・・・ +00F0 00 d2 46 d9 69 00 00 00 00 11 ba 00 00 00 00 00 ・・F・i・・・ ・・・・・・・・ +0100 00 00 c6 00 00 00 00 00 00 5c 52 56 4e 2d 53 43 ・・・・・・・・ ・\RVN-SC +0110 52 49 50 54 53 5c 32 30 32 36 2d 30 34 2d 31 30 RIPTS\20 26-04-10 +0120 54 31 38 35 31 32 35 5a 5f 33 5f 52 57 31 35 34 T185125Z _3_RW154 +0130 38 31 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 81004202 6RP1_RTR +0140 4d 41 44 43 5f 30 5f 48 55 4e 47 41 52 59 2d 45 MADC_0_H UNGARY-E +0150 4c 45 43 54 49 4f 4e 2d 4f 50 50 4f 53 49 54 49 LECTION- OPPOSITI +0160 4f 4e 2d 43 4f 4e 43 45 52 54 2d 4d 4f 52 45 2e ON-CONCE RT-MORE. +0170 58 4d 4c 00 01 00 00 00 01 00 04 00 22 24 00 80 XML・・・・・ ・・・・"$・・ diff --git a/Documentation/AI Food/WNE-related/hexdump2.txt b/Documentation/AI Food/WNE-related/hexdump2.txt new file mode 100644 index 0000000..25cb19c --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump2.txt @@ -0,0 +1,24 @@ +0000 00 01 7e 01 e1 91 85 92 f1 7c 00 04 00 00 6f 01 ・・~・・・・・ ・|・・・・o・ +0010 00 03 6e 01 3f 0e c2 9e 66 01 00 00 56 01 00 00 ・・n・?・・・ f・・・V・・・ +0020 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0030 97 12 0a 00 00 00 00 00 02 00 00 00 42 01 00 00 ・・・・・・・・ ・・・・B・・・ +0040 17 00 00 00 00 00 00 00 80 05 00 00 8b 47 d9 69 ・・・・・・・・ ・・・・・G・i +0050 11 00 00 00 00 00 00 00 11 00 00 00 00 00 00 00 ・・・・・・・・ ・・・・・・・・ +0060 01 00 00 00 32 30 32 36 2d 30 34 2d 31 30 54 31 ・・・・2026 -04-10T1 +0070 38 35 31 32 35 5a 5f 33 5f 57 44 31 35 34 38 31 85125Z_3 _WD15481 +0080 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 57 4e 0042026R P1_RTRWN +0090 45 43 5f 30 5f 31 35 34 38 2d 48 55 4e 47 41 52 EC_0_154 8-HUNGAR +00A0 59 2d 45 4c 45 43 54 49 4f 4e 2d 4f 50 50 4f 53 Y-ELECTI ON-OPPOS +00B0 49 54 49 4f 4e 2d 43 4f 4e 43 45 52 54 2d 4d 4f ITION-CO NCERT-MO +00C0 52 45 2e 58 4d 4c 00 83 01 00 00 00 c1 10 3d 83 RE.XML・・ ・・・・・・=・ +00D0 f8 ba dc 14 a1 48 80 25 a3 05 be a5 b1 ab a0 00 ・・・・・H・% ・・・・・・・・ +00E0 00 00 00 00 00 00 0c 00 00 00 11 00 00 00 00 00 ・・・・・・・・ ・・・・・・・・ +00F0 00 00 02 00 00 00 d2 46 d9 69 00 00 00 00 64 53 ・・・・・・・F ・i・・・・dS +0100 00 00 00 00 00 00 80 5d 00 00 00 00 00 00 5c 32 ・・・・・・・] ・・・・・・\2 +0110 30 32 36 2d 30 34 2d 31 30 54 31 38 35 31 32 35 026-04-1 0T185125 +0120 5a 5f 33 5f 57 44 31 35 34 38 31 30 30 34 32 30 Z_3_WD15 48100420 +0130 32 36 52 50 31 5f 52 54 52 57 4e 45 43 5f 30 5f 26RP1_RT RWNEC_0_ +0140 31 35 34 38 2d 48 55 4e 47 41 52 59 2d 45 4c 45 1548-HUN GARY-ELE +0150 43 54 49 4f 4e 2d 4f 50 50 4f 53 49 54 49 4f 4e CTION-OP POSITION +0160 2d 43 4f 4e 43 45 52 54 2d 4d 4f 52 45 2e 58 4d -CONCERT -MORE.XM +0170 4c 00 01 00 00 00 01 00 04 00 10 11 00 80 L・・・・・・・ ・・・・・・ diff --git a/Documentation/AI Food/WNE-related/hexdump3.txt b/Documentation/AI Food/WNE-related/hexdump3.txt new file mode 100644 index 0000000..836036a --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump3.txt @@ -0,0 +1,25 @@ +0000 00 01 8a 01 e1 91 85 92 fa 7c 00 00 00 21 7b 01 ・・・・・・・・ ・|・・・!{・ +0010 00 03 7a 01 d5 85 5f 96 72 01 00 00 62 01 00 00 ・・z・・・_・ r・・・b・・・ +0020 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0030 97 12 0a 00 00 00 00 00 02 00 00 00 4e 01 00 00 ・・・・・・・・ ・・・・N・・・ +0040 17 00 00 00 00 00 00 00 80 05 00 00 c7 47 d9 69 ・・・・・・・・ ・・・・・G・i +0050 0b 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 ・・・・・・・・ ・・・・・・・・ +0060 01 00 00 00 32 30 32 36 2d 30 34 2d 31 30 54 31 ・・・・2026 -04-10T1 +0070 38 35 31 32 35 5a 5f 31 5f 57 44 31 35 34 38 31 85125Z_1 _WD15481 +0080 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 57 4e 0042026R P1_RTRWN +0090 45 43 5f 30 5f 31 35 34 38 2d 48 55 4e 47 41 52 EC_0_154 8-HUNGAR +00A0 59 2d 45 4c 45 43 54 49 4f 4e 2d 4f 50 50 4f 53 Y-ELECTI ON-OPPOS +00B0 49 54 49 4f 4e 2d 43 4f 4e 43 45 52 54 2d 4d 4f ITION-CO NCERT-MO +00C0 52 45 2e 63 61 73 00 83 01 00 00 00 c1 10 a5 7c RE.cas・・ ・・・・・・・| +00D0 ce b3 91 90 a1 47 81 54 96 43 55 36 a2 b1 ac 00 ・・・・・G・T ・CU6・・・・ +00E0 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 00 00 ・・・・・・・・ ・・・・・・・・ +00F0 00 00 02 00 00 00 0e 47 d9 69 00 00 00 00 7a 36 ・・・・・・・G ・i・・・・z6 +0100 00 00 00 00 00 00 80 3c 00 00 00 00 00 00 5c 52 ・・・・・・・< ・・・・・・\R +0110 56 4e 2d 53 43 52 49 50 54 53 5c 32 30 32 36 2d VN-SCRIP TS\2026- +0120 30 34 2d 31 30 54 31 38 35 31 32 35 5a 5f 31 5f 04-10T18 5125Z_1_ +0130 57 44 31 35 34 38 31 30 30 34 32 30 32 36 52 50 WD154810 042026RP +0140 31 5f 52 54 52 57 4e 45 43 5f 30 5f 31 35 34 38 1_RTRWNE C_0_1548 +0150 2d 48 55 4e 47 41 52 59 2d 45 4c 45 43 54 49 4f -HUNGARY -ELECTIO +0160 4e 2d 4f 50 50 4f 53 49 54 49 4f 4e 2d 43 4f 4e N-OPPOSI TION-CON +0170 43 45 52 54 2d 4d 4f 52 45 2e 63 61 73 00 01 00 CERT-MOR E.cas・・・ +0180 00 00 01 00 04 00 0a 0b 00 80 ・・・・・・・・ ・・ diff --git a/Documentation/AI Food/WNE-related/hexdump4.txt b/Documentation/AI Food/WNE-related/hexdump4.txt new file mode 100644 index 0000000..b74582a --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump4.txt @@ -0,0 +1,24 @@ +0000 00 03 78 01 cc 30 69 01 70 01 00 00 60 01 00 00 ・・x・・0i・ p・・・`・・・ +0010 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0020 97 12 0a 00 00 00 00 00 02 00 00 00 4c 01 00 00 ・・・・・・・・ ・・・・L・・・ +0030 17 00 00 00 00 00 00 00 80 05 00 00 db 47 d9 69 ・・・・・・・・ ・・・・・G・i +0040 00 90 02 00 00 00 00 00 00 90 02 00 00 00 00 00 ・・・・・・・・ ・・・・・・・・ +0050 01 00 00 00 32 30 32 36 2d 30 34 2d 31 30 54 31 ・・・・2026 -04-10T1 +0060 38 34 34 31 34 5a 5f 31 5f 4c 57 44 31 35 34 38 84414Z_1 _LWD1548 +0070 31 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 57 10042026 RP1_RTRW +0080 4e 45 56 5f 42 5f 31 35 34 38 2d 48 55 4e 47 41 NEV_B_15 48-HUNGA +0090 52 59 2d 45 4c 45 43 54 49 4f 4e 2d 4f 50 50 4f RY-ELECT ION-OPPO +00A0 53 49 54 49 4f 4e 2d 43 4f 4e 43 45 52 54 2d 4d SITION-C ONCERT-M +00B0 4f 52 45 2e 4d 50 47 2e 65 6e 63 00 83 01 00 00 ORE.MPG. enc・・・・・ +00C0 00 c1 10 de b4 12 f2 c5 cb d8 47 a7 10 17 e4 bb ・・・・・・・・ ・・G・・・・・ +00D0 7e 62 1c a5 00 00 00 00 00 00 00 0c 00 00 00 00 ~b・・・・・・ ・・・・・・・・ +00E0 90 02 00 00 00 00 00 02 00 00 00 13 47 d9 69 00 ・・・・・・・・ ・・・・G・i・ +00F0 00 00 00 00 40 5c 0d 00 00 00 00 00 00 18 0e 00 ・・・・@\・・ ・・・・・・・・ +0100 00 00 00 5c 32 30 32 36 2d 30 34 2d 31 30 54 31 ・・・\2026 -04-10T1 +0110 38 34 34 31 34 5a 5f 31 5f 4c 57 44 31 35 34 38 84414Z_1 _LWD1548 +0120 31 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 57 10042026 RP1_RTRW +0130 4e 45 56 5f 42 5f 31 35 34 38 2d 48 55 4e 47 41 NEV_B_15 48-HUNGA +0140 52 59 2d 45 4c 45 43 54 49 4f 4e 2d 4f 50 50 4f RY-ELECT ION-OPPO +0150 53 49 54 49 4f 4e 2d 43 4f 4e 43 45 52 54 2d 4d SITION-C ONCERT-M +0160 4f 52 45 2e 4d 50 47 2e 65 6e 63 00 01 00 00 00 ORE.MPG. enc・・・・・ +0170 01 00 04 00 f3 00 00 80 ・・・・・・・・ diff --git a/Documentation/AI Food/WNE-related/hexdump5.txt b/Documentation/AI Food/WNE-related/hexdump5.txt new file mode 100644 index 0000000..3afdcb4 --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump5.txt @@ -0,0 +1,22 @@ +0000 00 03 54 01 12 65 f2 b5 4c 01 00 00 3c 01 00 00 ・・T・・e・・ L・・・<・・・ +0010 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0020 97 12 0a 00 00 00 00 00 02 00 00 00 28 01 00 00 ・・・・・・・・ ・・・・(・・・ +0030 17 00 00 00 00 00 00 00 80 05 00 00 b4 48 d9 69 ・・・・・・・・ ・・・・・H・i +0040 d8 13 00 00 00 00 00 00 d8 13 00 00 00 00 00 00 ・・・・・・・・ ・・・・・・・・ +0050 01 00 00 00 32 30 31 32 2d 31 32 2d 30 36 54 31 ・・・・2012 -12-06T1 +0060 37 32 34 34 31 5a 5f 37 36 5f 4c 57 44 30 30 30 72441Z_7 6_LWD000 +0070 52 54 54 49 4f 4f 44 5f 52 54 52 57 4e 45 56 5f RTTIOOD_ RTRWNEV_ +0080 42 5f 54 39 39 39 2d 50 4c 45 41 53 45 2d 49 47 B_T999-P LEASE-IG +0090 4e 4f 52 45 2d 32 36 30 34 31 30 30 30 30 37 35 NORE-260 41000075 +00A0 33 2e 4d 50 47 2e 65 6e 63 00 83 01 00 00 00 c1 3.MPG.en c・・・・・・・ +00B0 10 eb d9 18 9c ea 9a af 48 8b 2c 06 c4 25 6a 2c ・・・・・・・・ H・,・・%j, +00C0 bf 93 00 00 00 00 00 00 00 0c 00 00 00 d8 13 00 ・・・・・・・・ ・・・・・・・・ +00D0 00 00 00 00 00 02 00 00 00 85 47 d9 69 00 00 00 ・・・・・・・・ ・・G・i・・・ +00E0 00 00 20 67 00 00 00 00 00 00 24 6d 00 00 00 00 ・・ g・・・・ ・・$m・・・・ +00F0 00 5c 32 30 31 32 2d 31 32 2d 30 36 54 31 37 32 ・\2012-1 2-06T172 +0100 34 34 31 5a 5f 37 36 5f 4c 57 44 30 30 30 52 54 441Z_76_ LWD000RT +0110 54 49 4f 4f 44 5f 52 54 52 57 4e 45 56 5f 42 5f TIOOD_RT RWNEV_B_ +0120 54 39 39 39 2d 50 4c 45 41 53 45 2d 49 47 4e 4f T999-PLE ASE-IGNO +0130 52 45 2d 32 36 30 34 31 30 30 30 30 37 35 33 2e RE-26041 0000753. +0140 4d 50 47 2e 65 6e 63 00 01 00 00 00 01 00 04 00 MPG.enc・ ・・・・・・・・ +0150 f1 fe 00 80 ・・・・ diff --git a/Documentation/AI Food/WNE-related/hexdump6.txt b/Documentation/AI Food/WNE-related/hexdump6.txt new file mode 100644 index 0000000..d97db6a --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump6.txt @@ -0,0 +1,21 @@ +0000 00 03 4c 01 11 ab 5d bc 44 01 00 00 34 01 00 00 ・・L・・・]・ D・・・4・・・ +0010 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0020 97 12 0a 00 00 00 00 00 02 00 00 00 20 01 00 00 ・・・・・・・・ ・・・・ ・・・ +0030 17 00 00 00 00 00 00 00 80 05 00 00 af 49 d9 69 ・・・・・・・・ ・・・・・I・i +0040 00 4f 01 00 00 00 00 00 00 4f 01 00 00 00 00 00 ・O・・・・・・ ・O・・・・・・ +0050 01 00 00 00 32 30 32 36 2d 30 34 2d 31 30 54 31 ・・・・2026 -04-10T1 +0060 38 35 34 34 31 5a 5f 31 5f 4c 57 44 31 35 32 30 85441Z_1 _LWD1520 +0070 31 30 30 34 32 30 32 36 52 50 31 5f 52 54 52 57 10042026 RP1_RTRW +0080 4e 45 56 5f 42 5f 31 35 32 30 2d 53 50 41 43 45 NEV_B_15 20-SPACE +0090 2d 4d 4f 4f 4e 2d 41 52 54 45 4d 49 53 2e 4d 50 -MOON-AR TEMIS.MP +00A0 47 2e 65 6e 63 00 83 01 00 00 00 c1 10 e5 8c 5f G.enc・・・ ・・・・・・・_ +00B0 e0 01 87 38 48 bf be 18 db b0 93 27 d7 8f 00 00 ・・・8H・・・ ・・・'・・・・ +00C0 00 00 00 00 00 0c 00 00 00 00 4f 01 00 00 00 00 ・・・・・・・・ ・・O・・・・・ +00D0 00 02 00 00 00 f0 48 d9 69 00 00 00 00 00 c8 d0 ・・・・・・H・ i・・・・・・・ +00E0 06 00 00 00 00 00 80 32 07 00 00 00 00 5c 32 30 ・・・・・・・2 ・・・・・\20 +00F0 32 36 2d 30 34 2d 31 30 54 31 38 35 34 34 31 5a 26-04-10 T185441Z +0100 5f 31 5f 4c 57 44 31 35 32 30 31 30 30 34 32 30 _1_LWD15 20100420 +0110 32 36 52 50 31 5f 52 54 52 57 4e 45 56 5f 42 5f 26RP1_RT RWNEV_B_ +0120 31 35 32 30 2d 53 50 41 43 45 2d 4d 4f 4f 4e 2d 1520-SPA CE-MOON- +0130 41 52 54 45 4d 49 53 2e 4d 50 47 2e 65 6e 63 00 ARTEMIS. MPG.enc・ +0140 01 00 00 00 01 00 04 00 f3 00 00 80 ・・・・・・・・ ・・・・ diff --git a/Documentation/AI Food/WNE-related/hexdump7-short.txt b/Documentation/AI Food/WNE-related/hexdump7-short.txt new file mode 100644 index 0000000..d35c342 --- /dev/null +++ b/Documentation/AI Food/WNE-related/hexdump7-short.txt @@ -0,0 +1,13 @@ +0000 00 01 c8 00 e1 91 85 92 f9 7d 00 02 00 00 b9 00 ・・・・・・・・ ・}・・・・・・ +0010 00 03 b8 00 2b f0 ab 81 b0 00 00 00 a0 00 00 00 ・・・・+・・・ ・・・・・・・・ +0020 00 00 00 00 01 00 00 00 02 00 00 70 00 00 00 00 ・・・・・・・・ ・・・p・・・・ +0030 97 12 0a 00 00 00 00 00 02 00 00 00 8c 00 00 00 ・・・・・・・・ ・・・・・・・・ +0040 17 00 00 00 00 00 00 00 80 05 00 00 5c 4a d9 69 ・・・・・・・・ ・・・・\J・i +0050 61 02 00 00 00 00 00 00 61 02 00 00 00 00 00 00 a・・・・・・・ a・・・・・・・ +0060 01 00 00 00 77 6e 65 2e 64 61 74 00 83 01 00 00 ・・・・wne. dat・・・・・ +0070 00 c1 10 97 70 db bc 3e 3a 9e 45 9f 03 8d 2a fb ・・・・p・・> :・E・・・*・ +0080 ea eb ce 45 00 00 00 00 00 00 00 0c 00 00 00 61 ・・・E・・・・ ・・・・・・・a +0090 02 00 00 00 00 00 00 02 00 00 00 a2 49 d9 69 00 ・・・・・・・・ ・・・・I・i・ +00A0 00 00 00 6b 5e 0c 00 00 00 00 00 80 15 0d 00 00 ・・・k^・・・ ・・・・・・・・ +00B0 00 00 00 5c 77 6e 65 2e 64 61 74 00 01 00 00 00 ・・・\wne. dat・・・・・ +00C0 01 00 04 00 c0 cb 00 80 ・・・・・・・・ diff --git a/Documentation/Reuters WNE Packet Format.md b/Documentation/Reuters WNE Packet Format.md new file mode 100644 index 0000000..21691e5 --- /dev/null +++ b/Documentation/Reuters WNE Packet Format.md @@ -0,0 +1,31 @@ +# Reuters WNE Packet Format + +Format is little endian. + +Common bytes: + +- Byte 0: always 0x00 +- Byte 1: "Packet family", always 0x01, 0x03 or 0xff +- Bytes 2-3: Packet length +- Bytes 4-7: Session ID. This changes for every news story. + +## A/V File Payloads: +- Byte 1: always 0x01 +- Bytes 8-11: Block number, always incremented by 1 each packet. +- Bytes 12-15: always 0x00000000 +- Bytes 16-end: payload + +## A/V File announcement: +- Byte 1: always 0x03 +- Bytes 8-11: ??? +- Bytes 12-13: Packet length, minus header (16) minus 8. + +## Metadata File Payloads: +-Bytes 16-end: payload + +## Metadata file announcement: + +- Bytes 12-13: ??? +- Bytes 14-15: (uint16) Packet length minus 15 +- Bytes 16-17: always seems to be 0x0003 +- Bytes 18-19: (uint16) Always seems to be the value of bytes 14-15 minus one. diff --git a/skyscraper8/ReutersWne/ReutersWneExtractor.cs b/skyscraper8/ReutersWne/ReutersWneExtractor.cs index 2aca3be..59e8474 100644 --- a/skyscraper8/ReutersWne/ReutersWneExtractor.cs +++ b/skyscraper8/ReutersWne/ReutersWneExtractor.cs @@ -1,3 +1,4 @@ +using log4net; using skyscraper5.Ietf.Rfc971; using skyscraper5.Skyscraper.Plugins; @@ -6,8 +7,10 @@ namespace skyscraper8.ReutersWne; [SkyscraperPlugin] public class ReutersWneExtractor : ISkyscraperMpePlugin { + private ILog _logger; public void ConnectToStorage(object[] connector) { + _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); //TODO: connect to stroage plugins } @@ -36,7 +39,7 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin return true; } - + public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) { Span udpPayload = new Span(ipv4Packet,8,ipv4Packet.Length-8); @@ -45,13 +48,18 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin if (byte0 != 0x00) return; - byte opcodeL = udpPayload[1]; - byte unkn1 = udpPayload[2]; - byte opcodeR = udpPayload[3]; - uint session = udpPayload.ReadUInt32BigEndian(4); - + byte msgFamily = udpPayload[1]; + ushort length = udpPayload.ReadUInt16LittleEndian(2); + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); - //throw new NotImplementedException(); + if (msgFamily == 0xff) + return; + + if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18) && msgFamily != 0xff) + { + return; + } + } public bool StopProcessingAfterThis() diff --git a/skyscraper8/Skyscraper/SpanByteExtensions.cs b/skyscraper8/Skyscraper/SpanByteExtensions.cs index 459ee6e..63a5ac8 100644 --- a/skyscraper8/Skyscraper/SpanByteExtensions.cs +++ b/skyscraper8/Skyscraper/SpanByteExtensions.cs @@ -2,6 +2,26 @@ using System; public static class SpanByteExtensions { + public static ushort ReadUInt16LittleEndian(this ReadOnlySpan span, int offset) + { + if ((uint)offset > (uint)(span.Length - 2)) + throw new ArgumentOutOfRangeException(nameof(offset)); + + return (ushort)( + span[offset] | + (span[offset + 1] << 8)); + } + + public static ushort ReadUInt16BigEndian(this ReadOnlySpan span, int offset) + { + if ((uint)offset > (uint)(span.Length - 2)) + throw new ArgumentOutOfRangeException(nameof(offset)); + + return (ushort)( + (span[offset] << 8) | + span[offset + 1]); + } + public static uint ReadUInt32LittleEndian(this ReadOnlySpan span, int offset) { if ((uint)offset > (uint)(span.Length - 4)) @@ -26,9 +46,17 @@ public static class SpanByteExtensions span[offset + 3]); } + public static ushort ReadUInt16LittleEndian(this Span span, int offset) + => ReadUInt16LittleEndian((ReadOnlySpan)span, offset); + + public static ushort ReadUInt16BigEndian(this Span span, int offset) + => ReadUInt16BigEndian((ReadOnlySpan)span, offset); + public static uint ReadUInt32LittleEndian(this Span span, int offset) => ReadUInt32LittleEndian((ReadOnlySpan)span, offset); public static uint ReadUInt32BigEndian(this Span span, int offset) => ReadUInt32BigEndian((ReadOnlySpan)span, offset); + + }