feat: parse reforged replays successfully (#39)

* feat: support reforged replays * improvement: another reforged replay test * improvement: skip 2 bytes before compressed blocks * improvement: parsing works except actions * improvement: parse actions aswell and allow to fail * improvement: action 0x7b has 16 byte payload * improvement: re-enable ReplayParser timeslotblock event emitting * improvement: parse 0x7b as buffer, output 0x7b actions in test * feat: parse new reforged game metadata * test: updated 1.32 reforged replay test case with assertions * improvement: handle classic and latest reforged * improvement: add buildNumber to parser output, update tests * test: add reforged release replay file and test * chore: update dependencies * test: rename new reforged release test
PBug90 · Jan 29, 2020 · 2dfa447 · 2dfa447
1 parent 01062b4
commit 2dfa447
Show file tree

Hide file tree

Showing 14 changed files with 2,309 additions and 2,266 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -16,39 +16,39 @@
     "test": "jest",
     "coverage": "jest --collectCoverage",
     "lint": "eslint --ext .ts,.js src/ test/",
-    "lint:fix": "eslint --ext .jsx,.js lib/ --fix",
+    "lint:fix": "eslint --ext .jsx,.js src/** --fix",
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
     "build:docs": "typedoc --out docs --target es6 --theme minimal --mode file src"
   },
   "dependencies": {
-    "binary-parser": "^1.4.0"
+    "binary-parser": "^1.5.0"
   },
   "devDependencies": {
     "@types/binary-parser": "^1.3.1",
-    "@types/jest": "^24.0.18",
-    "@types/node": "^12.7.2",
-    "@typescript-eslint/eslint-plugin": "^2.0.0",
-    "@typescript-eslint/parser": "^2.0.0",
-    "eslint": "^6.2.2",
+    "@types/jest": "^25.1.0",
+    "@types/node": "^13.5.1",
+    "@typescript-eslint/eslint-plugin": "^2.18.0",
+    "@typescript-eslint/parser": "^2.18.0",
+    "eslint": "^6.8.0",
     "eslint-config-standard": "^14.1.0",
-    "eslint-plugin-import": "^2.18.2",
-    "eslint-plugin-node": "^9.1.0",
+    "eslint-plugin-import": "^2.20.0",
+    "eslint-plugin-node": "^11.0.0",
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
-    "jest": "^24.9.0",
-    "jsonschema": "^1.2.4",
+    "jest": "^25.1.0",
+    "jsonschema": "^1.2.5",
     "lodash.camelcase": "^4.3.0",
-    "rollup": "^1.20.3",
+    "rollup": "^1.30.1",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-json": "^4.0.0",
     "rollup-plugin-node-builtins": "^2.1.2",
     "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-node-resolve": "^5.2.0",
-    "rollup-plugin-sourcemaps": "^0.4.2",
-    "rollup-plugin-typescript2": "^0.23.0",
-    "ts-jest": "^24.0.2",
-    "typedoc": "^0.15.0",
-    "typescript": "^3.6.2"
+    "rollup-plugin-sourcemaps": "^0.5.0",
+    "rollup-plugin-typescript2": "^0.25.3",
+    "ts-jest": "^25.0.0",
+    "typedoc": "^0.16.9",
+    "typescript": "^3.7.5"
   },
   "jest": {
     "transform": {

diff --git a/replays/reforged1.w3g b/replays/reforged1.w3g
diff --git a/replays/reforged2.w3g b/replays/reforged2.w3g
diff --git a/replays/reforged2010.w3g b/replays/reforged2010.w3g
diff --git a/replays/reforged_release.w3g b/replays/reforged_release.w3g
diff --git a/rollup.config.ts b/rollup.config.ts
@@ -10,35 +10,35 @@ import builtins from 'rollup-plugin-node-builtins'
 const pkg = require('./package.json')
 
 export default {
-  input: `src/index.ts`,
-  output: [
-    { file: pkg.main, name: camelCase('index'), format: 'umd', sourcemap: true },
-    { file: pkg.module, format: 'es', sourcemap: true },
-  ],
-  // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
-  external: [],
-  watch: {
-    include: 'src/**',
-  },
-  plugins: [
+    input: 'src/index.ts',
+    output: [
+        { file: pkg.main, name: camelCase('index'), format: 'umd', sourcemap: true },
+        { file: pkg.module, format: 'es', sourcemap: true }
+    ],
+    // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+    external: [],
+    watch: {
+        include: 'src/**'
+    },
+    plugins: [
     // Allow json resolution
-    json(),
-    // Compile TypeScript files
-    typescript({ useTsconfigDeclarationDir: true }),
-    // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
-    commonjs(),
-    // Allow node_modules resolution, so you can use 'external' to control
-    // which external modules to include in the bundle
-    // https://github.com/rollup/rollup-plugin-node-resolve#usage
-    resolve(),
+        json(),
+        // Compile TypeScript files
+        typescript({ useTsconfigDeclarationDir: true, clean: true }),
+        // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+        commonjs(),
+        // Allow node_modules resolution, so you can use 'external' to control
+        // which external modules to include in the bundle
+        // https://github.com/rollup/rollup-plugin-node-resolve#usage
+        resolve(),
 
-    // Resolve source maps to the original source
-    sourceMaps(),
-    // Node globals
-    globals(),
-    // Node built-ins
-    // Causing Circular dependency warning
-    // https://github.com/calvinmetcalf/rollup-plugin-node-builtins/issues/39
-    builtins(),
-  ],
-}
+        // Resolve source maps to the original source
+        sourceMaps(),
+        // Node globals
+        globals(),
+        // Node built-ins
+        // Causing Circular dependency warning
+        // https://github.com/calvinmetcalf/rollup-plugin-node-builtins/issues/39
+        builtins()
+    ]
+}
diff --git a/src/ReplayParser.ts b/src/ReplayParser.ts
@@ -1,6 +1,6 @@
 import { Parser } from 'binary-parser'
 import { ActionBlockList } from './parsers/actions'
-import { ReplayHeader, EncodedMapMetaString, GameMetaData } from './parsers/header'
+import { ReplayHeader, EncodedMapMetaString, GameMetaDataParserFactory } from './parsers/header'
 import { GameDataParser } from './parsers/gamedata'
 
 import {
@@ -15,9 +15,13 @@ import {
 // https://rollupjs.org/guide/en#error-name-is-not-exported-by-module-
 const { readFileSync } = require('fs')
 const { inflateSync, constants } = require('zlib')
-const GameDataParserComposed = new Parser()
-    .nest('meta', { type: GameMetaData })
-    .nest('blocks', { type: GameDataParser })
+
+const GameParserFactory = (buildNo: number): any => {
+    return new Parser()
+        .nest('meta', { type: GameMetaDataParserFactory(buildNo) })
+        .nest('blocks', { type: GameDataParser })
+}
+
 const EventEmitter = require('events')
 
 class ReplayParser extends EventEmitter {
@@ -40,7 +44,7 @@ class ReplayParser extends EventEmitter {
         this.decompressed = Buffer.from('')
     }
 
-    parse ($buffer: string | Buffer) {
+    parse ($buffer: string | Buffer): void {
         this.msElapsed = 0
         this.buffer = Buffer.isBuffer($buffer) ? $buffer : readFileSync($buffer)
         this.buffer = this.buffer.slice(this.buffer.indexOf('Warcraft III recorded game'))
@@ -63,7 +67,7 @@ class ReplayParser extends EventEmitter {
         })
         this.decompressed = Buffer.concat(decompressed)
 
-        this.gameMetaDataDecoded = GameDataParserComposed.parse(this.decompressed)
+        this.gameMetaDataDecoded = GameParserFactory(this.header.buildNo).parse(this.decompressed)
         const decodedMetaStringBuffer = this.decodeGameMetaString(this.gameMetaDataDecoded.meta.encodedString)
         const meta = { ...this.gameMetaDataDecoded, ...this.gameMetaDataDecoded.meta, ...EncodedMapMetaString.parse(decodedMetaStringBuffer) }
         const newMeta = meta

diff --git a/src/W3GReplay.ts b/src/W3GReplay.ts
@@ -297,6 +297,7 @@ class W3GReplay extends ReplayParser {
                 checksum: this.meta.mapChecksum
             },
             version: convert.gameVersion(this.header.version),
+            buildNumber: this.header.buildNo,
             duration: this.header.replayLengthMS,
             expansion: this.header.gameIdentifier === 'PX3W',
             settings,

diff --git a/src/parsers/actions.ts b/src/parsers/actions.ts
@@ -279,7 +279,8 @@ const ActionBlock = new Parser()
             0x6c: new Parser(),
             0x6d: new Parser(),
             0x75: UnknownAction75,
-            0x7a: new Parser().skip(20)
+            0x7a: new Parser().skip(20),
+            0x7B: new Parser().buffer('data', { length: 16 })
         }
     })
 

diff --git a/src/parsers/header.ts b/src/parsers/header.ts
@@ -27,13 +27,24 @@ const SubHeaderV0 = new Parser()
   .int32le('checksum')
 */
 
+const DataBlockVanilla = new Parser()
+    .uint16le('blockSize')
+    .uint16le('blockDecompressedSize')
+    .string('unknown', { encoding: 'hex', length: 4 })
+    .buffer('compressed', { length: 'blockSize' })
+
 const DataBlock = new Parser()
-    .int16le('blockSize')
-    .int16le('blockDecompressedSize')
+    .uint16le('blockSize')
+    .skip(2)
+    .uint16le('blockDecompressedSize')
     .string('unknown', { encoding: 'hex', length: 4 })
+    .skip(2)
     .buffer('compressed', { length: 'blockSize' })
 
-const DataBlocks = new Parser()
+const DataBlocksVanilla = new Parser()
+    .array('blocks', { type: DataBlockVanilla, readUntil: 'eof' })
+
+const DataBlocksReforged = new Parser()
     .array('blocks', { type: DataBlock, readUntil: 'eof' })
 
 const ReplayHeader = new Parser()
@@ -42,7 +53,15 @@ const ReplayHeader = new Parser()
         type: Header
     })
     .nest(null, { type: SubHeaderV1 })
-    .nest(null, { type: DataBlocks })
+    .choice(null, {
+        tag: function () {
+            return this.buildNo < 6089 ? 1 : 0
+        },
+        choices: {
+            1: DataBlocksVanilla
+        },
+        defaultChoice: DataBlocksReforged
+    })
 
 const PlayerRecordLadder = new Parser()
     .string('runtimeMS', { encoding: 'hex', length: 4 })
@@ -127,6 +146,62 @@ const GameMetaData = new Parser()
     .string('selectMode', { length: 1, encoding: 'hex' })
     .int8('startSpotCount')
 
+const GameMetaDataReforged = (buildNo: number) => new Parser()
+    .skip(5)
+    .nest('player', { type: HostRecord })
+    .string('gameName', { zeroTerminated: true })
+    .skip(1)
+    .string('encodedString', { zeroTerminated: true, encoding: 'hex' })
+    .int32le('playerCount')
+    .string('gameType', { length: 4, encoding: 'hex' })
+    .string('languageId', { length: 4, encoding: 'hex' })
+    .array('playerList', {
+        type: new Parser()
+            .int8('hasRecord')
+        // @ts-ignore
+            .choice(null, {
+                tag: 'hasRecord',
+                choices: {
+                    22: PlayerRecordInList
+
+                },
+                defaultChoice: new Parser().skip(-1)
+            }),
+        readUntil (item, buffer) {
+            // @ts-ignore
+            const next = buffer.readInt8()
+            return next === 57
+        }
+    })
+    .skip(4) // GamestartRecord etc used to go here
+    .skip(8) // More stuff that happens before the next list of players
+    .array('extraPlayerList', {
+        type: new Parser()
+            .int8('preVars1')
+            .buffer('pre', { length: 4 })
+            .int8('nameLength')
+            .string('name', { length: 'nameLength' })
+            .skip(1)
+            .int8('clanLength')
+            .string('clan', { length: 'clanLength' })
+            .skip(1)
+            .int8('extraLength')
+            .buffer('extra', { length: 'extraLength' })
+            .buffer('post', { length: buildNo >= 6103 ? 4 : 2 }),
+        readUntil (item, buffer) {
+            // @ts-ignore
+            const next = buffer.readInt8()
+            return next === 25
+        }
+    })
+    .int8('gameStartRecord')
+    .int16('dataByteCount')
+    .int8('slotRecordCount')
+    .array('playerSlotRecords', { type: PlayerSlotRecord, length: 'slotRecordCount' })
+    .int32le('randomSeed')
+    .string('selectMode', { length: 1, encoding: 'hex' })
+    .int8('startSpotCount')
+
 const EncodedMapMetaString = new Parser()
     .uint8('speed')
     .bit1('hideTerrain')
@@ -148,9 +223,17 @@ const EncodedMapMetaString = new Parser()
     .string('mapName', { zeroTerminated: true })
     .string('creator', { zeroTerminated: true })
 
+const GameMetaDataParserFactory = (buildNo: number) => {
+    if (buildNo <= 6091) {
+        return GameMetaData
+    } else {
+        return GameMetaDataReforged(buildNo)
+    }
+}
+
 export {
     ReplayHeader,
     EncodedMapMetaString,
-    GameMetaData,
+    GameMetaDataParserFactory,
     DataBlock
 }
diff --git a/src/types.ts b/src/types.ts
@@ -137,6 +137,7 @@ export interface ParserOutput {
         file: string;
         checksum: GameMetaDataDecoded['mapChecksum'];
     };
+    buildNumber: number;
     version: string;
     duration: number;
     expansion: boolean;

diff --git a/test/replays.test.ts b/test/replays.test.ts
@@ -192,12 +192,28 @@ describe('Replay parsing tests', () => {
     it('parses a standard 1.30.4 2on2 replay properly', () => {
         const test = Parser.parse('./replays/standard_1304.2on2.w3g')
         expect(test.version).toBe('1.30.2+')
+        expect(test.buildNumber).toBe(6061)
         expect(test.players.length).toBe(4)
     })
 
+    it('parses a reforged replay properly #1', () => {
+        const test = Parser.parse('./replays/reforged1.w3g')
+        expect(test.version).toBe('1.32')
+        expect(test.buildNumber).toBe(6091)
+        expect(test.players.length).toBe(2)
+    })
+
+    it('parses a reforged replay properly #2', () => {
+        const test = Parser.parse('./replays/reforged2.w3g')
+        expect(test.version).toBe('1.32')
+        expect(test.buildNumber).toBe(6091)
+        expect(test.players.length).toBe(2)
+    })
+
     it('parses a standard 1.30.4 1on1 tome of retraining', () => {
         const test = Parser.parse('./replays/standard_tomeofretraining_1.w3g')
         expect(test.version).toBe('1.31')
+        expect(test.buildNumber).toBe(6072)
         expect(test.players.length).toBe(2)
         expect(test.players[0].heroes[0]).toEqual({
             id: 'Hamg',
@@ -281,4 +297,21 @@ describe('Replay parsing tests', () => {
         expect(test.players[0].currentTimePlayed).toEqual(4371069)
         expect(Parser.msElapsed).toEqual(6433136)
     })
+
+    it('parses a replay with new reforged metadata successfully', () => {
+        const test = Parser.parse('./replays/reforged2010.w3g')
+        expect(test.version).toBe('1.32')
+        expect(test.buildNumber).toBe(6102)
+        expect(test.players.length).toBe(6)
+        expect(test.players[0].name).toBe('BEARAND#1604')
+    })
+
+    it('parses a reforged replay of version 1.32, build 6105 successfully', () => {
+        const test = Parser.parse('./replays/reforged_release.w3g')
+        expect(test.version).toBe('1.32')
+        expect(test.buildNumber).toBe(6105)
+        expect(test.players.length).toBe(2)
+        expect(test.players[0].name).toBe('anXieTy#2932')
+        expect(test.players[1].name).toBe('IroNSoul#22724')
+    })
 })
diff --git a/test/schema.json b/test/schema.json
@@ -14,11 +14,15 @@
     "creator",
     "map",
     "version",
+    "buildNumber",
     "settings",
     "observers",
     "chat"
   ],
   "properties": {
+    "buildNumber": {
+      "type":"number"
+    },
     "players": {
       "$id": "#/properties/players",
       "type": "array",