diff --git a/build.gradle b/build.gradle index 5710cf7..4b4bd5c 100644 --- a/build.gradle +++ b/build.gradle @@ -5,6 +5,14 @@ apply plugin: 'maven-publish' apply from: 'shared.gradle' group = 'rehlds' +apply plugin: 'idea' + +idea { + project { + languageLevel = 'JDK_1_7' + } +} + def gitInfo = GitVersioner.versionForDir(project.rootDir) if (!gitInfo) { throw new RuntimeException('Running outside git repository') diff --git a/flightrec/decoder/build.gradle b/flightrec/decoder/build.gradle new file mode 100644 index 0000000..cdfccc4 --- /dev/null +++ b/flightrec/decoder/build.gradle @@ -0,0 +1,32 @@ +apply plugin: 'java' + +group = 'org.rehlds.flightrec' + +sourceCompatibility = '1.7' +targetCompatibility = '1.7' + +repositories { + mavenCentral() +} + +dependencies { + testCompile "junit:junit:4.12" + compile project(':flightrec/decoder_api') +} + +task uberjar(type: Jar, dependsOn: ['check', ':flightrec/decoder_api:build']) { + from files(sourceSets.main.output.classesDir) + from { configurations.runtime.collect { it.isDirectory() ? it : zipTree(it) } } + exclude('META-INF/*.DSA', 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.LIST') //exclude all signing stuff + + manifest { + attributes 'Main-Class': 'org.rehlds.flightrec.main.FlightRecorder' + attributes 'Implementation-Vendor': 'Sun Microsystems, Inc' + attributes 'Implementation-Title': 'Java Runtime Environment' + attributes 'Implementation-Version': '1.7.0' + } +} + +tasks.withType(AbstractCompile) { + options.encoding = 'UTF-8' +} diff --git a/flightrec/decoder/pub/decoder.bat b/flightrec/decoder/pub/decoder.bat new file mode 100644 index 0000000..1026be9 --- /dev/null +++ b/flightrec/decoder/pub/decoder.bat @@ -0,0 +1,50 @@ +@if "%DEBUG%" == "" @echo off +setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +set CMD_LINE_ARGS=%* +"%JAVA_EXE%" -jar "%DIRNAME%/decoder.jar" %CMD_LINE_ARGS% + +:end +goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +endlocal diff --git a/flightrec/decoder/src/main/java/com/google/cloud/Crc32c.java b/flightrec/decoder/src/main/java/com/google/cloud/Crc32c.java new file mode 100644 index 0000000..28f898a --- /dev/null +++ b/flightrec/decoder/src/main/java/com/google/cloud/Crc32c.java @@ -0,0 +1,156 @@ +// Copyright 2011 Google Inc. All rights reserved. + +package com.google.cloud; + +import java.util.zip.Checksum; + +/** + * This class generates a CRC32C checksum, defined by rfc3720 section B.4. + * + * + */ +public final class Crc32c implements Checksum { + + private static final long[] CRC_TABLE = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 + }; + + private static final long LONG_MASK = 0xffffffffL; + private static final long BYTE_MASK = 0xff; + + private long crc; + + public Crc32c() { + crc = 0; + } + + /** + * Updates the checksum with a new byte. + * @param b the new byte. + */ + @Override + public void update(int b) { + long newCrc = crc; + newCrc = updateByte((byte) b, newCrc); + crc = newCrc; + } + + /** + * Updates the checksum with an array of bytes. + * @param bArray the array of bytes. + * @param off the offset into the array where the update should begin. + * @param len the length of data to examine. + */ + @Override + public void update(byte[] bArray, int off, int len) { + long newCrc = crc; + for (int i = off; i < off + len; i++) { + newCrc = updateByte(bArray[i], newCrc); + } + crc = newCrc; + } + + public void update(byte[] bArray) { + update(bArray, 0, bArray.length); + } + + /** + * Returns the value of the checksum. + * @return the long representation of the checksum (high bits set to zero). + */ + @Override + public long getValue() { + return crc; + } + + /** + * Returns the value of the checksum. + * @return the 4-byte array representation of the checksum in network byte order (big endian). + */ + public byte[] getValueAsBytes() { + long value = crc; + byte[] result = new byte[4]; + for (int i = 3; i >= 0; i--) { + result[i] = (byte) (value & 0xffL); + value >>= 8; + } + return result; + } + + /** + * Resets the crc. + */ + @Override + public void reset() { + crc = 0; + } + + private long updateByte(byte newByte, long crc) { + byte b = (byte) (newByte & BYTE_MASK); + int index = (int) ((crc ^ b) & BYTE_MASK); + return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK; + } +} \ No newline at end of file diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/Consts.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/Consts.java new file mode 100644 index 0000000..7c307ee --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/Consts.java @@ -0,0 +1,14 @@ +package org.rehlds.flightrec; + +public class Consts { + public final static String META_HEADER_SIG_STR = "REHLDS_FLIGHTREC_META"; + public final static String DATA_HEADER_SIG_STR = "REHLDS_FLIGHTREC_DATA"; + + public static byte[] META_HEADER_SIG_BYTES = (META_HEADER_SIG_STR + META_HEADER_SIG_STR + META_HEADER_SIG_STR + ":").getBytes(); + public static byte[] DATA_HEADER_SIG_BYTES = (DATA_HEADER_SIG_STR + DATA_HEADER_SIG_STR + DATA_HEADER_SIG_STR + ":").getBytes(); + + public static int META_HEADER_SIZE = 128; + public static int DATA_HEADER_SIZE = 128; + + public static int MAX_HEADER_SIZE = Math.max(META_HEADER_SIZE, DATA_HEADER_SIZE); +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/AllocEntPrivateDataV1Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/AllocEntPrivateDataV1Decoder.java new file mode 100644 index 0000000..a02ef7b --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/AllocEntPrivateDataV1Decoder.java @@ -0,0 +1,22 @@ +package org.rehlds.flightrec.decoders.rehlds; + + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class AllocEntPrivateDataV1Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "AllocEntPrivateData", 1, false); + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + long ptr = sb.readUInt32(); + return DecodedExtraData.create("pPrivData", "0x" + Long.toHexString(ptr)); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV1Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV1Decoder.java new file mode 100644 index 0000000..e013329 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV1Decoder.java @@ -0,0 +1,30 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FrameV1Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "Frame", 1, true); + } + + DecodedExtraData decodeStart(UtilSizeBuf sb) { + double startTime = sb.readDouble(); + return DecodedExtraData.create("startTime", "" + startTime); + } + + DecodedExtraData decodeEnd(UtilSizeBuf sb) { + return DecodedExtraData.EMPTY; + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + return msg.isEnterMessage() ? decodeStart(sb) : decodeEnd(sb); + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV2Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV2Decoder.java new file mode 100644 index 0000000..d731005 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV2Decoder.java @@ -0,0 +1,32 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FrameV2Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "Frame", 2, true); + } + + DecodedExtraData decodeStart(UtilSizeBuf sb) { + long frameId = sb.readInt64(); + double startTime = sb.readDouble(); + return DecodedExtraData.create("frameId", "" + frameId, "startTime", "" + startTime); + } + + DecodedExtraData decodeEnd(UtilSizeBuf sb) { + long frameId = sb.readInt64(); + return DecodedExtraData.create("frameId", "" + frameId); + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + return msg.isEnterMessage() ? decodeStart(sb) : decodeEnd(sb); + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FreeEntPrivateDataV1Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FreeEntPrivateDataV1Decoder.java new file mode 100644 index 0000000..b306e92 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FreeEntPrivateDataV1Decoder.java @@ -0,0 +1,21 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FreeEntPrivateDataV1Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "FreeEntPrivateData", 1, false); + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + long ptr = sb.readUInt32(); + return DecodedExtraData.create("pPrivData", "0x" + Long.toHexString(ptr)); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/RehldsDecodersModule.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/RehldsDecodersModule.java new file mode 100644 index 0000000..e143e20 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/RehldsDecodersModule.java @@ -0,0 +1,15 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.SimpleDecoderModule; + +public class RehldsDecodersModule extends SimpleDecoderModule { + + public RehldsDecodersModule() { + super("Rehlds decoders (built-in)", "0.2"); + registerDecoder(new FrameV1Decoder()); + registerDecoder(new FreeEntPrivateDataV1Decoder()); + registerDecoder(new AllocEntPrivateDataV1Decoder()); + + registerDecoder(new FrameV2Decoder()); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FileScanResult.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FileScanResult.java new file mode 100644 index 0000000..ed91a5a --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FileScanResult.java @@ -0,0 +1,9 @@ +package org.rehlds.flightrec.filescan; + +import java.util.ArrayList; +import java.util.List; + +public class FileScanResult { + public List metaHeaders = new ArrayList<>(); + public List dataHeaders = new ArrayList<>(); +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java new file mode 100644 index 0000000..a67f82e --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java @@ -0,0 +1,100 @@ +package org.rehlds.flightrec.filescan; + +import com.google.cloud.Crc32c; +import org.rehlds.flightrec.api.util.UtilByteBuffer; +import org.rehlds.flightrec.api.util.UtilSizeBuf; +import static org.rehlds.flightrec.Consts.*; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Arrays; +import java.util.List; + + + +public class FlightRecFileScanner { + RandomAccessFile file; + long fileLen; + FileScanResult scanRes = new FileScanResult(); + + private byte readBuf[] = new byte[65536]; + private byte header[] = new byte[MAX_HEADER_SIZE]; + private UtilSizeBuf headerSBuf = new UtilSizeBuf("header", new UtilByteBuffer(header), 0, header.length); + + private FlightRecFileScanner(RandomAccessFile file) throws IOException { + this.file = file; + this.fileLen = file.length(); + } + + private void examineHeader(byte[] data, int size, int pos) throws IOException { + if (pos + MAX_HEADER_SIZE < size) { + System.arraycopy(data, pos, header, 0, MAX_HEADER_SIZE); + } else { + return; //will be read in next iteration + } + + headerSBuf.reset(); + String matchedType = null; + if (Arrays.equals(META_HEADER_SIG_BYTES, Arrays.copyOfRange(header, 0, META_HEADER_SIG_BYTES.length))) { + matchedType = META_HEADER_SIG_STR; + headerSBuf.skip(META_HEADER_SIG_BYTES.length); + } else if (Arrays.equals(DATA_HEADER_SIG_BYTES, Arrays.copyOfRange(header, 0, DATA_HEADER_SIG_BYTES.length))) { + matchedType = DATA_HEADER_SIG_STR; + headerSBuf.skip(DATA_HEADER_SIG_BYTES.length); + } + + if (matchedType == null) { + return; + } + + List resList = (matchedType.equals(META_HEADER_SIG_STR)) ? scanRes.metaHeaders : scanRes.dataHeaders; + int version = headerSBuf.readInt32(); + int allocSize = headerSBuf.readInt32(); + + Crc32c crc32 = new Crc32c(); + crc32.update(header, 0, headerSBuf.tell()); + long calculatedChecksum = crc32.getValue(); + long bufChecksum = headerSBuf.readUInt32(); + + if (calculatedChecksum != bufChecksum) { + resList.add(new HeaderScanResult(file.getFilePointer() - size + pos, allocSize, false, "Checksum mismatch", version)); + return; + } + + long endPos = file.getFilePointer() - size + pos + allocSize; + if (endPos >= file.length()) { + resList.add(new HeaderScanResult(file.getFilePointer() - size + pos, allocSize, false, "Regions partially lays outside the file", version)); + return; + } + resList.add(new HeaderScanResult(file.getFilePointer() - size + pos, allocSize, true, null, version)); + } + + private void scanForHeaders(byte[] data, int size) throws IOException { + int maxHeaderSize = Math.max(META_HEADER_SIG_STR.length(), DATA_HEADER_SIG_STR.length()); + for (int i = 0; i < size - maxHeaderSize; i++) { + if (data[i + 15] == META_HEADER_SIG_BYTES[15] && data[i + 16] == META_HEADER_SIG_BYTES[16] && data[i + 17] == META_HEADER_SIG_BYTES[17] && data[i + 18] == META_HEADER_SIG_BYTES[18]) { + examineHeader(data, size, i); + } else if (data[i + 15] == DATA_HEADER_SIG_BYTES[15] && data[i + 16] == DATA_HEADER_SIG_BYTES[16] && data[i + 17] == DATA_HEADER_SIG_BYTES[17] && data[i + 18] == DATA_HEADER_SIG_BYTES[18]) { + examineHeader(data, size, i); + } + } + } + + private void doScan() throws IOException { + file.seek(0); + int read; + + while (-1 != (read = file.read(readBuf))) { + scanForHeaders(readBuf, read); + if (read == readBuf.length) { + file.seek(file.getFilePointer() - MAX_HEADER_SIZE * 2); + } + } + } + + public static FileScanResult scan(RandomAccessFile file) throws IOException { + FlightRecFileScanner scanner = new FlightRecFileScanner(file); + scanner.doScan(); + return scanner.scanRes; + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java new file mode 100644 index 0000000..46e93c4 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java @@ -0,0 +1,17 @@ +package org.rehlds.flightrec.filescan; + +public class HeaderScanResult { + public long pos; + public int len; + public boolean valid; + public String error; + public int version; + + public HeaderScanResult(long pos, int len, boolean valid, String error, int version) { + this.pos = pos; + this.len = len; + this.valid = valid; + this.error = error; + this.version = version; + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/DataHeader.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/DataHeader.java new file mode 100644 index 0000000..c0b961f --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/DataHeader.java @@ -0,0 +1,12 @@ +package org.rehlds.flightrec.logparser; + +public class DataHeader { + public int prevItrLastPos; + + public DataHeader(int prevItrLastPos) { + this.prevItrLastPos = prevItrLastPos; + } + + public DataHeader() { + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/FlightLogParser.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/FlightLogParser.java new file mode 100644 index 0000000..cde2e84 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/FlightLogParser.java @@ -0,0 +1,179 @@ +package org.rehlds.flightrec.logparser; + +import org.rehlds.flightrec.api.EntranceKind; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageDef; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.filescan.HeaderScanResult; +import org.rehlds.flightrec.api.util.UtilByteBuffer; +import org.rehlds.flightrec.api.util.UtilSizeBuf; +import static org.rehlds.flightrec.Consts.*; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.*; + +public class FlightLogParser { + UtilByteBuffer metaRegion; + UtilByteBuffer dataRegion; + + MetaHeader metaHeader; + RecorderState recorderState; + DataHeader dataHeader; + + Map msgTypes = new HashMap<>(); + List messages = new ArrayList<>(); + + void parseMessageDefinition(UtilSizeBuf sbuf) { + int msgId = sbuf.readUInt16(); + String module = sbuf.readString(); + String messageName = sbuf.readString(); + long msgVersion = sbuf.readUInt32(); + boolean inOut = sbuf.readBool(); + + FlightrecMessageDef msgDef = new FlightrecMessageDef(module, messageName, msgVersion, inOut, msgId); + + if (msgTypes.containsKey(msgId)) { + System.out.println("Duplicate message id: " + msgTypes.get(msgId) + " and " + msgDef); + } + + msgTypes.put(msgId, msgDef.msgType); + } + + void parseMetaRegion() { + metaHeader = new MetaHeader(); + UtilSizeBuf metaSBuf = new UtilSizeBuf("meta region", metaRegion); + metaSBuf.skip(META_HEADER_SIG_BYTES.length); //skip signature + metaSBuf.readInt32(); //version + metaSBuf.readInt32(); //allocSize + metaSBuf.readInt32(); //checksum + metaHeader.numDefinitions = metaSBuf.readInt32(); + metaHeader.metaRegionPos = metaSBuf.readInt32(); + + recorderState = new RecorderState(); + recorderState.wpos = metaSBuf.readInt32(); + recorderState.lastMsgBeginPos = metaSBuf.readInt32(); + recorderState.curMessage = metaSBuf.readUInt16(); + + metaSBuf = new UtilSizeBuf("meta region defs", metaRegion, META_HEADER_SIZE, metaHeader.metaRegionPos); + for (int i = 0; i < metaHeader.numDefinitions; i++) { + int defKind = metaSBuf.readUInt8(); + switch (defKind) { + case 1: //MRT_MESSAGE_DEF + parseMessageDefinition(metaSBuf); + break; + + default: + throw new RuntimeException("Invalid meta definition type" + defKind); + } + } + + dataHeader = new DataHeader(); + dataHeader.prevItrLastPos = dataRegion.readInt32(DATA_HEADER_SIG_BYTES.length + 12); + } + + public FlightLogParser(UtilByteBuffer metaRegion, UtilByteBuffer dataRegion) { + this.metaRegion = metaRegion; + this.dataRegion = dataRegion; + } + + void doParseMessage(UtilSizeBuf msg) { + int opc = msg.readUInt16(); + boolean entrance = (0 != (opc & 0x8000)); + opc &= 0x7FFF; + + FlightrecMessageType msgType = msgTypes.get(opc); + if (msgType == null) { + throw new RuntimeException("Invalid message opcode @" + Long.toHexString(msg.getAbsoluteCurrentPos() - 2) + ": " + opc); + } + + EntranceKind entranceKind; + if (msgType.inout) { + entranceKind = entrance ? EntranceKind.ENTRANCE_ENTER : EntranceKind.ENTRANCE_LEAVE; + } else { + entranceKind = EntranceKind.ENTRANCE_UNUSED; + } + + FlightrecMessage flMsg = new FlightrecMessage(msgType, entranceKind, msg.getBuffer(), msg.getAbsoluteCurrentPos(), msg.getMaxSize() - 2); + messages.add(flMsg); + } + + void parseMessage(UtilSizeBuf msg) { + int startPos = msg.getStartPos(); + try { + doParseMessage(msg); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("Error while parsing message @" + startPos); + } + } + + List parse() { + parseMetaRegion(); + + UtilByteBuffer flightData = dataRegion.cutLeft(DATA_HEADER_SIZE); + + boolean flippedToEnd = false; + + /* + Each message has following layout: + Opcode [2 bytes] + Data [0+ bytes] + Length of opcode + data [2 bytes] + */ + int curMsgPos = (recorderState.curMessage == 0) ? recorderState.wpos : recorderState.lastMsgBeginPos; + curMsgPos -= 2; //position of the Length field of the message + + UtilSizeBuf msg = new UtilSizeBuf("flightrec_message", flightData, 0, 0); + + while (true) { + if (flippedToEnd && curMsgPos <= recorderState.wpos) + break; + + if (curMsgPos <= 0) { //move read pointer to the end of the data region + if (dataHeader.prevItrLastPos == -1) //wpos never reached end of the region + break; + + curMsgPos = dataHeader.prevItrLastPos - 2; + flippedToEnd = true; + continue; + } + + int msgLen = flightData.readUInt16(curMsgPos); + int msgStartPos = curMsgPos - msgLen; + if (msgStartPos < 0) { + throw new RuntimeException("Corrupted data region; read msgLen=" + msgLen + " at " + curMsgPos + ", but it is too large (startPos < 0)"); + } + + if (flippedToEnd && msgStartPos < recorderState.wpos) { + break; + } + + msg.init(msgStartPos, msgLen); + parseMessage(msg); + + curMsgPos = msgStartPos - 2; + } + + return messages; + } + + public static List doParse(RandomAccessFile f, HeaderScanResult metaHeader, HeaderScanResult dataHeader) throws IOException { + //read regions to byte buffers + f.seek(metaHeader.pos); + byte[] metaRegionData = new byte[metaHeader.len]; + f.readFully(metaRegionData); + + f.seek(dataHeader.pos); + byte[] dataRegionData = new byte[dataHeader.len]; + f.readFully(dataRegionData); + + UtilByteBuffer metaRegion = new UtilByteBuffer(metaRegionData); + UtilByteBuffer dataRegion = new UtilByteBuffer(dataRegionData); + + List res = new FlightLogParser(metaRegion, dataRegion).parse(); + Collections.reverse(res); + return res; + } +} + diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/LogParsingException.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/LogParsingException.java new file mode 100644 index 0000000..bd65f3d --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/LogParsingException.java @@ -0,0 +1,12 @@ +package org.rehlds.flightrec.logparser; + +public class LogParsingException extends RuntimeException { + + public LogParsingException(String message) { + super(message); + } + + public LogParsingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/MetaHeader.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/MetaHeader.java new file mode 100644 index 0000000..19b3eba --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/MetaHeader.java @@ -0,0 +1,14 @@ +package org.rehlds.flightrec.logparser; + +public class MetaHeader { + public int numDefinitions; + public int metaRegionPos; + + public MetaHeader(int numMessages, int metaRegionPos) { + this.numDefinitions = numMessages; + this.metaRegionPos = metaRegionPos; + } + + public MetaHeader() { + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/RecorderState.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/RecorderState.java new file mode 100644 index 0000000..a965b64 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/RecorderState.java @@ -0,0 +1,16 @@ +package org.rehlds.flightrec.logparser; + +public class RecorderState { + public int wpos; + public int lastMsgBeginPos; + public int curMessage; + + public RecorderState(int wpos, int lastMsgBeginPos, int curMessage) { + this.wpos = wpos; + this.lastMsgBeginPos = lastMsgBeginPos; + this.curMessage = curMessage; + } + + public RecorderState() { + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/FlightLogTreeBuilder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/FlightLogTreeBuilder.java new file mode 100644 index 0000000..9648822 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/FlightLogTreeBuilder.java @@ -0,0 +1,71 @@ +package org.rehlds.flightrec.logtree; + +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; + +import java.util.List; + +public class FlightLogTreeBuilder { + LogTreeNodeComplex rootNode = new LogTreeNodeComplex(null, null, null); + LogTreeNodeComplex currentNode = rootNode; + + void handleEnterMessage(FlightrecMessage msg) { + LogTreeNodeComplex n = new LogTreeNodeComplex(currentNode, msg, null); + currentNode.addChild(n); + currentNode = n; + } + + void handleLeaveMessage(FlightrecMessage msg) { + if (currentNode == rootNode) { + rootNode = new LogTreeNodeComplex(null, null, msg); + rootNode.addChild(currentNode); + currentNode.setParent(rootNode); + currentNode = rootNode; + return; + } + + if (currentNode.enterMsg != null) { + FlightrecMessageType startType = currentNode.enterMsg.messageType; + FlightrecMessageType endType = msg.messageType; + if (!startType.equals(endType)) { + throw new RuntimeException("Closing message @" + Long.toHexString(msg.rawDataPos) + " has invalid type " + endType + "; expected " + startType); + } + } + + currentNode.leaveMsg = msg; + currentNode = currentNode.parent; + } + + void handleSimpleMessage(FlightrecMessage msg) { + LogTreeNodeLeaf leafNode = new LogTreeNodeLeaf(currentNode, msg); + currentNode.addChild(leafNode); + } + + void doBuildLogTree(List messages) { + for (FlightrecMessage msg : messages) { + switch (msg.entranceKind) { + case ENTRANCE_ENTER: + handleEnterMessage(msg); + break; + + case ENTRANCE_LEAVE: + handleLeaveMessage(msg); + break; + + case ENTRANCE_UNUSED: + handleSimpleMessage(msg); + break; + + default: + throw new RuntimeException("Invalid exntrance kind"); + } + } + } + + + public static LogTreeNodeComplex buildTree(List messages) { + FlightLogTreeBuilder builder = new FlightLogTreeBuilder(); + builder.doBuildLogTree(messages); + return builder.rootNode; + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNode.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNode.java new file mode 100644 index 0000000..de6d14f --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNode.java @@ -0,0 +1,21 @@ +package org.rehlds.flightrec.logtree; + +import java.util.List; + +public abstract class LogTreeNode { + LogTreeNodeComplex parent; + + protected LogTreeNode(LogTreeNodeComplex parent) { + this.parent = parent; + } + + abstract List getChildren(); + + LogTreeNodeComplex getParent() { + return parent; + } + + public void setParent(LogTreeNodeComplex parent) { + this.parent = parent; + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeComplex.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeComplex.java new file mode 100644 index 0000000..3f99094 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeComplex.java @@ -0,0 +1,36 @@ +package org.rehlds.flightrec.logtree; + +import org.rehlds.flightrec.api.FlightrecMessage; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class LogTreeNodeComplex extends LogTreeNode { + public FlightrecMessage enterMsg; + public FlightrecMessage leaveMsg; + + public LogTreeNodeComplex(LogTreeNodeComplex parent, FlightrecMessage enterMsg, FlightrecMessage leaveMsg) { + super(parent); + this.enterMsg = enterMsg; + this.leaveMsg = leaveMsg; + } + + List children = Collections.emptyList(); + + + + @Override + public List getChildren() { + return children; + } + + public void addChild(LogTreeNode node) { + if (children.isEmpty()) { + children = new ArrayList<>(); + } + + children.add(node); + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeLeaf.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeLeaf.java new file mode 100644 index 0000000..bfdb6ca --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeLeaf.java @@ -0,0 +1,20 @@ +package org.rehlds.flightrec.logtree; + +import org.rehlds.flightrec.api.FlightrecMessage; + +import java.util.Collections; +import java.util.List; + +public class LogTreeNodeLeaf extends LogTreeNode { + public FlightrecMessage msg; + + public LogTreeNodeLeaf(LogTreeNodeComplex parent, FlightrecMessage msg) { + super(parent); + this.msg = msg; + } + + @Override + List getChildren() { + return Collections.emptyList(); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/FlightRecorder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/FlightRecorder.java new file mode 100644 index 0000000..231ae8a --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/FlightRecorder.java @@ -0,0 +1,229 @@ +package org.rehlds.flightrec.main; + +import org.rehlds.flightrec.api.DecoderModule; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.decoders.rehlds.RehldsDecodersModule; +import org.rehlds.flightrec.filescan.FileScanResult; +import org.rehlds.flightrec.filescan.FlightRecFileScanner; +import org.rehlds.flightrec.filescan.HeaderScanResult; +import org.rehlds.flightrec.logtree.FlightLogTreeBuilder; +import org.rehlds.flightrec.logtree.LogTreeNodeComplex; +import org.rehlds.flightrec.logparser.FlightLogParser; +import org.rehlds.flightrec.textlogwriter.TextLogWriter; +import org.rehlds.flightrec.util.JarUtils; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; + +public class FlightRecorder { + RunConfig cfg; + List decoderModules = new ArrayList<>(); + + public FlightRecorder(RunConfig cfg) { + this.cfg = cfg; + } + + private boolean checkConfig() { + if (cfg.dumpFile == null) { + System.out.println("Dump file is not selected, please use --dump-file parameter to specify it"); + return false; + } + + if (cfg.outFile == null) { + cfg.outFile = new File(cfg.dumpFile.getAbsolutePath() + ".flog"); + } + + return true; + } + + private List scanFile(RandomAccessFile f) throws IOException { + FileScanResult scanResult = FlightRecFileScanner.scan(f); + + System.out.println("Dump file scan results: "); + for (HeaderScanResult metaHeader : scanResult.metaHeaders) { + System.out.print("\tMeta header @ 0x" + Long.toHexString(metaHeader.pos) + "; valid: " + (metaHeader.error == null)); + if (metaHeader.error != null) { + System.out.print("; error: " + metaHeader.error); + } + System.out.println(); + } + for (HeaderScanResult dataHeader : scanResult.dataHeaders) { + System.out.print("\tData header @ 0x" + Long.toHexString(dataHeader.pos) + "; valid: " + (dataHeader.error == null)); + if (dataHeader.error != null) { + System.out.print("; error: " + dataHeader.error); + } + System.out.println(); + } + + HeaderScanResult validMetaHeader = null; + HeaderScanResult validDataHeader = null; + + for (HeaderScanResult metaHeader : scanResult.metaHeaders) { + if (metaHeader.error != null) { + continue; + } + + if (validMetaHeader != null) { + System.out.println("Multiple meta headers found, exiting"); + return null; + } + + validMetaHeader = metaHeader; + } + + for (HeaderScanResult dataHeader : scanResult.dataHeaders) { + if (dataHeader.error != null) { + continue; + } + + if (validDataHeader != null) { + System.out.println("Multiple data headers found, exiting"); + return null; + } + + validDataHeader = dataHeader; + } + + if (validMetaHeader == null) { + System.out.println("Meta header not found, exiting"); + return null; + } + + if (validDataHeader == null) { + System.out.println("Data header not found, exiting"); + return null; + } + + return FlightLogParser.doParse(f, validMetaHeader, validDataHeader); + } + + private LogTreeNodeComplex buildTree(List messages) { + return FlightLogTreeBuilder.buildTree(messages); + } + + private boolean writeOutputFile(LogTreeNodeComplex logTreeRoot) { + TextLogWriter.decodeAndWrite(logTreeRoot, cfg.outFile, decoderModules); + System.out.println("Written decoded log to '" + cfg.outFile.getAbsolutePath() + ";"); + return true; + } + + public boolean run() { + registerBuiltinDecoders(); + loadExternalDecoders(); + + if (!checkConfig()) { + return false; + } + + List messages; + try(RandomAccessFile f = new RandomAccessFile(cfg.dumpFile, "r")) { + messages = scanFile(f); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + + if (messages == null) { + return false; + } + + LogTreeNodeComplex treeRootNode = buildTree(messages); + if (treeRootNode == null) { + return false; + } + + if (!writeOutputFile(treeRootNode)) { + return false; + } + + return true; + } + + private void loadExternalDecoders() { + File f = JarUtils.getJarFileOfClass(FlightRecorder.class); + if (f == null) { + System.out.println("Could not locate main JAR, external decoders will not be loaded"); + return; + } + + File extDir = new File(f.getParentFile(), "extDecoders"); + if (!extDir.exists() || !extDir.isDirectory()) { + System.out.println("Directory '" + extDir.getAbsolutePath() + "' doesn't exist"); + } + + File[] jarFiles = extDir.listFiles(new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.getName().toLowerCase().endsWith(".jar"); + } + }); + + ArrayList jarUrls = new ArrayList<>(); + for (File jf : jarFiles) { + try { + jarUrls.add(jf.toURI().toURL()); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + URLClassLoader extDecodersClassloader = new URLClassLoader(jarUrls.toArray(new URL[jarUrls.size()]), this.getClass().getClassLoader()); + ServiceLoader srvLoader = ServiceLoader.load(DecoderModule.class, extDecodersClassloader); + for (DecoderModule decoderModule : srvLoader) { + System.out.println("Loaded external decoder module " + decoderModule.getDescription() + " version " + decoderModule.getVersion()); + decoderModules.add(decoderModule); + } + } + + private void registerBuiltinDecoders() { + decoderModules.add(new RehldsDecodersModule()); + } + + public static void main(String args[]) { + RunConfig cfg; + try { + cfg = parseArgs(args); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + return; + } + + new FlightRecorder(cfg).run(); + } + + private static RunConfig parseArgs(String args[]) { + RunConfig cfg = new RunConfig(); + for (int i = 0; i < args.length; i++) { + String a = args[i]; + + if ("--dump-file".equals(a)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--dump-file should be followed by file name"); + } + i++; + cfg.dumpFile = new File(args[i]); + continue; + } + + if ("--out-file".equals(a)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--out-file should be followed by file name"); + } + i++; + cfg.outFile = new File(args[i]); + continue; + } + + throw new IllegalArgumentException("Invalid command line parameter: '" + a + "'"); + } + + return cfg; + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/RunConfig.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/RunConfig.java new file mode 100644 index 0000000..191dd33 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/RunConfig.java @@ -0,0 +1,8 @@ +package org.rehlds.flightrec.main; + +import java.io.File; + +public class RunConfig { + public File dumpFile; + public File outFile; +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/textlogwriter/TextLogWriter.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/textlogwriter/TextLogWriter.java new file mode 100644 index 0000000..4ec3e2b --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/textlogwriter/TextLogWriter.java @@ -0,0 +1,182 @@ +package org.rehlds.flightrec.textlogwriter; + +import org.rehlds.flightrec.api.*; +import org.rehlds.flightrec.logtree.LogTreeNode; +import org.rehlds.flightrec.logtree.LogTreeNodeComplex; +import org.rehlds.flightrec.logtree.LogTreeNodeLeaf; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TextLogWriter { + Writer writer; + List decoderModules; + + int indent; + HashMap indents = new HashMap<>(); + Map decodersByMsgType = new HashMap<>(); + + public TextLogWriter(Writer writer, List decoderModules) { + this.writer = writer; + this.decoderModules = decoderModules; + } + + MessageDecoder lookupDecoder(FlightrecMessageType msgType) { + for (DecoderModule dm : decoderModules) { + MessageDecoder d = dm.lookupDecoder(msgType); + if (d != null) { + return d; + } + } + + return null; + } + + MessageDecoder getDecoder(FlightrecMessage message) { + FlightrecMessageType msgType = message.messageType; + if (!decodersByMsgType.containsKey(msgType)) { + decodersByMsgType.put(msgType, lookupDecoder(msgType)); + } + + return decodersByMsgType.get(msgType); + } + + DecodedExtraData tryDecode(FlightrecMessage message) { + MessageDecoder decoder = getDecoder(message); + if (decoder == null) { + return null; + } + return decoder.decode(message); + } + + String escapeString(String s) { + return s.replace("\"", "\\\"").replace("'", "\\'"); + } + + String generateIndent() { + String res = indents.get(indent); + if (res != null) { + return res; + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < indent; i++) { + sb.append(" "); + } + + res = sb.toString(); + indents.put(indent, res); + return res; + } + + void writeExtraData(StringBuilder sb, DecodedExtraData extraData) { + boolean first = true; + for (ImmutablePair kv : extraData.data) { + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append(kv.first).append(": '").append(escapeString(kv.second)).append("'"); + } + } + + String prepareMessageText(FlightrecMessage msg) { + StringBuilder sb = new StringBuilder(); + sb.append(generateIndent()); + + switch (msg.entranceKind) { + case ENTRANCE_ENTER: + sb.append(">>"); + break; + + case ENTRANCE_LEAVE: + sb.append("<<"); + break; + + case ENTRANCE_UNUSED: + sb.append("--"); + break; + + default: + throw new RuntimeException("Invalid entrance kind " + msg.entranceKind); + } + + FlightrecMessageType msgType = msg.messageType; + sb.append(" ").append(msgType.module).append(".").append(msgType.message).append(":").append(msgType.version).append(" "); + + DecodedExtraData extraData = tryDecode(msg); + if (extraData != null) { + writeExtraData(sb, extraData); + } else { + sb.append("undecoded["); + boolean firstByte = true; + for (int i = msg.rawDataPos; i < msg.rawDataLen + msg.rawDataPos; i++) { + if (firstByte) { + firstByte = false; + } else { + sb.append(" "); + } + sb.append(String.format("%02X", msg.rawData[i] & 0xFF)); + } + sb.append("]"); + } + + sb.append("\n"); + return sb.toString(); + } + + void writeMessage(FlightrecMessage msg) throws IOException { + String text = prepareMessageText(msg); + writer.write(text); + } + + void writeLeafNode(LogTreeNodeLeaf node) throws IOException { + writeMessage(node.msg); + } + + void writeComplexNode(LogTreeNodeComplex node) throws IOException { + if (node.enterMsg != null) { + writeMessage(node.enterMsg); + } else { + writer.write(generateIndent() + ">> [Unknown]"); + } + + indent++; + writeNodes(node.getChildren()); + indent--; + + if (node.leaveMsg != null) { + writeMessage(node.leaveMsg); + } else { + writer.write(generateIndent() + "<< [Unknown]"); + } + } + + void writeNodes(List nodes) throws IOException { + for (LogTreeNode node : nodes) { + if (node instanceof LogTreeNodeComplex) { + writeComplexNode((LogTreeNodeComplex) node); + } else if (node instanceof LogTreeNodeLeaf) { + writeLeafNode((LogTreeNodeLeaf) node); + } else { + throw new RuntimeException("Invalid node class " + node.getClass().getName()); + } + } + } + + + public static void decodeAndWrite(LogTreeNodeComplex rootNode, File outFile, List decoderModules) { + try (FileWriter fw = new FileWriter(outFile)) { + TextLogWriter logWriter = new TextLogWriter(fw, decoderModules); + logWriter.writeNodes(rootNode.getChildren()); + } catch (IOException e) { + throw new RuntimeException("Failed to open/write file '" + outFile + "': " + e.getMessage(), e); + } + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/util/JarUtils.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/util/JarUtils.java new file mode 100644 index 0000000..41d062a --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/util/JarUtils.java @@ -0,0 +1,31 @@ +package org.rehlds.flightrec.util; + +import java.io.File; +import java.net.URL; + +public class JarUtils { + public static File getJarFileOfClass(Class c) { + String classFileName = c.getName().replace('.', '/') + ".class"; + ClassLoader classLoader = c.getClassLoader(); + if (classLoader == null) { + classLoader = JarUtils.class.getClassLoader(); + } + URL url = classLoader.getResource(classFileName); + if (url == null) { + return null; + } + + String strUrl = url.toString(); + if (!strUrl.startsWith("jar:file:/")) { + return null; + } + + int jarSeparator = strUrl.indexOf('!'); + if (jarSeparator == -1) { + return null; + } + + String jarFilePath = strUrl.substring("jar:file:/".length(), jarSeparator); + return new File(jarFilePath); + } +} diff --git a/flightrec/decoder/src/test/java/com/google/cloud/Crc32cTest.java b/flightrec/decoder/src/test/java/com/google/cloud/Crc32cTest.java new file mode 100644 index 0000000..51328da --- /dev/null +++ b/flightrec/decoder/src/test/java/com/google/cloud/Crc32cTest.java @@ -0,0 +1,43 @@ +package com.google.cloud; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class Crc32cTest { + + static class TestData { + public String src; + public long hash; + + TestData(String src, long hash) { + this.src = src; + this.hash = hash; + } + } + + @Test + public void testCrc32c() { + TestData testData[] = { + new TestData("a", 0x93AD1061L), + new TestData("ab", 0x13C35EE4L), + new TestData("abc", 0x562F9CCDL), + new TestData("abcd", 0xDAAF41F6L), + new TestData("abcde", 0x8122A0A2L), + new TestData("abcdef", 0x0496937BL), + new TestData("abcdefg", 0x5D199E2CL), + new TestData("abcdefgh", 0x86BC933DL), + new TestData("abcdefghi", 0x9639F15FL), + new TestData("abcdefghij", 0x0584645CL), + }; + + for (TestData t : testData) { + Crc32c crc32c = new Crc32c(); + crc32c.update(t.src.getBytes()); + long cksum = crc32c.getValue(); + + assertEquals(t.hash, cksum); + } + } + +} \ No newline at end of file diff --git a/flightrec/decoder_api/build.gradle b/flightrec/decoder_api/build.gradle new file mode 100644 index 0000000..574f92d --- /dev/null +++ b/flightrec/decoder_api/build.gradle @@ -0,0 +1,57 @@ +apply plugin: 'java' +apply plugin: 'maven-publish' + +group = 'org.rehlds.flightrec' + +sourceCompatibility = '1.7' +targetCompatibility = '1.7' + +repositories { + mavenCentral() +} + +dependencies { + testCompile "junit:junit:4.12" +} + +publishing { + publications { + maven(MavenPublication) { + version rootProject.version + artifactId 'decoder-api' + artifact jar + + pom.withXml { + asNode().children().last() + { + resolveStrategy = DELEGATE_FIRST + name 'decoder-api' + description project.description + //url github + //scm { + // url "${github}.git" + // connection "scm:git:${github}.git" + //} + /* + licenses { + license { + name 'The Apache Software License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution 'repo' + } + } + developers { + developer { + id 'dreamstalker' + name 'dreamstalker' + } + } + */ + } + } + } + } +} + +tasks.withType(AbstractCompile) { + options.encoding = 'UTF-8' +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java new file mode 100644 index 0000000..a5564a5 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java @@ -0,0 +1,28 @@ +package org.rehlds.flightrec.api; + +public class DecodedExtraData { + public ImmutablePair[] data; + + public DecodedExtraData(ImmutablePair[] data) { + this.data = data; + } + + public static final DecodedExtraData EMPTY = new DecodedExtraData(new ImmutablePair[0]); + + public static DecodedExtraData create(String... args) { + if ((args.length % 2) == 1) { + throw new RuntimeException("DecodedExtraData.create: number of arguments must be even"); + } + + int numPairs = args.length / 2; + DecodedExtraData res = new DecodedExtraData(new ImmutablePair[numPairs]); + + for (int i = 0; i < numPairs; i++) { + res.data[i] = new ImmutablePair<>(args[i * 2], args[i * 2 + 1]); + } + + return res; + } + + +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecoderModule.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecoderModule.java new file mode 100644 index 0000000..3ed144a --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecoderModule.java @@ -0,0 +1,7 @@ +package org.rehlds.flightrec.api; + +public interface DecoderModule { + public MessageDecoder lookupDecoder(FlightrecMessageType msgType); + public String getDescription(); + public String getVersion(); +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/EntranceKind.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/EntranceKind.java new file mode 100644 index 0000000..9f96aa2 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/EntranceKind.java @@ -0,0 +1,7 @@ +package org.rehlds.flightrec.api; + +public enum EntranceKind { + ENTRANCE_ENTER, + ENTRANCE_LEAVE, + ENTRANCE_UNUSED, +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java new file mode 100644 index 0000000..6d3011d --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java @@ -0,0 +1,35 @@ +package org.rehlds.flightrec.api; + +import org.rehlds.flightrec.api.util.UtilByteBuffer; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FlightrecMessage { + public final FlightrecMessageType messageType; + public final EntranceKind entranceKind; + + public final byte[] rawData; + public final int rawDataPos; + public final int rawDataLen; + + DecodedExtraData decodedData; + + public FlightrecMessage(FlightrecMessageType messageType, EntranceKind entranceKind, byte[] rawData, int rawDataOffset, int rawDataLen) { + this.messageType = messageType; + this.entranceKind = entranceKind; + this.rawData = rawData; + this.rawDataPos = rawDataOffset; + this.rawDataLen = rawDataLen; + } + + public UtilSizeBuf getDataSizebuf() { + return new UtilSizeBuf("msg: '" + messageType + "' @" + rawDataPos, new UtilByteBuffer(rawData), rawDataPos, rawDataLen); + } + + public boolean isEnterMessage() { + return (entranceKind == EntranceKind.ENTRANCE_ENTER); + } + + public boolean isLeaveMessage() { + return (entranceKind == EntranceKind.ENTRANCE_LEAVE); + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageDef.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageDef.java new file mode 100644 index 0000000..4c40a48 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageDef.java @@ -0,0 +1,22 @@ +package org.rehlds.flightrec.api; + +public class FlightrecMessageDef { + public final FlightrecMessageType msgType; + public final int opcode; + + public FlightrecMessageDef(String module, String message, long version, boolean inout, int opcode) { + msgType = new FlightrecMessageType(module, message, version, inout); + this.opcode = opcode; + } + + @Override + public String toString() { + return "FlightrecMessageDef{" + + "module='" + msgType.module + '\'' + + ", message='" + msgType.message + '\'' + + ", version=" + msgType.version + + ", inout=" + msgType.inout + + ", opcode=" + opcode + + '}'; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageType.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageType.java new file mode 100644 index 0000000..102cc49 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageType.java @@ -0,0 +1,49 @@ +package org.rehlds.flightrec.api; + +public class FlightrecMessageType { + public final String module; + public final String message; + public final long version; + public final boolean inout; + + public FlightrecMessageType(String module, String message, long version, boolean inout) { + this.module = module; + this.message = message; + this.version = version; + this.inout = inout; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FlightrecMessageType that = (FlightrecMessageType) o; + + if (inout != that.inout) return false; + if (version != that.version) return false; + if (!message.equals(that.message)) return false; + if (!module.equals(that.module)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = module.hashCode(); + result = 31 * result + message.hashCode(); + result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + (inout ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "FlightrecMessageType{" + + "module='" + module + '\'' + + ", message='" + message + '\'' + + ", version=" + version + + ", inout=" + inout + + '}'; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/ImmutablePair.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/ImmutablePair.java new file mode 100644 index 0000000..ed6f7ef --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/ImmutablePair.java @@ -0,0 +1,11 @@ +package org.rehlds.flightrec.api; + +public class ImmutablePair { + public final T first; + public final U second; + + public ImmutablePair(T first, U second) { + this.first = first; + this.second = second; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/MessageDecoder.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/MessageDecoder.java new file mode 100644 index 0000000..69d3f51 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/MessageDecoder.java @@ -0,0 +1,6 @@ +package org.rehlds.flightrec.api; + +public interface MessageDecoder { + FlightrecMessageType getMessageType(); + DecodedExtraData decode(FlightrecMessage msg); +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/SimpleDecoderModule.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/SimpleDecoderModule.java new file mode 100644 index 0000000..a1c3ead --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/SimpleDecoderModule.java @@ -0,0 +1,35 @@ +package org.rehlds.flightrec.api; + +import java.util.HashMap; +import java.util.Map; + +public class SimpleDecoderModule implements DecoderModule { + Map decoders = new HashMap<>(); + + public final String description; + public final String version; + + public SimpleDecoderModule(String description, String version) { + this.description = description; + this.version = version; + } + + @Override + public MessageDecoder lookupDecoder(FlightrecMessageType msgType) { + return decoders.get(msgType); + } + + @Override + public String getDescription() { + return description; + } + + @Override + public String getVersion() { + return version; + } + + public void registerDecoder(MessageDecoder msgDecoder) { + decoders.put(msgDecoder.getMessageType(), msgDecoder); + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/Globals.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/Globals.java new file mode 100644 index 0000000..ba583b7 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/Globals.java @@ -0,0 +1,8 @@ +package org.rehlds.flightrec.api.util; + +import java.nio.charset.Charset; + +public class Globals { + public static final Charset UTF8 = Charset.forName("UTF-8"); + +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/SizebufOverflowException.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/SizebufOverflowException.java new file mode 100644 index 0000000..ad4ec97 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/SizebufOverflowException.java @@ -0,0 +1,10 @@ +package org.rehlds.flightrec.api.util; + +public class SizebufOverflowException extends RuntimeException { + public final String sizebufName; + + public SizebufOverflowException(String sizebufName) { + super(sizebufName + " overflowed"); + this.sizebufName = sizebufName; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilByteBuffer.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilByteBuffer.java new file mode 100644 index 0000000..aa76d21 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilByteBuffer.java @@ -0,0 +1,79 @@ +package org.rehlds.flightrec.api.util; + +public class UtilByteBuffer { + byte data[]; + + public UtilByteBuffer(byte[] data) { + this.data = data; + } + + public byte[] getData() { + return data; + } + + public UtilByteBuffer cutLeft(int newStart) { + byte[] newData = new byte[data.length - newStart]; + System.arraycopy(data, newStart, newData, 0, data.length - newStart); + return new UtilByteBuffer(newData); + } + + public int getDataLength() { + return data.length; + } + + public int readUInt8(int pos) { + return data[pos] & 0xFF; + } + + public boolean readBool(int pos) { + return data[pos] != 0; + } + + public int readUInt16(int pos) { + return (data[pos] & 0xFF) | ((data[pos + 1] & 0xFF) << 8); + } + + public long readUInt32(int pos) { + return (data[pos] & 0xFF) | ((data[pos + 1] & 0xFF) << 8) | ((data[pos + 2] & 0xFF) << 16) | ((long)(data[pos + 3] & 0xFF) << 24); + } + + public int readInt32(int pos) { + return (data[pos] & 0xFF) | ((data[pos + 1] & 0xFF) << 8) | ((data[pos + 2] & 0xFF) << 16) | ((data[pos + 3] & 0xFF) << 24); + } + + public long readInt64(int pos) { + long lowBits = readUInt32(pos); + long highBits = readUInt32(pos + 4); + + return lowBits | (highBits << 32); + } + + public double readDouble(int pos) { + long bits = readInt64(pos); + return Double.longBitsToDouble(bits); + } + + public float readFloat(int pos) { + int bits = readInt32(pos); + return Float.intBitsToFloat(bits); + } + + public String readString(int pos) { + return readString(pos, data.length - pos, true); + } + + public String readString(int pos, int maxSize, boolean errorOnMaxSizeHit) { + int iMax = Math.min(data.length, pos + maxSize); + for (int i = pos; i < iMax; i++) { + if (data[i] == 0) { + return new String(data, pos, i - pos, Globals.UTF8); + } + } + + if (errorOnMaxSizeHit) { + return null; + } + + return new String(data, pos, iMax - pos, Globals.UTF8); + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilSizeBuf.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilSizeBuf.java new file mode 100644 index 0000000..d8e52a7 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilSizeBuf.java @@ -0,0 +1,159 @@ +package org.rehlds.flightrec.api.util; + +public class UtilSizeBuf { + String name; + UtilByteBuffer buf; + int startPos; + int maxSize; + int curPos; + + public UtilSizeBuf(String name, UtilByteBuffer buf, int startPos, int maxSize) { + this.name = name; + this.buf = buf; + this.startPos = startPos; + this.maxSize = maxSize; + curPos = 0; + } + + public UtilSizeBuf(String name, UtilByteBuffer buf) { + this(name, buf, 0, buf.getDataLength()); + } + + public void init(int startPos, int maxSize) { + this.startPos = startPos; + this.maxSize = maxSize; + curPos = 0; + } + + public void reset() { + this.curPos = 0; + } + + public int tell() { + return curPos; + } + + public int getAbsoluteCurrentPos() { + return curPos + startPos; + } + + public void skip(int count) { + if (curPos + count > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + curPos += count; + } + + public int readUInt8() { + if (curPos + 1 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos++; + return buf.readUInt8(pos + this.startPos); + } + + public boolean readBool() { + if (curPos + 1 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos++; + return buf.readBool(pos + this.startPos); + } + + public int readUInt16() { + if (curPos + 2 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 2; + return buf.readUInt16(pos + this.startPos); + } + + public long readUInt32() { + if (curPos + 4 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 4; + return buf.readUInt32(pos + this.startPos); + } + + public int readInt32() { + if (curPos + 4 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 4; + return buf.readInt32(pos + this.startPos); + } + + public long readInt64() { + if (curPos + 8 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 8; + return buf.readInt64(pos + this.startPos); + } + + public double readDouble() { + if (curPos + 8 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 8; + return buf.readDouble(pos + this.startPos); + } + + public float readFloat() { + if (curPos + 4 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 4; + return buf.readFloat(pos + this.startPos); + } + + public String readString() { + String s = buf.readString(curPos + this.startPos); + if (s == null) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + curPos += s.getBytes(Globals.UTF8).length + 1; + return s; + } + + public int getStartPos() { + return startPos; + } + + public int getMaxSize() { + return maxSize; + } + + public byte[] getBuffer() { + return buf.getData(); + } +} diff --git a/flightrec/decoder_api/src/test/java/org/rehlds/flightrec/util/UtilByteBufferTest.java b/flightrec/decoder_api/src/test/java/org/rehlds/flightrec/util/UtilByteBufferTest.java new file mode 100644 index 0000000..22ed565 --- /dev/null +++ b/flightrec/decoder_api/src/test/java/org/rehlds/flightrec/util/UtilByteBufferTest.java @@ -0,0 +1,127 @@ +package org.rehlds.flightrec.util; + +import org.junit.Test; +import org.rehlds.flightrec.api.util.UtilByteBuffer; + +import static org.junit.Assert.*; + +public class UtilByteBufferTest { + + @Test + public void testReadUInt8() throws Exception { + byte data[] = { 0x10, 0x00, 0x7F, (byte)0x80, (byte)0xFF }; + UtilByteBuffer bb = new UtilByteBuffer(data); + + assertEquals(0x10, bb.readUInt8(0)); + assertEquals(0x00, bb.readUInt8(1)); + assertEquals(0x7F, bb.readUInt8(2)); + assertEquals(0x80, bb.readUInt8(3)); + assertEquals(0xFF, bb.readUInt8(4)); + } + + @Test + public void testReadUInt16() throws Exception { + byte data[] = { 0x10, 0x00, 0x7F, (byte)0x80, (byte)0xFF, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + + assertEquals(0x10, bb.readUInt16(0)); + assertEquals(0x7F00, bb.readUInt16(1)); + assertEquals(0x807F, bb.readUInt16(2)); + assertEquals(0xFF80, bb.readUInt16(3)); + assertEquals(0x00FF, bb.readUInt16(4)); + } + + @Test + public void testReadUInt32() throws Exception { + byte data[] = { 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, (byte)0xFF, 0x00, 0x00, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + assertEquals(0x10000000, bb.readUInt32(0)); + assertEquals(0x100000, bb.readUInt32(1)); + assertEquals(0x1000, bb.readUInt32(2)); + assertEquals(0x10, bb.readUInt32(3)); + assertEquals(0x0, bb.readUInt32(4)); + assertEquals(0x7F000000, bb.readUInt32(5)); + assertEquals(0x7F0000, bb.readUInt32(6)); + assertEquals(0x7F00, bb.readUInt32(7)); + assertEquals(0x7F, bb.readUInt32(8)); + assertEquals(0xFF000000L, bb.readUInt32(9)); + assertEquals(0xFF0000, bb.readUInt32(10)); + assertEquals(0xFF00, bb.readUInt32(11)); + assertEquals(0xFF, bb.readUInt32(12)); + } + + @Test + public void testReadInt32() throws Exception { + byte data[] = { 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, (byte)0xFF, 0x00, 0x00, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + assertEquals(0x10000000, bb.readInt32(0)); + assertEquals(0x100000, bb.readInt32(1)); + assertEquals(0x1000, bb.readInt32(2)); + assertEquals(0x10, bb.readInt32(3)); + + assertEquals(0x0, bb.readInt32(4)); + + assertEquals(0x7F000000, bb.readInt32(5)); + assertEquals(0x7F0000, bb.readInt32(6)); + assertEquals(0x7F00, bb.readInt32(7)); + assertEquals(0x7F, bb.readInt32(8)); + + assertEquals((int)0xFF000000, bb.readInt32(9)); + assertEquals(0xFF0000, bb.readInt32(10)); + assertEquals(0xFF00, bb.readInt32(11)); + assertEquals(0xFF, bb.readInt32(12)); + } + + @Test + public void testReadInt64() throws Exception { + byte data[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte)0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + assertEquals(0x1000000000000000L, bb.readInt64(0)); + assertEquals(0x10000000000000L, bb.readInt64(1)); + assertEquals(0x100000000000L, bb.readInt64(2)); + assertEquals(0x1000000000L, bb.readInt64(3)); + assertEquals(0x10000000L, bb.readInt64(4)); + assertEquals(0x100000L, bb.readInt64(5)); + assertEquals(0x1000L, bb.readInt64(6)); + assertEquals(0x10L, bb.readInt64(7)); + + assertEquals(0x00L, bb.readInt64(8)); + + assertEquals(0x7F00000000000000L, bb.readInt64(9)); + assertEquals(0x7F000000000000L, bb.readInt64(10)); + assertEquals(0x7F0000000000L, bb.readInt64(11)); + assertEquals(0x7F00000000L, bb.readInt64(12)); + assertEquals(0x7F000000L, bb.readInt64(13)); + assertEquals(0x7F0000L, bb.readInt64(14)); + assertEquals(0x7F00L, bb.readInt64(15)); + assertEquals(0x7FL, bb.readInt64(16)); + + assertEquals(0xFF00000000000000L, bb.readInt64(17)); + assertEquals(0xFF000000000000L, bb.readInt64(18)); + assertEquals(0xFF0000000000L, bb.readInt64(19)); + assertEquals(0xFF00000000L, bb.readInt64(20)); + assertEquals(0xFF000000L, bb.readInt64(21)); + assertEquals(0xFF0000L, bb.readInt64(22)); + assertEquals(0xFF00L, bb.readInt64(23)); + assertEquals(0xFFL, bb.readInt64(24)); + } + + @Test + public void testReadString() throws Exception { + byte data[] = { 0x00, 0x41, 0x00, 0x41, 0x42, 0x00, 0x50, 0x75, 0x62, 0x6C, 0x69, 0x63, 0x69, 0x74, (byte)0xC3, (byte)0xA9, 0x00, 0x41, 0x42 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + + assertEquals("", bb.readString(0)); + assertEquals("A", bb.readString(1)); + assertEquals("", bb.readString(2)); + assertEquals("AB", bb.readString(3)); + assertEquals("Publicité", bb.readString(6)); + assertNull(bb.readString(17)); + + assertEquals("Public", bb.readString(6, 6, false)); + assertNull(bb.readString(6, 6, true)); + + assertEquals("AB", bb.readString(17, 2, false)); + assertNull(bb.readString(17, 2, true)); + } +} \ No newline at end of file diff --git a/publish.gradle b/publish.gradle index 7b6fbd3..b40caa3 100644 --- a/publish.gradle +++ b/publish.gradle @@ -10,50 +10,69 @@ void _copyFile(String from, String to) { GradleCppUtils.copyFile(project.file(from), project.file(to), false) } -task publishPrepareFiles << { - def pubRootDir = project.file('publish/publishRoot') - if (pubRootDir.exists()) { - if (!pubRootDir.deleteDir()) { - throw new RuntimeException("Failed to delete ${pubRootDir}") +task publishPrepareFiles { + dependsOn ':flightrec/decoder:uberjar' + doLast { + def pubRootDir = project.file('publish/publishRoot') + if (pubRootDir.exists()) { + if (!pubRootDir.deleteDir()) { + throw new RuntimeException("Failed to delete ${pubRootDir}") + } } - } - pubRootDir.mkdirs() + pubRootDir.mkdirs() - //bugfixed binaries - project.file('publish/publishRoot/bin/bugfixed').mkdirs() - _copyFileToDir('publish/releaseRehldsFixes/swds.dll', 'publish/publishRoot/bin/bugfixed/') - _copyFileToDir('publish/releaseRehldsFixes/swds.pdb', 'publish/publishRoot/bin/bugfixed/') - _copyFile('publish/releaseRehldsFixes/libengine_i486.so', 'publish/publishRoot/bin/bugfixed/engine_i486.so') + //bugfixed binaries + project.file('publish/publishRoot/bin/bugfixed').mkdirs() + _copyFileToDir('publish/releaseRehldsFixes/swds.dll', 'publish/publishRoot/bin/bugfixed/') + _copyFileToDir('publish/releaseRehldsFixes/swds.pdb', 'publish/publishRoot/bin/bugfixed/') + _copyFile('publish/releaseRehldsFixes/libengine_i486.so', 'publish/publishRoot/bin/bugfixed/engine_i486.so') - //pure binaries - project.file('publish/publishRoot/bin/pure').mkdirs() - _copyFileToDir('publish/releaseRehldsNofixes/swds.dll', 'publish/publishRoot/bin/pure/') - _copyFileToDir('publish/releaseRehldsNofixes/swds.pdb', 'publish/publishRoot/bin/pure/') - _copyFile('publish/releaseRehldsNofixes/libengine_i486.so', 'publish/publishRoot/bin/pure/engine_i486.so') + //pure binaries + project.file('publish/publishRoot/bin/pure').mkdirs() + _copyFileToDir('publish/releaseRehldsNofixes/swds.dll', 'publish/publishRoot/bin/pure/') + _copyFileToDir('publish/releaseRehldsNofixes/swds.pdb', 'publish/publishRoot/bin/pure/') + _copyFile('publish/releaseRehldsNofixes/libengine_i486.so', 'publish/publishRoot/bin/pure/engine_i486.so') - //hlsdk - project.file('publish/publishRoot/hlsdk').mkdirs() - copy { - from 'rehlds/common' - into 'publish/publishRoot/hlsdk/common' - } - copy { - from 'rehlds/dlls' - into 'publish/publishRoot/hlsdk/dlls' - } - copy { - from 'rehlds/pm_shared' - into 'publish/publishRoot/hlsdk/pm_shared' - } - copy { - from 'rehlds/public' - into 'publish/publishRoot/hlsdk/public' - include 'interface.h', 'interface.cpp', 'FileSystem.h' - } - copy { - from 'rehlds/public/rehlds' - into 'publish/publishRoot/hlsdk/engine' + //hlsdk + project.file('publish/publishRoot/hlsdk').mkdirs() + copy { + from 'rehlds/common' + into 'publish/publishRoot/hlsdk/common' + } + copy { + from 'rehlds/dlls' + into 'publish/publishRoot/hlsdk/dlls' + } + copy { + from 'rehlds/pm_shared' + into 'publish/publishRoot/hlsdk/pm_shared' + } + copy { + from 'rehlds/public' + into 'publish/publishRoot/hlsdk/public' + include 'interface.h', 'interface.cpp', 'FileSystem.h' + } + copy { + from 'rehlds/public/rehlds' + into 'publish/publishRoot/hlsdk/engine' + } + + //flightrecorder + + def flightRecJarTask = project(':flightrec/decoder').tasks.getByName('uberjar') + println flightRecJarTask + println flightRecJarTask.class.name + File flightRecJarFile = flightRecJarTask.archivePath + project.file('publish/publishRoot/flighrec').mkdirs() + copy { + from flightRecJarFile + into 'publish/publishRoot/flighrec' + } + copy { + from new File(project(':flightrec/decoder').projectDir, 'pub') + into 'publish/publishRoot/flighrec' + } } } diff --git a/rehlds/build.gradle b/rehlds/build.gradle index ebb9af3..ad4c03c 100644 --- a/rehlds/build.gradle +++ b/rehlds/build.gradle @@ -122,7 +122,7 @@ void setupToolchain(NativeBinarySpec b) { } b.lib LazyNativeDepSet.create(dep_bzip2, 'bzip2', b.buildType.name, true) - cfg.singleDefines 'USE_BREAKPAD_HANDLER', 'DEDICATED', 'SWDS', 'REHLDS_SELF', 'REHLDS_OPT_PEDANTIC' + cfg.singleDefines 'USE_BREAKPAD_HANDLER', 'DEDICATED', 'SWDS', 'REHLDS_SELF', 'REHLDS_OPT_PEDANTIC', 'REHLDS_FLIGHT_REC' if (cfg instanceof MsvcToolchainConfig) { cfg.compilerOptions.pchConfig = new MsvcToolchainConfig.PrecompiledHeadersConfig( @@ -184,7 +184,7 @@ class RehldsSrc { include "**/*.cpp" exclude "precompiled.cpp" exclude GradleCppUtils.windows ? "tier0/platform_linux.cpp" : "tier0/platform_win32.cpp" - exclude "interface.cpp", "rehlds/crc32.cpp", "rehlds/sys_shared.cpp" + exclude "interface.cpp", "rehlds/crc32c.cpp", "rehlds/sys_shared.cpp" } source { diff --git a/rehlds/engine/ed_strpool.cpp b/rehlds/engine/ed_strpool.cpp index bbed2e3..e7787dc 100644 --- a/rehlds/engine/ed_strpool.cpp +++ b/rehlds/engine/ed_strpool.cpp @@ -4,7 +4,7 @@ class CStringPoolMap : public CStaticMap { protected: virtual uint32 hash(const char* const &val) { unsigned int len = strlen(val); - return crc32((const uint8*)val, len); + return crc32c((const uint8*)val, len); } virtual bool equals(const char* const &val1, const char* const &val2) { diff --git a/rehlds/engine/host.cpp b/rehlds/engine/host.cpp index 8114beb..30aa804 100644 --- a/rehlds/engine/host.cpp +++ b/rehlds/engine/host.cpp @@ -883,6 +883,8 @@ void Host_CheckConnectionFailure(void) /* <364fd> ../engine/host.c:1350 */ void _Host_Frame(float time) { + + static double host_times[6]; if (setjmp(host_enddemo)) return; @@ -891,7 +893,12 @@ void _Host_Frame(float time) if (!Host_FilterTime(time)) return; - FR_StartFrame(); +#ifdef REHLDS_FLIGHT_REC + static long frameCounter = 0; + if (rehlds_flrec_frame.string[0] != '0') { + FR_StartFrame(frameCounter); + } +#endif //REHLDS_FLIGHT_REC //SystemWrapper_RunFrame(host_frametime); @@ -951,7 +958,12 @@ void _Host_Frame(float time) Host_Quit_f(); } - FR_EndFrame(); +#ifdef REHLDS_FLIGHT_REC + if (rehlds_flrec_frame.string[0] != '0') { + FR_EndFrame(frameCounter); + } + frameCounter++; +#endif //REHLDS_FLIGHT_REC } /* <36628> ../engine/host.c:1501 */ @@ -1136,12 +1148,17 @@ int Host_Init(quakeparms_t *parms) Ed_StrPool_Init(); #endif //REHLDS_FIXES - FR_Init(); + FR_Init(); //don't put it under REHLDS_FLIGHT_REC to allow recording via Rehlds API Cbuf_Init(); Cmd_Init(); Cvar_Init(); Cvar_CmdInit(); + +#ifdef REHLDS_FLIGHT_REC + FR_Rehlds_Init(); +#endif //REHLDS_FLIGHT_REC + V_Init(); Chase_Init(); COM_Init(parms->basedir); diff --git a/rehlds/engine/pr_edict.cpp b/rehlds/engine/pr_edict.cpp index cef96af..c0976c5 100644 --- a/rehlds/engine/pr_edict.cpp +++ b/rehlds/engine/pr_edict.cpp @@ -452,7 +452,11 @@ void *PvAllocEntPrivateData(edict_t *pEdict, int32 cb) pEdict->pvPrivateData = Mem_Calloc(1, cb); - FR_AllocEntPrivateData(pEdict->pvPrivateData); +#ifdef REHLDS_FLIGHT_REC + if (rehlds_flrec_pvdata.string[0] != '0') { + FR_AllocEntPrivateData(pEdict->pvPrivateData, cb); + } +#endif //REHLDS_FLIGHT_REC return pEdict->pvPrivateData; } @@ -478,7 +482,11 @@ void FreeEntPrivateData(edict_t *pEdict) gNewDLLFunctions.pfnOnFreeEntPrivateData(pEdict); } - FR_FreeEntPrivateData(pEdict->pvPrivateData); +#ifdef REHLDS_FLIGHT_REC + if (rehlds_flrec_pvdata.string[0] != '0') { + FR_FreeEntPrivateData(pEdict->pvPrivateData); + } +#endif //REHLDS_FLIGHT_REC Mem_Free(pEdict->pvPrivateData); pEdict->pvPrivateData = 0; diff --git a/rehlds/engine/public_amalgamation.cpp b/rehlds/engine/public_amalgamation.cpp index cebff6f..5738457 100644 --- a/rehlds/engine/public_amalgamation.cpp +++ b/rehlds/engine/public_amalgamation.cpp @@ -1,5 +1,5 @@ #include "precompiled.h" #include "sys_shared.cpp" -#include "crc32.cpp" +#include "crc32c.cpp" #include "interface.cpp" diff --git a/rehlds/msvc/ReHLDS.vcxproj b/rehlds/msvc/ReHLDS.vcxproj index bb4d2e9..96d68b3 100644 --- a/rehlds/msvc/ReHLDS.vcxproj +++ b/rehlds/msvc/ReHLDS.vcxproj @@ -145,16 +145,14 @@ true - - true + true true - true true - true true - true true + true + true true @@ -230,6 +228,7 @@ true true + true true @@ -448,7 +447,7 @@ - + @@ -689,7 +688,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;REHLDS_FIXES;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;REHLDS_FIXES;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Precise /arch:IA32 %(AdditionalOptions) MultiThreadedDebug @@ -727,7 +726,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Precise /arch:IA32 %(AdditionalOptions) MultiThreadedDebug @@ -764,7 +763,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Precise /arch:IA32 %(AdditionalOptions) MultiThreadedDebug @@ -800,7 +799,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Precise /arch:IA32 %(AdditionalOptions) MultiThreadedDebug @@ -839,7 +838,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Precise /arch:IA32 %(AdditionalOptions) MultiThreadedDebug @@ -875,7 +874,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;_BUILD_FROM_IDE;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;_BUILD_FROM_IDE;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) MultiThreadedDebug Use precompiled.h @@ -911,7 +910,7 @@ Level3 Disabled true - REHLDS_OPT_PEDANTIC;REHLDS_FIXES;REHLDS_SELF;_BUILD_FROM_IDE;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_FIXES;REHLDS_SELF;_BUILD_FROM_IDE;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) MultiThreadedDebug Use precompiled.h @@ -948,7 +947,7 @@ true true true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;HOOK_ENGINE;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded /arch:IA32 %(AdditionalOptions) Use @@ -987,7 +986,7 @@ true true true - REHLDS_SELF;HOOK_ENGINE;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_SELF;HOOK_ENGINE;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded /arch:IA32 %(AdditionalOptions) Use @@ -1027,7 +1026,7 @@ true true true - REHLDS_OPT_PEDANTIC;REHLDS_SELF;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + REHLDS_FLIGHT_REC;REHLDS_OPT_PEDANTIC;REHLDS_SELF;REHLDS_CHECKS;USE_BREAKPAD_HANDLER;DEDICATED;SWDS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded /arch:IA32 %(AdditionalOptions) Use diff --git a/rehlds/msvc/ReHLDS.vcxproj.filters b/rehlds/msvc/ReHLDS.vcxproj.filters index 4713188..d263099 100644 --- a/rehlds/msvc/ReHLDS.vcxproj.filters +++ b/rehlds/msvc/ReHLDS.vcxproj.filters @@ -307,9 +307,6 @@ public\rehlds - - public\rehlds - engine\common @@ -331,6 +328,12 @@ unittests + + public\rehlds + + + unittests + @@ -990,9 +993,6 @@ public\rehlds - - public\rehlds - public\rehlds @@ -1041,6 +1041,9 @@ engine + + public\rehlds + diff --git a/rehlds/public/rehlds/crc32.cpp b/rehlds/public/rehlds/crc32.cpp deleted file mode 100644 index 872a9f4..0000000 --- a/rehlds/public/rehlds/crc32.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/* The implementation here was originally done by Gary S. Brown. I have - borrowed the tables directly, and made some minor changes to the - crc32-function (including changing the interface). //ylo */ - - /* ============================================================= */ - /* COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or */ - /* code or tables extracted from it, as desired without restriction. */ - /* */ - /* First, the polynomial itself and its table of feedback terms. The */ - /* polynomial is */ - /* X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 */ - /* */ - /* Note that we take it "backwards" and put the highest-order term in */ - /* the lowest-order bit. The X^32 term is "implied"; the LSB is the */ - /* X^31 term, etc. The X^0 term (usually shown as "+1") results in */ - /* the MSB being 1. */ - /* */ - /* Note that the usual hardware shift register implementation, which */ - /* is what we're using (we're merely optimizing it by doing eight-bit */ - /* chunks at a time) shifts bits into the lowest-order term. In our */ - /* implementation, that means shifting towards the right. Why do we */ - /* do it this way? Because the calculated CRC must be transmitted in */ - /* order from highest-order term to lowest-order term. UARTs transmit */ - /* characters in order from LSB to MSB. By storing the CRC this way, */ - /* we hand it to the UART in the order low-byte to high-byte; the UART */ - /* sends each low-bit to hight-bit; and the result is transmission bit */ - /* by bit from highest- to lowest-order term without requiring any bit */ - /* shuffling on our part. Reception works similarly. */ - /* */ - /* The feedback terms table consists of 256, 32-bit entries. Notes: */ - /* */ - /* The table can be generated at runtime if desired; code to do so */ - /* is shown later. It might not be obvious, but the feedback */ - /* terms simply represent the results of eight shift/xor opera- */ - /* tions for all combinations of data and CRC register values. */ - /* */ - /* The values must be right-shifted by eight bits by the "updcrc" */ - /* logic; the shift must be unsigned (bring in zeroes). On some */ - /* hardware you could probably optimize the shift in assembler by */ - /* using byte-swap instructions. */ - /* polynomial $edb88320 */ - /* */ - /* -------------------------------------------------------------------- */ -#include "crc32.h" -#include "sys_shared.h" - -static uint32 crc32_tab[] = { - - 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, - 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, - 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, - 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, - 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, - 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, - 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, - 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, - 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, - 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, - 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, - 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, - 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, - 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, - 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, - 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, - 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, - 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, - 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, - 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, - 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, - 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, - 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, - 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, - 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, - 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, - 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, - 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, - 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, - 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, - 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, - 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, - 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, - 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, - 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, - 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, - 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, - 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, - 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, - 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, - 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, - 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, - 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, - 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, - 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, - 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, - 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, - 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, - 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, - 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, - 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, - 0x2d02ef8dL - }; - - -uint32 crc32_t_nosse(uint32 iCRC, const uint8 *s, unsigned int len) { - uint32 crc32val = iCRC; - for (unsigned int i = 0; i < len; i++) { - crc32val = crc32_tab[(crc32val ^ s[i]) & 0xff] ^ (crc32val >> 8); - } - return crc32val; -} - -uint32 crc32_t(uint32 iCRC, const uint8 *s, unsigned int len) { - if (!cpuinfo.sse4_2) { - return crc32_t_nosse(iCRC, s, len); - } - - uint32 crc32val = iCRC; - unsigned int i = 0; - - for (; i < (len >> 2); i += 4) { - crc32val = _mm_crc32_u32(crc32val, *(uint32*)&s[i]); - } - - for (; i < len; i++) { - crc32val = _mm_crc32_u8(crc32val, s[i]); - } - - return crc32val; -} - -uint32 crc32(const uint8 *buf, unsigned int len) { - return crc32_t(0, buf, len); -} - -uint32 crc32_t8_sse(uint32 iCRC, uint8 data) { - return _mm_crc32_u8(iCRC, data); -} - -uint32 crc32_t8_nosse(uint32 iCRC, uint8 data) { - return crc32_tab[(iCRC ^ data) & 0xff] ^ (iCRC >> 8); -} diff --git a/rehlds/public/rehlds/crc32.h b/rehlds/public/rehlds/crc32.h deleted file mode 100644 index 6ca4458..0000000 --- a/rehlds/public/rehlds/crc32.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * $Id: crc32.h,v 1.1.1.1 1996/02/18 21:38:11 ylo Exp $ - * $Log: crc32.h,v $ - * Revision 1.1.1.1 1996/02/18 21:38:11 ylo - * Imported ssh-1.2.13. - * - * Revision 1.2 1995/07/13 01:21:45 ylo - * Removed "Last modified" header. - * Added cvs log. - * - * $Endlog$ - */ - -#pragma once -#include "archtypes.h" -/* This computes a 32 bit CRC of the data in the buffer, and returns the - CRC. The polynomial used is 0xedb88320. */ - -uint32 crc32(const uint8 *buf, unsigned int len); -uint32 crc32_t(uint32 iCRC, const uint8 *s, unsigned int len); -uint32 crc32_t8_sse(uint32 iCRC, uint8 data); -uint32 crc32_t8_nosse(uint32 iCRC, uint8 data); - diff --git a/rehlds/public/rehlds/crc32c.cpp b/rehlds/public/rehlds/crc32c.cpp new file mode 100644 index 0000000..6dd7758 --- /dev/null +++ b/rehlds/public/rehlds/crc32c.cpp @@ -0,0 +1,143 @@ +/* +Copyright (C) 2010 by Ronnie Sahlberg +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License +along with this program; if not, see . +*/ + +#include "crc32c.h" +#include "sys_shared.h" +#include "immintrin.h" + +/*****************************************************************/ +/* */ +/* CRC LOOKUP TABLE */ +/* ================ */ +/* The following CRC lookup table was generated automagically */ +/* by the Rocksoft^tm Model CRC Algorithm Table Generation */ +/* Program V1.0 using the following model parameters: */ +/* */ +/* Width : 4 bytes. */ +/* Poly : 0x1EDC6F41L */ +/* Reverse : TRUE. */ +/* */ +/* For more information on the Rocksoft^tm Model CRC Algorithm, */ +/* see the document titled "A Painless Guide to CRC Error */ +/* Detection Algorithms" by Ross Williams */ +/* (ross@guest.adelaide.edu.au.). This document is likely to be */ +/* in the FTP archive "ftp.adelaide.edu.au/pub/rocksoft". */ +/* */ +/*****************************************************************/ + +static uint32 crctable[256] = { + 0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L, + 0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL, + 0x8AD958CFL, 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL, + 0x4D43CFD0L, 0xBF284CD3L, 0xAC78BF27L, 0x5E133C24L, + 0x105EC76FL, 0xE235446CL, 0xF165B798L, 0x030E349BL, + 0xD7C45070L, 0x25AFD373L, 0x36FF2087L, 0xC494A384L, + 0x9A879FA0L, 0x68EC1CA3L, 0x7BBCEF57L, 0x89D76C54L, + 0x5D1D08BFL, 0xAF768BBCL, 0xBC267848L, 0x4E4DFB4BL, + 0x20BD8EDEL, 0xD2D60DDDL, 0xC186FE29L, 0x33ED7D2AL, + 0xE72719C1L, 0x154C9AC2L, 0x061C6936L, 0xF477EA35L, + 0xAA64D611L, 0x580F5512L, 0x4B5FA6E6L, 0xB93425E5L, + 0x6DFE410EL, 0x9F95C20DL, 0x8CC531F9L, 0x7EAEB2FAL, + 0x30E349B1L, 0xC288CAB2L, 0xD1D83946L, 0x23B3BA45L, + 0xF779DEAEL, 0x05125DADL, 0x1642AE59L, 0xE4292D5AL, + 0xBA3A117EL, 0x4851927DL, 0x5B016189L, 0xA96AE28AL, + 0x7DA08661L, 0x8FCB0562L, 0x9C9BF696L, 0x6EF07595L, + 0x417B1DBCL, 0xB3109EBFL, 0xA0406D4BL, 0x522BEE48L, + 0x86E18AA3L, 0x748A09A0L, 0x67DAFA54L, 0x95B17957L, + 0xCBA24573L, 0x39C9C670L, 0x2A993584L, 0xD8F2B687L, + 0x0C38D26CL, 0xFE53516FL, 0xED03A29BL, 0x1F682198L, + 0x5125DAD3L, 0xA34E59D0L, 0xB01EAA24L, 0x42752927L, + 0x96BF4DCCL, 0x64D4CECFL, 0x77843D3BL, 0x85EFBE38L, + 0xDBFC821CL, 0x2997011FL, 0x3AC7F2EBL, 0xC8AC71E8L, + 0x1C661503L, 0xEE0D9600L, 0xFD5D65F4L, 0x0F36E6F7L, + 0x61C69362L, 0x93AD1061L, 0x80FDE395L, 0x72966096L, + 0xA65C047DL, 0x5437877EL, 0x4767748AL, 0xB50CF789L, + 0xEB1FCBADL, 0x197448AEL, 0x0A24BB5AL, 0xF84F3859L, + 0x2C855CB2L, 0xDEEEDFB1L, 0xCDBE2C45L, 0x3FD5AF46L, + 0x7198540DL, 0x83F3D70EL, 0x90A324FAL, 0x62C8A7F9L, + 0xB602C312L, 0x44694011L, 0x5739B3E5L, 0xA55230E6L, + 0xFB410CC2L, 0x092A8FC1L, 0x1A7A7C35L, 0xE811FF36L, + 0x3CDB9BDDL, 0xCEB018DEL, 0xDDE0EB2AL, 0x2F8B6829L, + 0x82F63B78L, 0x709DB87BL, 0x63CD4B8FL, 0x91A6C88CL, + 0x456CAC67L, 0xB7072F64L, 0xA457DC90L, 0x563C5F93L, + 0x082F63B7L, 0xFA44E0B4L, 0xE9141340L, 0x1B7F9043L, + 0xCFB5F4A8L, 0x3DDE77ABL, 0x2E8E845FL, 0xDCE5075CL, + 0x92A8FC17L, 0x60C37F14L, 0x73938CE0L, 0x81F80FE3L, + 0x55326B08L, 0xA759E80BL, 0xB4091BFFL, 0x466298FCL, + 0x1871A4D8L, 0xEA1A27DBL, 0xF94AD42FL, 0x0B21572CL, + 0xDFEB33C7L, 0x2D80B0C4L, 0x3ED04330L, 0xCCBBC033L, + 0xA24BB5A6L, 0x502036A5L, 0x4370C551L, 0xB11B4652L, + 0x65D122B9L, 0x97BAA1BAL, 0x84EA524EL, 0x7681D14DL, + 0x2892ED69L, 0xDAF96E6AL, 0xC9A99D9EL, 0x3BC21E9DL, + 0xEF087A76L, 0x1D63F975L, 0x0E330A81L, 0xFC588982L, + 0xB21572C9L, 0x407EF1CAL, 0x532E023EL, 0xA145813DL, + 0x758FE5D6L, 0x87E466D5L, 0x94B49521L, 0x66DF1622L, + 0x38CC2A06L, 0xCAA7A905L, 0xD9F75AF1L, 0x2B9CD9F2L, + 0xFF56BD19L, 0x0D3D3E1AL, 0x1E6DCDEEL, 0xEC064EEDL, + 0xC38D26C4L, 0x31E6A5C7L, 0x22B65633L, 0xD0DDD530L, + 0x0417B1DBL, 0xF67C32D8L, 0xE52CC12CL, 0x1747422FL, + 0x49547E0BL, 0xBB3FFD08L, 0xA86F0EFCL, 0x5A048DFFL, + 0x8ECEE914L, 0x7CA56A17L, 0x6FF599E3L, 0x9D9E1AE0L, + 0xD3D3E1ABL, 0x21B862A8L, 0x32E8915CL, 0xC083125FL, + 0x144976B4L, 0xE622F5B7L, 0xF5720643L, 0x07198540L, + 0x590AB964L, 0xAB613A67L, 0xB831C993L, 0x4A5A4A90L, + 0x9E902E7BL, 0x6CFBAD78L, 0x7FAB5E8CL, 0x8DC0DD8FL, + 0xE330A81AL, 0x115B2B19L, 0x020BD8EDL, 0xF0605BEEL, + 0x24AA3F05L, 0xD6C1BC06L, 0xC5914FF2L, 0x37FACCF1L, + 0x69E9F0D5L, 0x9B8273D6L, 0x88D28022L, 0x7AB90321L, + 0xAE7367CAL, 0x5C18E4C9L, 0x4F48173DL, 0xBD23943EL, + 0xF36E6F75L, 0x0105EC76L, 0x12551F82L, 0xE03E9C81L, + 0x34F4F86AL, 0xC69F7B69L, 0xD5CF889DL, 0x27A40B9EL, + 0x79B737BAL, 0x8BDCB4B9L, 0x988C474DL, 0x6AE7C44EL, + 0xBE2DA0A5L, 0x4C4623A6L, 0x5F16D052L, 0xAD7D5351L +}; + +uint32 crc32c_t8_nosse(uint32 iCRC, uint8 u8) { + return (iCRC >> 8) ^ crctable[(iCRC ^ u8) & 0xFF]; +} + +uint32 crc32c_t_nosse(uint32 iCRC, const uint8 *buf, int len) { + uint32 crc = iCRC; + while (len-- > 0) { + crc = (crc >> 8) ^ crctable[(crc ^ (*buf++)) & 0xFF]; + } + return crc; +} + +uint32 crc32c_t8_sse(uint32 iCRC, uint8 u8) { + return _mm_crc32_u8(iCRC, u8); +} + +uint32 crc32c_t_sse(uint32 iCRC, const uint8 *buf, unsigned int len) { + uint32 crc32cval = iCRC; + unsigned int i = 0; + + for (; i < (len >> 2); i += 4) { + crc32cval = _mm_crc32_u32(crc32cval, *(uint32*)&buf[i]); + } + + for (; i < len; i++) { + crc32cval = _mm_crc32_u8(crc32cval, buf[i]); + } + + return crc32cval; +} + +uint32 crc32c_t(uint32 iCRC, const uint8 *buf, unsigned int len) { + return cpuinfo.sse4_2 ? crc32c_t_nosse(iCRC, buf, len) : crc32c_t_nosse(iCRC, buf, len); +} + +uint32 crc32c(const uint8 *buf, int len) { + return crc32c_t(0xffffffff, buf, len); +} diff --git a/rehlds/public/rehlds/crc32c.h b/rehlds/public/rehlds/crc32c.h new file mode 100644 index 0000000..d3a78a3 --- /dev/null +++ b/rehlds/public/rehlds/crc32c.h @@ -0,0 +1,22 @@ +/* +Copyright (C) 2010 by Ronnie Sahlberg +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License +along with this program; if not, see . +*/ +#pragma once +#include "archtypes.h" + +extern uint32 crc32c_t8_nosse(uint32 iCRC, uint8 u8); +extern uint32 crc32c_t8_sse(uint32 iCRC, uint8 u8); +extern uint32 crc32c_t_nosse(uint32 iCRC, const uint8 *buf, int len); +extern uint32 crc32c_t_sse(uint32 iCRC, const uint8 *buf, unsigned int len); +extern uint32 crc32c_t(uint32 iCRC, const uint8 *buf, unsigned int len); +extern uint32 crc32c(const uint8 *buf, int len); diff --git a/rehlds/public/rehlds/static_map.h b/rehlds/public/rehlds/static_map.h index 010ffc9..52fc7b2 100644 --- a/rehlds/public/rehlds/static_map.h +++ b/rehlds/public/rehlds/static_map.h @@ -1,13 +1,13 @@ #pragma once #include "archtypes.h" -#include "crc32.h" +#include "crc32c.h" template class CStaticMap { protected: virtual uint32 hash(const T_KEY& val) { - return crc32((const unsigned char*)&val, sizeof(T_KEY)); + return crc32c((const unsigned char*)&val, sizeof(T_KEY)); } virtual bool equals(const T_KEY& val1, const T_KEY& val2) { @@ -207,7 +207,7 @@ template class CStringKeyStaticMap : public CStaticMap { protected: virtual uint32 hash(const char* const &val) { - return crc32((const unsigned char*)val, strlen(val)); + return crc32c((const unsigned char*)val, strlen(val)); } virtual bool equals(const char* const &val1, const char* const &val2) { @@ -232,7 +232,7 @@ protected: if (cc >= 'A' || cc <= 'Z') { cc |= 0x20; } - cksum = crc32_t8_sse(cksum, cc); + cksum = crc32c_t8_sse(cksum, cc); } } else { while (*pcc) { @@ -240,7 +240,7 @@ protected: if (cc >= 'A' || cc <= 'Z') { cc |= 0x20; } - cksum = crc32_t8_nosse(cksum, cc); + cksum = crc32c_t8_nosse(cksum, cc); } } return cksum; diff --git a/rehlds/rehlds/FlightRecorderImpl.cpp b/rehlds/rehlds/FlightRecorderImpl.cpp index d8bdc3f..2c8cca4 100644 --- a/rehlds/rehlds/FlightRecorderImpl.cpp +++ b/rehlds/rehlds/FlightRecorderImpl.cpp @@ -18,15 +18,15 @@ #include "precompiled.h" CRehldsFlightRecorder::CRehldsFlightRecorder() { - m_MetaRegionPtr = (uint8*) sys_allocmem(META_REGION_SIZE); - m_DataRegionPtr = (uint8*) sys_allocmem(DATA_REGION_SIZE); + m_MetaRegion = (uint8*) sys_allocmem(META_REGION_SIZE); + m_DataRegion = (uint8*) sys_allocmem(DATA_REGION_SIZE); - if (!m_MetaRegionPtr || !m_DataRegionPtr) { + if (!m_MetaRegion || !m_DataRegion) { rehlds_syserror("%s: direct allocation failed", __FUNCTION__); } //initialize meta region header - char* metaPos = (char*)m_MetaRegionPtr; + char* metaPos = (char*)m_MetaRegion; const char* metaSignature = "REHLDS_FLIGHTREC_META"; metaPos += sprintf(metaPos, "%s%s%s:", metaSignature, metaSignature, metaSignature); @@ -35,12 +35,12 @@ CRehldsFlightRecorder::CRehldsFlightRecorder() { m_pRecorderState = (recorder_state*)metaPos; metaPos += sizeof(recorder_state); - if ((metaPos - (char*)m_MetaRegionPtr) > META_REGION_HEADER) { + if ((metaPos - (char*)m_MetaRegion) > META_REGION_HEADER) { rehlds_syserror("%s: Meta header overflow", __FUNCTION__); } //initialize data region header - char* dataPos = (char*)m_DataRegionPtr; + char* dataPos = (char*)m_DataRegion; const char* dataSignature = "REHLDS_FLIGHTREC_DATA"; dataPos += sprintf(dataPos, "%s%s%s:", dataSignature, dataSignature, dataSignature); @@ -53,21 +53,25 @@ CRehldsFlightRecorder::CRehldsFlightRecorder() { InitHeadersContent(); - m_MetaRegionPtr += META_REGION_HEADER; - m_DataRegionPtr += DATA_REGION_HEADER; + m_MetaRegionPtr = m_MetaRegion + META_REGION_HEADER; + m_DataRegionPtr = m_DataRegion + DATA_REGION_HEADER; } void CRehldsFlightRecorder::InitHeadersContent() { m_pMetaHeader->version = FLIGHT_RECORDER_VERSION; + m_pMetaHeader->regionSize = META_REGION_SIZE; m_pMetaHeader->metaRegionPos = 0; m_pMetaHeader->numMessages = 0; + m_pMetaHeader->headerCrc32 = crc32c_t(0, m_MetaRegion, ((uint8*)m_pMetaHeader - m_MetaRegion) + offsetof(meta_header, headerCrc32)); m_pRecorderState->wpos = 0; m_pRecorderState->lastMsgBeginPos = 0xFFFFFFFF; m_pRecorderState->curMessage = 0; m_pDataHeader->version = FLIGHT_RECORDER_VERSION; + m_pDataHeader->regionSize = DATA_REGION_SIZE; m_pDataHeader->prevItrLastPos = 0xFFFFFFFF; + m_pDataHeader->headerCrc32 = crc32c_t(0, m_DataRegion, ((uint8*)m_pDataHeader - m_DataRegion) + offsetof(data_header, headerCrc32)); } void CRehldsFlightRecorder::MoveToStart() { @@ -100,8 +104,6 @@ void CRehldsFlightRecorder::StartMessage(uint16 msg, bool entrance) { MoveToStart(); } - - m_pRecorderState->curMessage = msg; m_pRecorderState->lastMsgBeginPos = m_pRecorderState->wpos; *(uint16*)(m_DataRegionPtr + m_pRecorderState->wpos) = msg; @@ -208,10 +210,11 @@ uint16 CRehldsFlightRecorder::RegisterMessage(const char* module, const char *me sizebuf_t sb; sb.buffername = "FlightRecorded Meta"; sb.cursize = m_pMetaHeader->metaRegionPos; - sb.maxsize = META_REGION_MAIN_SIZE; + sb.maxsize = META_REGION_MAIN_SIZE - META_REGION_HEADER; sb.flags = 0; sb.data = m_MetaRegionPtr; + MSG_WriteByte(&sb, MRT_MESSAGE_DEF); MSG_WriteShort(&sb, msgId); MSG_WriteString(&sb, module); MSG_WriteString(&sb, message); @@ -222,3 +225,17 @@ uint16 CRehldsFlightRecorder::RegisterMessage(const char* module, const char *me return msgId; } + +void CRehldsFlightRecorder::dump(const char* fname) { + FILE* fl = fopen(fname, "wb"); + if (fl == NULL) { + Con_Printf("Failed to write Flight Recorder dump: could not open '%s' for writing\n", fname); + return; + } + + fwrite(m_MetaRegion, META_REGION_SIZE, 1, fl); + fwrite(m_DataRegion, DATA_REGION_SIZE, 1, fl); + fclose(fl); + + Con_Printf("Written flightrecorder dump to '%s'\n", fname); +} diff --git a/rehlds/rehlds/FlightRecorderImpl.h b/rehlds/rehlds/FlightRecorderImpl.h index ec9aea4..b503030 100644 --- a/rehlds/rehlds/FlightRecorderImpl.h +++ b/rehlds/rehlds/FlightRecorderImpl.h @@ -36,6 +36,11 @@ public: private: + enum meta_record_type_t { + MRT_MESSAGE_DEF = 1, + MRT_SHUNT = 2, + }; + #pragma pack(push, 1) struct recorder_state { unsigned int wpos; @@ -45,16 +50,23 @@ private: struct meta_header { unsigned int version; + unsigned int regionSize; + unsigned int headerCrc32; unsigned int numMessages; unsigned int metaRegionPos; }; struct data_header { unsigned int version; + unsigned int regionSize; + unsigned int headerCrc32; unsigned int prevItrLastPos; }; #pragma pack(pop) + uint8* m_MetaRegion; + uint8* m_DataRegion; + uint8* m_MetaRegionPtr; uint8* m_DataRegionPtr; meta_header* m_pMetaHeader; @@ -84,6 +96,7 @@ private: public: CRehldsFlightRecorder(); + void dump(const char* fname); virtual void StartMessage(uint16 msg, bool entrance); virtual void EndMessage(uint16 msg, bool entrance); diff --git a/rehlds/rehlds/flight_recorder.cpp b/rehlds/rehlds/flight_recorder.cpp index ceb936e..4a12fff 100644 --- a/rehlds/rehlds/flight_recorder.cpp +++ b/rehlds/rehlds/flight_recorder.cpp @@ -19,10 +19,18 @@ CRehldsFlightRecorder* g_FlightRecorder; +void FR_Init() { + g_FlightRecorder = new CRehldsFlightRecorder(); +} + +#ifdef REHLDS_FLIGHT_REC + uint16 g_FRMsg_Frame; uint16 g_FRMsg_FreeEntPrivateData; uint16 g_FRMsg_AllocEntPrivateData; +cvar_t rehlds_flrec_frame = { "rehlds_flrec_frame", "1", 0, 1.0f, NULL }; +cvar_t rehlds_flrec_pvdata = { "rehlds_flrec_privdata", "1", 0, 1.0f, NULL }; void FR_CheckInit() { #ifdef HOOK_ENGINE if (!g_FlightRecorder) @@ -30,31 +38,47 @@ void FR_CheckInit() { #endif } -void FR_Init() { - g_FlightRecorder = new CRehldsFlightRecorder(); +void FR_Dump_f() { + const char* fname = "rehlds_flightrec.bin"; + if (Cmd_Argc() == 1) { + fname = Cmd_Argv(0); + } else if (Cmd_Argc() > 1) { + Con_Printf("usage: rehlds_flrec_dump < filename >\n"); + } - g_FRMsg_Frame = g_FlightRecorder->RegisterMessage("rehlds", "Frame", 1, true); - g_FRMsg_FreeEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "FreeEntPrivateData", 1, false); - g_FRMsg_AllocEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "AllocEntPrivateData", 1, false); + g_FlightRecorder->dump(fname); } -void FR_StartFrame() { +void FR_Rehlds_Init() { + Cvar_RegisterVariable(&rehlds_flrec_frame); + Cvar_RegisterVariable(&rehlds_flrec_pvdata); + Cmd_AddCommand("rehlds_flrec_dump", &FR_Dump_f); + + g_FRMsg_Frame = g_FlightRecorder->RegisterMessage("rehlds", "Frame", 2, true); + g_FRMsg_FreeEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "FreeEntPrivateData", 1, false); + g_FRMsg_AllocEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "AllocEntPrivateData", 2, false); +} + +void FR_StartFrame(long frameCounter) { FR_CheckInit(); g_FlightRecorder->StartMessage(g_FRMsg_Frame, true); + g_FlightRecorder->WriteInt64(frameCounter); g_FlightRecorder->WriteDouble(realtime); g_FlightRecorder->EndMessage(g_FRMsg_Frame, true); } -void FR_EndFrame() { +void FR_EndFrame(long frameCounter) { FR_CheckInit(); g_FlightRecorder->StartMessage(g_FRMsg_Frame, false); + g_FlightRecorder->WriteInt64(frameCounter); g_FlightRecorder->EndMessage(g_FRMsg_Frame, false); } -void FR_AllocEntPrivateData(void* res) { +void FR_AllocEntPrivateData(void* res, int size) { FR_CheckInit(); g_FlightRecorder->StartMessage(g_FRMsg_AllocEntPrivateData, true); g_FlightRecorder->WriteUInt32((size_t)res); + g_FlightRecorder->WriteInt32(size); g_FlightRecorder->EndMessage(g_FRMsg_AllocEntPrivateData, true); } @@ -64,3 +88,5 @@ void FR_FreeEntPrivateData(void* data) { g_FlightRecorder->WriteUInt32((size_t)data); g_FlightRecorder->EndMessage(g_FRMsg_FreeEntPrivateData, true); } + +#endif //REHLDS_FLIGHT_REC diff --git a/rehlds/rehlds/flight_recorder.h b/rehlds/rehlds/flight_recorder.h index 5ee2c12..9b80f43 100644 --- a/rehlds/rehlds/flight_recorder.h +++ b/rehlds/rehlds/flight_recorder.h @@ -21,13 +21,24 @@ extern CRehldsFlightRecorder* g_FlightRecorder; +extern void FR_Init(); + +#ifdef REHLDS_FLIGHT_REC + +extern cvar_t rehlds_flrec_frame; +extern cvar_t rehlds_flrec_pvdata; + extern uint16 g_FRMsg_Frame; extern uint16 g_FRMsg_FreeEntPrivateData; extern uint16 g_FRMsg_AllocEntPrivateData; -extern void FR_Init(); +extern void FR_Dump_f(); +extern void FR_Rehlds_Init(); -extern void FR_StartFrame(); -extern void FR_EndFrame(); + +extern void FR_StartFrame(long frameCounter); +extern void FR_EndFrame(long frameCounter); extern void FR_FreeEntPrivateData(void* data); -extern void FR_AllocEntPrivateData(void* res); +extern void FR_AllocEntPrivateData(void* res, int size); + +#endif //REHLDS_FLIGHT_REC diff --git a/rehlds/rehlds/precompiled.h b/rehlds/rehlds/precompiled.h index 0ac70bb..0bc1c3b 100644 --- a/rehlds/rehlds/precompiled.h +++ b/rehlds/rehlds/precompiled.h @@ -8,7 +8,7 @@ #include "mathlib.h" #include "sys_shared.h" -#include "crc32.h" +#include "crc32c.h" #include "static_map.h" #include "ed_strpool.h" diff --git a/rehlds/unittests/crc32c_tests.cpp b/rehlds/unittests/crc32c_tests.cpp new file mode 100644 index 0000000..5f5f575 --- /dev/null +++ b/rehlds/unittests/crc32c_tests.cpp @@ -0,0 +1,40 @@ +#include "precompiled.h" +#include "cppunitlite/TestHarness.h" + +TEST(CRC32C_Hash, CRC32C, 1000) { + Sys_CheckCpuInstructionsSupport(); + CHECK("SSE4.2 Support", cpuinfo.sse4_2); + + struct testdata_t { + const char* src; + uint32 hash0; + uint32 hashFF; + }; + + testdata_t testdata[] = { + { "a", 0x93AD1061, 0x3E2FBCCF }, + { "ab", 0x13C35EE4, 0x1D5DD6C9 }, + { "abc", 0x562F9CCD, 0xC9B4C048 }, + { "abcd", 0xDAAF41F6, 0x6D37F5CE }, + { "abcde", 0x8122A0A2, 0x3BAF2968 }, + { "abcdef", 0x0496937B, 0xAC43100E }, + { "abcdefg", 0x5D199E2C, 0x19D80BBE }, + { "abcdefgh", 0x86BC933D, 0xF56BDE48 }, + { "abcdefghi", 0x9639F15F, 0xD2236603 }, + { "abcdefghij", 0x0584645C, 0x19A66BC8 }, + }; + + for (int i = 0; i < ARRAYSIZE(testdata); i++) { + uint32 pureCksum0 = crc32c_t_nosse(0, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + uint32 sseCksum0 = crc32c_t_sse(0, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + + uint32 pureCksumFF = crc32c_t_nosse(-1, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + uint32 sseCksumFF = crc32c_t_sse(-1, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + + UINT32_EQUALS("Pure crc32c checksum-0 mismatch", testdata[i].hash0, pureCksum0); + UINT32_EQUALS("SSE crc32c checksum-0 mismatch", testdata[i].hash0, sseCksum0); + + UINT32_EQUALS("Pure crc32c checksum-FF mismatch", testdata[i].hashFF, pureCksumFF); + UINT32_EQUALS("SSE crc32c checksum-FF mismatch", testdata[i].hashFF, sseCksumFF); + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 9e17208..0da4d36 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,3 +2,4 @@ rootProject.name = 'rehlds' include 'dep/cppunitlite' include 'dep/bzip2' include 'rehlds' +include 'flightrec/decoder_api', 'flightrec/decoder'