diff --git a/flightrec/decoder/build.gradle b/flightrec/decoder/build.gradle index 957f4ae..cdfccc4 100644 --- a/flightrec/decoder/build.gradle +++ b/flightrec/decoder/build.gradle @@ -1,5 +1,7 @@ apply plugin: 'java' +group = 'org.rehlds.flightrec' + sourceCompatibility = '1.7' targetCompatibility = '1.7' @@ -9,4 +11,22 @@ repositories { dependencies { testCompile "junit:junit:4.12" + compile project(':flightrec/decoder_api') +} + +task uberjar(type: Jar, dependsOn: ['check', ':flightrec/decoder_api:build']) { + from files(sourceSets.main.output.classesDir) + from { configurations.runtime.collect { it.isDirectory() ? it : zipTree(it) } } + exclude('META-INF/*.DSA', 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.LIST') //exclude all signing stuff + + manifest { + attributes 'Main-Class': 'org.rehlds.flightrec.main.FlightRecorder' + attributes 'Implementation-Vendor': 'Sun Microsystems, Inc' + attributes 'Implementation-Title': 'Java Runtime Environment' + attributes 'Implementation-Version': '1.7.0' + } +} + +tasks.withType(AbstractCompile) { + options.encoding = 'UTF-8' } diff --git a/flightrec/decoder/pub/decoder.bat b/flightrec/decoder/pub/decoder.bat new file mode 100644 index 0000000..1026be9 --- /dev/null +++ b/flightrec/decoder/pub/decoder.bat @@ -0,0 +1,50 @@ +@if "%DEBUG%" == "" @echo off +setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +set CMD_LINE_ARGS=%* +"%JAVA_EXE%" -jar "%DIRNAME%/decoder.jar" %CMD_LINE_ARGS% + +:end +goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +endlocal diff --git a/flightrec/decoder/src/main/java/com/google/cloud/Crc32c.java b/flightrec/decoder/src/main/java/com/google/cloud/Crc32c.java new file mode 100644 index 0000000..28f898a --- /dev/null +++ b/flightrec/decoder/src/main/java/com/google/cloud/Crc32c.java @@ -0,0 +1,156 @@ +// Copyright 2011 Google Inc. All rights reserved. + +package com.google.cloud; + +import java.util.zip.Checksum; + +/** + * This class generates a CRC32C checksum, defined by rfc3720 section B.4. + * + * + */ +public final class Crc32c implements Checksum { + + private static final long[] CRC_TABLE = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 + }; + + private static final long LONG_MASK = 0xffffffffL; + private static final long BYTE_MASK = 0xff; + + private long crc; + + public Crc32c() { + crc = 0; + } + + /** + * Updates the checksum with a new byte. + * @param b the new byte. + */ + @Override + public void update(int b) { + long newCrc = crc; + newCrc = updateByte((byte) b, newCrc); + crc = newCrc; + } + + /** + * Updates the checksum with an array of bytes. + * @param bArray the array of bytes. + * @param off the offset into the array where the update should begin. + * @param len the length of data to examine. + */ + @Override + public void update(byte[] bArray, int off, int len) { + long newCrc = crc; + for (int i = off; i < off + len; i++) { + newCrc = updateByte(bArray[i], newCrc); + } + crc = newCrc; + } + + public void update(byte[] bArray) { + update(bArray, 0, bArray.length); + } + + /** + * Returns the value of the checksum. + * @return the long representation of the checksum (high bits set to zero). + */ + @Override + public long getValue() { + return crc; + } + + /** + * Returns the value of the checksum. + * @return the 4-byte array representation of the checksum in network byte order (big endian). + */ + public byte[] getValueAsBytes() { + long value = crc; + byte[] result = new byte[4]; + for (int i = 3; i >= 0; i--) { + result[i] = (byte) (value & 0xffL); + value >>= 8; + } + return result; + } + + /** + * Resets the crc. + */ + @Override + public void reset() { + crc = 0; + } + + private long updateByte(byte newByte, long crc) { + byte b = (byte) (newByte & BYTE_MASK); + int index = (int) ((crc ^ b) & BYTE_MASK); + return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK; + } +} \ No newline at end of file diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/Consts.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/Consts.java new file mode 100644 index 0000000..7c307ee --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/Consts.java @@ -0,0 +1,14 @@ +package org.rehlds.flightrec; + +public class Consts { + public final static String META_HEADER_SIG_STR = "REHLDS_FLIGHTREC_META"; + public final static String DATA_HEADER_SIG_STR = "REHLDS_FLIGHTREC_DATA"; + + public static byte[] META_HEADER_SIG_BYTES = (META_HEADER_SIG_STR + META_HEADER_SIG_STR + META_HEADER_SIG_STR + ":").getBytes(); + public static byte[] DATA_HEADER_SIG_BYTES = (DATA_HEADER_SIG_STR + DATA_HEADER_SIG_STR + DATA_HEADER_SIG_STR + ":").getBytes(); + + public static int META_HEADER_SIZE = 128; + public static int DATA_HEADER_SIZE = 128; + + public static int MAX_HEADER_SIZE = Math.max(META_HEADER_SIZE, DATA_HEADER_SIZE); +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/AllocEntPrivateDataV1Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/AllocEntPrivateDataV1Decoder.java new file mode 100644 index 0000000..a02ef7b --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/AllocEntPrivateDataV1Decoder.java @@ -0,0 +1,22 @@ +package org.rehlds.flightrec.decoders.rehlds; + + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class AllocEntPrivateDataV1Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "AllocEntPrivateData", 1, false); + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + long ptr = sb.readUInt32(); + return DecodedExtraData.create("pPrivData", "0x" + Long.toHexString(ptr)); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV1Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV1Decoder.java new file mode 100644 index 0000000..e013329 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV1Decoder.java @@ -0,0 +1,30 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FrameV1Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "Frame", 1, true); + } + + DecodedExtraData decodeStart(UtilSizeBuf sb) { + double startTime = sb.readDouble(); + return DecodedExtraData.create("startTime", "" + startTime); + } + + DecodedExtraData decodeEnd(UtilSizeBuf sb) { + return DecodedExtraData.EMPTY; + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + return msg.isEnterMessage() ? decodeStart(sb) : decodeEnd(sb); + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV2Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV2Decoder.java new file mode 100644 index 0000000..d731005 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FrameV2Decoder.java @@ -0,0 +1,32 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FrameV2Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "Frame", 2, true); + } + + DecodedExtraData decodeStart(UtilSizeBuf sb) { + long frameId = sb.readInt64(); + double startTime = sb.readDouble(); + return DecodedExtraData.create("frameId", "" + frameId, "startTime", "" + startTime); + } + + DecodedExtraData decodeEnd(UtilSizeBuf sb) { + long frameId = sb.readInt64(); + return DecodedExtraData.create("frameId", "" + frameId); + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + return msg.isEnterMessage() ? decodeStart(sb) : decodeEnd(sb); + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FreeEntPrivateDataV1Decoder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FreeEntPrivateDataV1Decoder.java new file mode 100644 index 0000000..b306e92 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/FreeEntPrivateDataV1Decoder.java @@ -0,0 +1,21 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.DecodedExtraData; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.api.MessageDecoder; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + +public class FreeEntPrivateDataV1Decoder implements MessageDecoder { + @Override + public FlightrecMessageType getMessageType() { + return new FlightrecMessageType("rehlds", "FreeEntPrivateData", 1, false); + } + + @Override + public DecodedExtraData decode(FlightrecMessage msg) { + UtilSizeBuf sb = msg.getDataSizebuf(); + long ptr = sb.readUInt32(); + return DecodedExtraData.create("pPrivData", "0x" + Long.toHexString(ptr)); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/RehldsDecodersModule.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/RehldsDecodersModule.java new file mode 100644 index 0000000..e143e20 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/decoders/rehlds/RehldsDecodersModule.java @@ -0,0 +1,15 @@ +package org.rehlds.flightrec.decoders.rehlds; + +import org.rehlds.flightrec.api.SimpleDecoderModule; + +public class RehldsDecodersModule extends SimpleDecoderModule { + + public RehldsDecodersModule() { + super("Rehlds decoders (built-in)", "0.2"); + registerDecoder(new FrameV1Decoder()); + registerDecoder(new FreeEntPrivateDataV1Decoder()); + registerDecoder(new AllocEntPrivateDataV1Decoder()); + + registerDecoder(new FrameV2Decoder()); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java index 1a27505..a67f82e 100644 --- a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/FlightRecFileScanner.java @@ -1,27 +1,25 @@ package org.rehlds.flightrec.filescan; +import com.google.cloud.Crc32c; +import org.rehlds.flightrec.api.util.UtilByteBuffer; +import org.rehlds.flightrec.api.util.UtilSizeBuf; +import static org.rehlds.flightrec.Consts.*; + import java.io.IOException; import java.io.RandomAccessFile; import java.util.Arrays; import java.util.List; -import java.util.zip.CRC32; + + public class FlightRecFileScanner { RandomAccessFile file; long fileLen; FileScanResult scanRes = new FileScanResult(); - private final static int HEADER_SIZE = 128; - private final static String META_HEADER = "REHLDS_FLIGHTREC_META"; - private final static String DATA_HEADER = "REHLDS_FLIGHTREC_DATA"; - // 012345678901234567890 - // 000000000011111111112 - - private static byte[] META_HEADER_SIGNATURE = (META_HEADER + META_HEADER + META_HEADER + ":").getBytes(); - private static byte[] DATA_HEADER_SIGNATURE = (META_HEADER + META_HEADER + META_HEADER + ":").getBytes(); - private byte readBuf[] = new byte[65536]; - private byte header[] = new byte[HEADER_SIZE]; + private byte header[] = new byte[MAX_HEADER_SIZE]; + private UtilSizeBuf headerSBuf = new UtilSizeBuf("header", new UtilByteBuffer(header), 0, header.length); private FlightRecFileScanner(RandomAccessFile file) throws IOException { this.file = file; @@ -29,54 +27,54 @@ public class FlightRecFileScanner { } private void examineHeader(byte[] data, int size, int pos) throws IOException { - if (pos + HEADER_SIZE < size) { - System.arraycopy(data, pos, header, 0, HEADER_SIZE); + if (pos + MAX_HEADER_SIZE < size) { + System.arraycopy(data, pos, header, 0, MAX_HEADER_SIZE); } else { return; //will be read in next iteration } + headerSBuf.reset(); String matchedType = null; - if (Arrays.equals(META_HEADER_SIGNATURE, Arrays.copyOfRange(header, 0, META_HEADER_SIGNATURE.length))) { - matchedType = META_HEADER; - } else if (Arrays.equals(DATA_HEADER_SIGNATURE, Arrays.copyOfRange(header, 0, DATA_HEADER_SIGNATURE.length))) { - matchedType = DATA_HEADER; + if (Arrays.equals(META_HEADER_SIG_BYTES, Arrays.copyOfRange(header, 0, META_HEADER_SIG_BYTES.length))) { + matchedType = META_HEADER_SIG_STR; + headerSBuf.skip(META_HEADER_SIG_BYTES.length); + } else if (Arrays.equals(DATA_HEADER_SIG_BYTES, Arrays.copyOfRange(header, 0, DATA_HEADER_SIG_BYTES.length))) { + matchedType = DATA_HEADER_SIG_STR; + headerSBuf.skip(DATA_HEADER_SIG_BYTES.length); } if (matchedType == null) { return; } - List resList = (matchedType.equals(META_HEADER)) ? scanRes.metaHeaders : scanRes.dataHeaders; - int versionPos = matchedType.length() * 3 + 1; - int version = header[versionPos] | header[versionPos+1] << 8 | header[versionPos+2] << 16 | header[versionPos+3] << 24; - int allocSizePos = matchedType.length() * 3 + 1 + 4; - int allocSize = header[allocSizePos] | header[allocSizePos+1] << 8 | header[allocSizePos+2] << 16 | header[allocSizePos+3] << 24; + List resList = (matchedType.equals(META_HEADER_SIG_STR)) ? scanRes.metaHeaders : scanRes.dataHeaders; + int version = headerSBuf.readInt32(); + int allocSize = headerSBuf.readInt32(); - CRC32 crc32 = new CRC32(); - int bufCksumPos = matchedType.length() * 3 + 1 + 4 + 4; //3*head + ":" + version + allocsize - crc32.update(header, 0, bufCksumPos); + Crc32c crc32 = new Crc32c(); + crc32.update(header, 0, headerSBuf.tell()); long calculatedChecksum = crc32.getValue(); - long bufChecksum = header[bufCksumPos] | header[bufCksumPos+1] << 8 | header[bufCksumPos+2] << 16 | header[bufCksumPos+3] << 24; + long bufChecksum = headerSBuf.readUInt32(); if (calculatedChecksum != bufChecksum) { - resList.add(new HeaderScanResult(file.getFilePointer() + pos, allocSize, false, "Checksum mismatch", version)); + resList.add(new HeaderScanResult(file.getFilePointer() - size + pos, allocSize, false, "Checksum mismatch", version)); return; } long endPos = file.getFilePointer() - size + pos + allocSize; if (endPos >= file.length()) { - resList.add(new HeaderScanResult(file.getFilePointer() + pos, allocSize, false, "Regions partially lays outside the file", version)); + resList.add(new HeaderScanResult(file.getFilePointer() - size + pos, allocSize, false, "Regions partially lays outside the file", version)); return; } - resList.add(new HeaderScanResult(file.getFilePointer() + pos, allocSize, true, null, version)); + resList.add(new HeaderScanResult(file.getFilePointer() - size + pos, allocSize, true, null, version)); } private void scanForHeaders(byte[] data, int size) throws IOException { - int maxHeaderSize = Math.max(META_HEADER.length(), DATA_HEADER.length()); + int maxHeaderSize = Math.max(META_HEADER_SIG_STR.length(), DATA_HEADER_SIG_STR.length()); for (int i = 0; i < size - maxHeaderSize; i++) { - if (data[i + 15] == META_HEADER_SIGNATURE[15] && data[i + 16] == META_HEADER_SIGNATURE[16] && data[i + 17] == META_HEADER_SIGNATURE[17] && data[i + 18] == META_HEADER_SIGNATURE[18]) { + if (data[i + 15] == META_HEADER_SIG_BYTES[15] && data[i + 16] == META_HEADER_SIG_BYTES[16] && data[i + 17] == META_HEADER_SIG_BYTES[17] && data[i + 18] == META_HEADER_SIG_BYTES[18]) { examineHeader(data, size, i); - } else if (data[i + 15] == DATA_HEADER_SIGNATURE[15] && data[i + 16] == DATA_HEADER_SIGNATURE[16] && data[i + 17] == DATA_HEADER_SIGNATURE[17] && data[i + 18] == DATA_HEADER_SIGNATURE[18]) { + } else if (data[i + 15] == DATA_HEADER_SIG_BYTES[15] && data[i + 16] == DATA_HEADER_SIG_BYTES[16] && data[i + 17] == DATA_HEADER_SIG_BYTES[17] && data[i + 18] == DATA_HEADER_SIG_BYTES[18]) { examineHeader(data, size, i); } } @@ -89,7 +87,7 @@ public class FlightRecFileScanner { while (-1 != (read = file.read(readBuf))) { scanForHeaders(readBuf, read); if (read == readBuf.length) { - file.seek(file.getFilePointer() - HEADER_SIZE * 2); + file.seek(file.getFilePointer() - MAX_HEADER_SIZE * 2); } } } diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java index fd03ae9..46e93c4 100644 --- a/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/filescan/HeaderScanResult.java @@ -7,9 +7,6 @@ public class HeaderScanResult { public String error; public int version; - public HeaderScanResult() { - } - public HeaderScanResult(long pos, int len, boolean valid, String error, int version) { this.pos = pos; this.len = len; diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/DataHeader.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/DataHeader.java new file mode 100644 index 0000000..c0b961f --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/DataHeader.java @@ -0,0 +1,12 @@ +package org.rehlds.flightrec.logparser; + +public class DataHeader { + public int prevItrLastPos; + + public DataHeader(int prevItrLastPos) { + this.prevItrLastPos = prevItrLastPos; + } + + public DataHeader() { + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/FlightLogParser.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/FlightLogParser.java new file mode 100644 index 0000000..cde2e84 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/FlightLogParser.java @@ -0,0 +1,179 @@ +package org.rehlds.flightrec.logparser; + +import org.rehlds.flightrec.api.EntranceKind; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageDef; +import org.rehlds.flightrec.api.FlightrecMessageType; +import org.rehlds.flightrec.filescan.HeaderScanResult; +import org.rehlds.flightrec.api.util.UtilByteBuffer; +import org.rehlds.flightrec.api.util.UtilSizeBuf; +import static org.rehlds.flightrec.Consts.*; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.*; + +public class FlightLogParser { + UtilByteBuffer metaRegion; + UtilByteBuffer dataRegion; + + MetaHeader metaHeader; + RecorderState recorderState; + DataHeader dataHeader; + + Map msgTypes = new HashMap<>(); + List messages = new ArrayList<>(); + + void parseMessageDefinition(UtilSizeBuf sbuf) { + int msgId = sbuf.readUInt16(); + String module = sbuf.readString(); + String messageName = sbuf.readString(); + long msgVersion = sbuf.readUInt32(); + boolean inOut = sbuf.readBool(); + + FlightrecMessageDef msgDef = new FlightrecMessageDef(module, messageName, msgVersion, inOut, msgId); + + if (msgTypes.containsKey(msgId)) { + System.out.println("Duplicate message id: " + msgTypes.get(msgId) + " and " + msgDef); + } + + msgTypes.put(msgId, msgDef.msgType); + } + + void parseMetaRegion() { + metaHeader = new MetaHeader(); + UtilSizeBuf metaSBuf = new UtilSizeBuf("meta region", metaRegion); + metaSBuf.skip(META_HEADER_SIG_BYTES.length); //skip signature + metaSBuf.readInt32(); //version + metaSBuf.readInt32(); //allocSize + metaSBuf.readInt32(); //checksum + metaHeader.numDefinitions = metaSBuf.readInt32(); + metaHeader.metaRegionPos = metaSBuf.readInt32(); + + recorderState = new RecorderState(); + recorderState.wpos = metaSBuf.readInt32(); + recorderState.lastMsgBeginPos = metaSBuf.readInt32(); + recorderState.curMessage = metaSBuf.readUInt16(); + + metaSBuf = new UtilSizeBuf("meta region defs", metaRegion, META_HEADER_SIZE, metaHeader.metaRegionPos); + for (int i = 0; i < metaHeader.numDefinitions; i++) { + int defKind = metaSBuf.readUInt8(); + switch (defKind) { + case 1: //MRT_MESSAGE_DEF + parseMessageDefinition(metaSBuf); + break; + + default: + throw new RuntimeException("Invalid meta definition type" + defKind); + } + } + + dataHeader = new DataHeader(); + dataHeader.prevItrLastPos = dataRegion.readInt32(DATA_HEADER_SIG_BYTES.length + 12); + } + + public FlightLogParser(UtilByteBuffer metaRegion, UtilByteBuffer dataRegion) { + this.metaRegion = metaRegion; + this.dataRegion = dataRegion; + } + + void doParseMessage(UtilSizeBuf msg) { + int opc = msg.readUInt16(); + boolean entrance = (0 != (opc & 0x8000)); + opc &= 0x7FFF; + + FlightrecMessageType msgType = msgTypes.get(opc); + if (msgType == null) { + throw new RuntimeException("Invalid message opcode @" + Long.toHexString(msg.getAbsoluteCurrentPos() - 2) + ": " + opc); + } + + EntranceKind entranceKind; + if (msgType.inout) { + entranceKind = entrance ? EntranceKind.ENTRANCE_ENTER : EntranceKind.ENTRANCE_LEAVE; + } else { + entranceKind = EntranceKind.ENTRANCE_UNUSED; + } + + FlightrecMessage flMsg = new FlightrecMessage(msgType, entranceKind, msg.getBuffer(), msg.getAbsoluteCurrentPos(), msg.getMaxSize() - 2); + messages.add(flMsg); + } + + void parseMessage(UtilSizeBuf msg) { + int startPos = msg.getStartPos(); + try { + doParseMessage(msg); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("Error while parsing message @" + startPos); + } + } + + List parse() { + parseMetaRegion(); + + UtilByteBuffer flightData = dataRegion.cutLeft(DATA_HEADER_SIZE); + + boolean flippedToEnd = false; + + /* + Each message has following layout: + Opcode [2 bytes] + Data [0+ bytes] + Length of opcode + data [2 bytes] + */ + int curMsgPos = (recorderState.curMessage == 0) ? recorderState.wpos : recorderState.lastMsgBeginPos; + curMsgPos -= 2; //position of the Length field of the message + + UtilSizeBuf msg = new UtilSizeBuf("flightrec_message", flightData, 0, 0); + + while (true) { + if (flippedToEnd && curMsgPos <= recorderState.wpos) + break; + + if (curMsgPos <= 0) { //move read pointer to the end of the data region + if (dataHeader.prevItrLastPos == -1) //wpos never reached end of the region + break; + + curMsgPos = dataHeader.prevItrLastPos - 2; + flippedToEnd = true; + continue; + } + + int msgLen = flightData.readUInt16(curMsgPos); + int msgStartPos = curMsgPos - msgLen; + if (msgStartPos < 0) { + throw new RuntimeException("Corrupted data region; read msgLen=" + msgLen + " at " + curMsgPos + ", but it is too large (startPos < 0)"); + } + + if (flippedToEnd && msgStartPos < recorderState.wpos) { + break; + } + + msg.init(msgStartPos, msgLen); + parseMessage(msg); + + curMsgPos = msgStartPos - 2; + } + + return messages; + } + + public static List doParse(RandomAccessFile f, HeaderScanResult metaHeader, HeaderScanResult dataHeader) throws IOException { + //read regions to byte buffers + f.seek(metaHeader.pos); + byte[] metaRegionData = new byte[metaHeader.len]; + f.readFully(metaRegionData); + + f.seek(dataHeader.pos); + byte[] dataRegionData = new byte[dataHeader.len]; + f.readFully(dataRegionData); + + UtilByteBuffer metaRegion = new UtilByteBuffer(metaRegionData); + UtilByteBuffer dataRegion = new UtilByteBuffer(dataRegionData); + + List res = new FlightLogParser(metaRegion, dataRegion).parse(); + Collections.reverse(res); + return res; + } +} + diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/LogParsingException.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/LogParsingException.java new file mode 100644 index 0000000..bd65f3d --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/LogParsingException.java @@ -0,0 +1,12 @@ +package org.rehlds.flightrec.logparser; + +public class LogParsingException extends RuntimeException { + + public LogParsingException(String message) { + super(message); + } + + public LogParsingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/MetaHeader.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/MetaHeader.java new file mode 100644 index 0000000..19b3eba --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/MetaHeader.java @@ -0,0 +1,14 @@ +package org.rehlds.flightrec.logparser; + +public class MetaHeader { + public int numDefinitions; + public int metaRegionPos; + + public MetaHeader(int numMessages, int metaRegionPos) { + this.numDefinitions = numMessages; + this.metaRegionPos = metaRegionPos; + } + + public MetaHeader() { + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/RecorderState.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/RecorderState.java new file mode 100644 index 0000000..a965b64 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logparser/RecorderState.java @@ -0,0 +1,16 @@ +package org.rehlds.flightrec.logparser; + +public class RecorderState { + public int wpos; + public int lastMsgBeginPos; + public int curMessage; + + public RecorderState(int wpos, int lastMsgBeginPos, int curMessage) { + this.wpos = wpos; + this.lastMsgBeginPos = lastMsgBeginPos; + this.curMessage = curMessage; + } + + public RecorderState() { + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/FlightLogTreeBuilder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/FlightLogTreeBuilder.java new file mode 100644 index 0000000..9648822 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/FlightLogTreeBuilder.java @@ -0,0 +1,71 @@ +package org.rehlds.flightrec.logtree; + +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.api.FlightrecMessageType; + +import java.util.List; + +public class FlightLogTreeBuilder { + LogTreeNodeComplex rootNode = new LogTreeNodeComplex(null, null, null); + LogTreeNodeComplex currentNode = rootNode; + + void handleEnterMessage(FlightrecMessage msg) { + LogTreeNodeComplex n = new LogTreeNodeComplex(currentNode, msg, null); + currentNode.addChild(n); + currentNode = n; + } + + void handleLeaveMessage(FlightrecMessage msg) { + if (currentNode == rootNode) { + rootNode = new LogTreeNodeComplex(null, null, msg); + rootNode.addChild(currentNode); + currentNode.setParent(rootNode); + currentNode = rootNode; + return; + } + + if (currentNode.enterMsg != null) { + FlightrecMessageType startType = currentNode.enterMsg.messageType; + FlightrecMessageType endType = msg.messageType; + if (!startType.equals(endType)) { + throw new RuntimeException("Closing message @" + Long.toHexString(msg.rawDataPos) + " has invalid type " + endType + "; expected " + startType); + } + } + + currentNode.leaveMsg = msg; + currentNode = currentNode.parent; + } + + void handleSimpleMessage(FlightrecMessage msg) { + LogTreeNodeLeaf leafNode = new LogTreeNodeLeaf(currentNode, msg); + currentNode.addChild(leafNode); + } + + void doBuildLogTree(List messages) { + for (FlightrecMessage msg : messages) { + switch (msg.entranceKind) { + case ENTRANCE_ENTER: + handleEnterMessage(msg); + break; + + case ENTRANCE_LEAVE: + handleLeaveMessage(msg); + break; + + case ENTRANCE_UNUSED: + handleSimpleMessage(msg); + break; + + default: + throw new RuntimeException("Invalid exntrance kind"); + } + } + } + + + public static LogTreeNodeComplex buildTree(List messages) { + FlightLogTreeBuilder builder = new FlightLogTreeBuilder(); + builder.doBuildLogTree(messages); + return builder.rootNode; + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNode.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNode.java new file mode 100644 index 0000000..de6d14f --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNode.java @@ -0,0 +1,21 @@ +package org.rehlds.flightrec.logtree; + +import java.util.List; + +public abstract class LogTreeNode { + LogTreeNodeComplex parent; + + protected LogTreeNode(LogTreeNodeComplex parent) { + this.parent = parent; + } + + abstract List getChildren(); + + LogTreeNodeComplex getParent() { + return parent; + } + + public void setParent(LogTreeNodeComplex parent) { + this.parent = parent; + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeComplex.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeComplex.java new file mode 100644 index 0000000..3f99094 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeComplex.java @@ -0,0 +1,36 @@ +package org.rehlds.flightrec.logtree; + +import org.rehlds.flightrec.api.FlightrecMessage; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class LogTreeNodeComplex extends LogTreeNode { + public FlightrecMessage enterMsg; + public FlightrecMessage leaveMsg; + + public LogTreeNodeComplex(LogTreeNodeComplex parent, FlightrecMessage enterMsg, FlightrecMessage leaveMsg) { + super(parent); + this.enterMsg = enterMsg; + this.leaveMsg = leaveMsg; + } + + List children = Collections.emptyList(); + + + + @Override + public List getChildren() { + return children; + } + + public void addChild(LogTreeNode node) { + if (children.isEmpty()) { + children = new ArrayList<>(); + } + + children.add(node); + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeLeaf.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeLeaf.java new file mode 100644 index 0000000..bfdb6ca --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/logtree/LogTreeNodeLeaf.java @@ -0,0 +1,20 @@ +package org.rehlds.flightrec.logtree; + +import org.rehlds.flightrec.api.FlightrecMessage; + +import java.util.Collections; +import java.util.List; + +public class LogTreeNodeLeaf extends LogTreeNode { + public FlightrecMessage msg; + + public LogTreeNodeLeaf(LogTreeNodeComplex parent, FlightrecMessage msg) { + super(parent); + this.msg = msg; + } + + @Override + List getChildren() { + return Collections.emptyList(); + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/FlightRecorder.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/FlightRecorder.java new file mode 100644 index 0000000..231ae8a --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/FlightRecorder.java @@ -0,0 +1,229 @@ +package org.rehlds.flightrec.main; + +import org.rehlds.flightrec.api.DecoderModule; +import org.rehlds.flightrec.api.FlightrecMessage; +import org.rehlds.flightrec.decoders.rehlds.RehldsDecodersModule; +import org.rehlds.flightrec.filescan.FileScanResult; +import org.rehlds.flightrec.filescan.FlightRecFileScanner; +import org.rehlds.flightrec.filescan.HeaderScanResult; +import org.rehlds.flightrec.logtree.FlightLogTreeBuilder; +import org.rehlds.flightrec.logtree.LogTreeNodeComplex; +import org.rehlds.flightrec.logparser.FlightLogParser; +import org.rehlds.flightrec.textlogwriter.TextLogWriter; +import org.rehlds.flightrec.util.JarUtils; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; + +public class FlightRecorder { + RunConfig cfg; + List decoderModules = new ArrayList<>(); + + public FlightRecorder(RunConfig cfg) { + this.cfg = cfg; + } + + private boolean checkConfig() { + if (cfg.dumpFile == null) { + System.out.println("Dump file is not selected, please use --dump-file parameter to specify it"); + return false; + } + + if (cfg.outFile == null) { + cfg.outFile = new File(cfg.dumpFile.getAbsolutePath() + ".flog"); + } + + return true; + } + + private List scanFile(RandomAccessFile f) throws IOException { + FileScanResult scanResult = FlightRecFileScanner.scan(f); + + System.out.println("Dump file scan results: "); + for (HeaderScanResult metaHeader : scanResult.metaHeaders) { + System.out.print("\tMeta header @ 0x" + Long.toHexString(metaHeader.pos) + "; valid: " + (metaHeader.error == null)); + if (metaHeader.error != null) { + System.out.print("; error: " + metaHeader.error); + } + System.out.println(); + } + for (HeaderScanResult dataHeader : scanResult.dataHeaders) { + System.out.print("\tData header @ 0x" + Long.toHexString(dataHeader.pos) + "; valid: " + (dataHeader.error == null)); + if (dataHeader.error != null) { + System.out.print("; error: " + dataHeader.error); + } + System.out.println(); + } + + HeaderScanResult validMetaHeader = null; + HeaderScanResult validDataHeader = null; + + for (HeaderScanResult metaHeader : scanResult.metaHeaders) { + if (metaHeader.error != null) { + continue; + } + + if (validMetaHeader != null) { + System.out.println("Multiple meta headers found, exiting"); + return null; + } + + validMetaHeader = metaHeader; + } + + for (HeaderScanResult dataHeader : scanResult.dataHeaders) { + if (dataHeader.error != null) { + continue; + } + + if (validDataHeader != null) { + System.out.println("Multiple data headers found, exiting"); + return null; + } + + validDataHeader = dataHeader; + } + + if (validMetaHeader == null) { + System.out.println("Meta header not found, exiting"); + return null; + } + + if (validDataHeader == null) { + System.out.println("Data header not found, exiting"); + return null; + } + + return FlightLogParser.doParse(f, validMetaHeader, validDataHeader); + } + + private LogTreeNodeComplex buildTree(List messages) { + return FlightLogTreeBuilder.buildTree(messages); + } + + private boolean writeOutputFile(LogTreeNodeComplex logTreeRoot) { + TextLogWriter.decodeAndWrite(logTreeRoot, cfg.outFile, decoderModules); + System.out.println("Written decoded log to '" + cfg.outFile.getAbsolutePath() + ";"); + return true; + } + + public boolean run() { + registerBuiltinDecoders(); + loadExternalDecoders(); + + if (!checkConfig()) { + return false; + } + + List messages; + try(RandomAccessFile f = new RandomAccessFile(cfg.dumpFile, "r")) { + messages = scanFile(f); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + + if (messages == null) { + return false; + } + + LogTreeNodeComplex treeRootNode = buildTree(messages); + if (treeRootNode == null) { + return false; + } + + if (!writeOutputFile(treeRootNode)) { + return false; + } + + return true; + } + + private void loadExternalDecoders() { + File f = JarUtils.getJarFileOfClass(FlightRecorder.class); + if (f == null) { + System.out.println("Could not locate main JAR, external decoders will not be loaded"); + return; + } + + File extDir = new File(f.getParentFile(), "extDecoders"); + if (!extDir.exists() || !extDir.isDirectory()) { + System.out.println("Directory '" + extDir.getAbsolutePath() + "' doesn't exist"); + } + + File[] jarFiles = extDir.listFiles(new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.getName().toLowerCase().endsWith(".jar"); + } + }); + + ArrayList jarUrls = new ArrayList<>(); + for (File jf : jarFiles) { + try { + jarUrls.add(jf.toURI().toURL()); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + URLClassLoader extDecodersClassloader = new URLClassLoader(jarUrls.toArray(new URL[jarUrls.size()]), this.getClass().getClassLoader()); + ServiceLoader srvLoader = ServiceLoader.load(DecoderModule.class, extDecodersClassloader); + for (DecoderModule decoderModule : srvLoader) { + System.out.println("Loaded external decoder module " + decoderModule.getDescription() + " version " + decoderModule.getVersion()); + decoderModules.add(decoderModule); + } + } + + private void registerBuiltinDecoders() { + decoderModules.add(new RehldsDecodersModule()); + } + + public static void main(String args[]) { + RunConfig cfg; + try { + cfg = parseArgs(args); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + return; + } + + new FlightRecorder(cfg).run(); + } + + private static RunConfig parseArgs(String args[]) { + RunConfig cfg = new RunConfig(); + for (int i = 0; i < args.length; i++) { + String a = args[i]; + + if ("--dump-file".equals(a)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--dump-file should be followed by file name"); + } + i++; + cfg.dumpFile = new File(args[i]); + continue; + } + + if ("--out-file".equals(a)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--out-file should be followed by file name"); + } + i++; + cfg.outFile = new File(args[i]); + continue; + } + + throw new IllegalArgumentException("Invalid command line parameter: '" + a + "'"); + } + + return cfg; + } + +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/RunConfig.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/RunConfig.java new file mode 100644 index 0000000..191dd33 --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/main/RunConfig.java @@ -0,0 +1,8 @@ +package org.rehlds.flightrec.main; + +import java.io.File; + +public class RunConfig { + public File dumpFile; + public File outFile; +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/textlogwriter/TextLogWriter.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/textlogwriter/TextLogWriter.java new file mode 100644 index 0000000..4ec3e2b --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/textlogwriter/TextLogWriter.java @@ -0,0 +1,182 @@ +package org.rehlds.flightrec.textlogwriter; + +import org.rehlds.flightrec.api.*; +import org.rehlds.flightrec.logtree.LogTreeNode; +import org.rehlds.flightrec.logtree.LogTreeNodeComplex; +import org.rehlds.flightrec.logtree.LogTreeNodeLeaf; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TextLogWriter { + Writer writer; + List decoderModules; + + int indent; + HashMap indents = new HashMap<>(); + Map decodersByMsgType = new HashMap<>(); + + public TextLogWriter(Writer writer, List decoderModules) { + this.writer = writer; + this.decoderModules = decoderModules; + } + + MessageDecoder lookupDecoder(FlightrecMessageType msgType) { + for (DecoderModule dm : decoderModules) { + MessageDecoder d = dm.lookupDecoder(msgType); + if (d != null) { + return d; + } + } + + return null; + } + + MessageDecoder getDecoder(FlightrecMessage message) { + FlightrecMessageType msgType = message.messageType; + if (!decodersByMsgType.containsKey(msgType)) { + decodersByMsgType.put(msgType, lookupDecoder(msgType)); + } + + return decodersByMsgType.get(msgType); + } + + DecodedExtraData tryDecode(FlightrecMessage message) { + MessageDecoder decoder = getDecoder(message); + if (decoder == null) { + return null; + } + return decoder.decode(message); + } + + String escapeString(String s) { + return s.replace("\"", "\\\"").replace("'", "\\'"); + } + + String generateIndent() { + String res = indents.get(indent); + if (res != null) { + return res; + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < indent; i++) { + sb.append(" "); + } + + res = sb.toString(); + indents.put(indent, res); + return res; + } + + void writeExtraData(StringBuilder sb, DecodedExtraData extraData) { + boolean first = true; + for (ImmutablePair kv : extraData.data) { + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append(kv.first).append(": '").append(escapeString(kv.second)).append("'"); + } + } + + String prepareMessageText(FlightrecMessage msg) { + StringBuilder sb = new StringBuilder(); + sb.append(generateIndent()); + + switch (msg.entranceKind) { + case ENTRANCE_ENTER: + sb.append(">>"); + break; + + case ENTRANCE_LEAVE: + sb.append("<<"); + break; + + case ENTRANCE_UNUSED: + sb.append("--"); + break; + + default: + throw new RuntimeException("Invalid entrance kind " + msg.entranceKind); + } + + FlightrecMessageType msgType = msg.messageType; + sb.append(" ").append(msgType.module).append(".").append(msgType.message).append(":").append(msgType.version).append(" "); + + DecodedExtraData extraData = tryDecode(msg); + if (extraData != null) { + writeExtraData(sb, extraData); + } else { + sb.append("undecoded["); + boolean firstByte = true; + for (int i = msg.rawDataPos; i < msg.rawDataLen + msg.rawDataPos; i++) { + if (firstByte) { + firstByte = false; + } else { + sb.append(" "); + } + sb.append(String.format("%02X", msg.rawData[i] & 0xFF)); + } + sb.append("]"); + } + + sb.append("\n"); + return sb.toString(); + } + + void writeMessage(FlightrecMessage msg) throws IOException { + String text = prepareMessageText(msg); + writer.write(text); + } + + void writeLeafNode(LogTreeNodeLeaf node) throws IOException { + writeMessage(node.msg); + } + + void writeComplexNode(LogTreeNodeComplex node) throws IOException { + if (node.enterMsg != null) { + writeMessage(node.enterMsg); + } else { + writer.write(generateIndent() + ">> [Unknown]"); + } + + indent++; + writeNodes(node.getChildren()); + indent--; + + if (node.leaveMsg != null) { + writeMessage(node.leaveMsg); + } else { + writer.write(generateIndent() + "<< [Unknown]"); + } + } + + void writeNodes(List nodes) throws IOException { + for (LogTreeNode node : nodes) { + if (node instanceof LogTreeNodeComplex) { + writeComplexNode((LogTreeNodeComplex) node); + } else if (node instanceof LogTreeNodeLeaf) { + writeLeafNode((LogTreeNodeLeaf) node); + } else { + throw new RuntimeException("Invalid node class " + node.getClass().getName()); + } + } + } + + + public static void decodeAndWrite(LogTreeNodeComplex rootNode, File outFile, List decoderModules) { + try (FileWriter fw = new FileWriter(outFile)) { + TextLogWriter logWriter = new TextLogWriter(fw, decoderModules); + logWriter.writeNodes(rootNode.getChildren()); + } catch (IOException e) { + throw new RuntimeException("Failed to open/write file '" + outFile + "': " + e.getMessage(), e); + } + } +} diff --git a/flightrec/decoder/src/main/java/org/rehlds/flightrec/util/JarUtils.java b/flightrec/decoder/src/main/java/org/rehlds/flightrec/util/JarUtils.java new file mode 100644 index 0000000..41d062a --- /dev/null +++ b/flightrec/decoder/src/main/java/org/rehlds/flightrec/util/JarUtils.java @@ -0,0 +1,31 @@ +package org.rehlds.flightrec.util; + +import java.io.File; +import java.net.URL; + +public class JarUtils { + public static File getJarFileOfClass(Class c) { + String classFileName = c.getName().replace('.', '/') + ".class"; + ClassLoader classLoader = c.getClassLoader(); + if (classLoader == null) { + classLoader = JarUtils.class.getClassLoader(); + } + URL url = classLoader.getResource(classFileName); + if (url == null) { + return null; + } + + String strUrl = url.toString(); + if (!strUrl.startsWith("jar:file:/")) { + return null; + } + + int jarSeparator = strUrl.indexOf('!'); + if (jarSeparator == -1) { + return null; + } + + String jarFilePath = strUrl.substring("jar:file:/".length(), jarSeparator); + return new File(jarFilePath); + } +} diff --git a/flightrec/decoder/src/test/java/com/google/cloud/Crc32cTest.java b/flightrec/decoder/src/test/java/com/google/cloud/Crc32cTest.java new file mode 100644 index 0000000..51328da --- /dev/null +++ b/flightrec/decoder/src/test/java/com/google/cloud/Crc32cTest.java @@ -0,0 +1,43 @@ +package com.google.cloud; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class Crc32cTest { + + static class TestData { + public String src; + public long hash; + + TestData(String src, long hash) { + this.src = src; + this.hash = hash; + } + } + + @Test + public void testCrc32c() { + TestData testData[] = { + new TestData("a", 0x93AD1061L), + new TestData("ab", 0x13C35EE4L), + new TestData("abc", 0x562F9CCDL), + new TestData("abcd", 0xDAAF41F6L), + new TestData("abcde", 0x8122A0A2L), + new TestData("abcdef", 0x0496937BL), + new TestData("abcdefg", 0x5D199E2CL), + new TestData("abcdefgh", 0x86BC933DL), + new TestData("abcdefghi", 0x9639F15FL), + new TestData("abcdefghij", 0x0584645CL), + }; + + for (TestData t : testData) { + Crc32c crc32c = new Crc32c(); + crc32c.update(t.src.getBytes()); + long cksum = crc32c.getValue(); + + assertEquals(t.hash, cksum); + } + } + +} \ No newline at end of file diff --git a/flightrec/decoder_api/build.gradle b/flightrec/decoder_api/build.gradle index 5fd35a5..574f92d 100644 --- a/flightrec/decoder_api/build.gradle +++ b/flightrec/decoder_api/build.gradle @@ -1,4 +1,57 @@ apply plugin: 'java' +apply plugin: 'maven-publish' + +group = 'org.rehlds.flightrec' sourceCompatibility = '1.7' targetCompatibility = '1.7' + +repositories { + mavenCentral() +} + +dependencies { + testCompile "junit:junit:4.12" +} + +publishing { + publications { + maven(MavenPublication) { + version rootProject.version + artifactId 'decoder-api' + artifact jar + + pom.withXml { + asNode().children().last() + { + resolveStrategy = DELEGATE_FIRST + name 'decoder-api' + description project.description + //url github + //scm { + // url "${github}.git" + // connection "scm:git:${github}.git" + //} + /* + licenses { + license { + name 'The Apache Software License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution 'repo' + } + } + developers { + developer { + id 'dreamstalker' + name 'dreamstalker' + } + } + */ + } + } + } + } +} + +tasks.withType(AbstractCompile) { + options.encoding = 'UTF-8' +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java index 77b2638..a5564a5 100644 --- a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecodedExtraData.java @@ -1,8 +1,28 @@ package org.rehlds.flightrec.api; public class DecodedExtraData { - String decoderId; - String decoderVersion; + public ImmutablePair[] data; + + public DecodedExtraData(ImmutablePair[] data) { + this.data = data; + } + + public static final DecodedExtraData EMPTY = new DecodedExtraData(new ImmutablePair[0]); + + public static DecodedExtraData create(String... args) { + if ((args.length % 2) == 1) { + throw new RuntimeException("DecodedExtraData.create: number of arguments must be even"); + } + + int numPairs = args.length / 2; + DecodedExtraData res = new DecodedExtraData(new ImmutablePair[numPairs]); + + for (int i = 0; i < numPairs; i++) { + res.data[i] = new ImmutablePair<>(args[i * 2], args[i * 2 + 1]); + } + + return res; + } + - ImmutablePair data[]; } diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecoderModule.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecoderModule.java new file mode 100644 index 0000000..3ed144a --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/DecoderModule.java @@ -0,0 +1,7 @@ +package org.rehlds.flightrec.api; + +public interface DecoderModule { + public MessageDecoder lookupDecoder(FlightrecMessageType msgType); + public String getDescription(); + public String getVersion(); +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java index 133467f..6d3011d 100644 --- a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessage.java @@ -1,10 +1,35 @@ package org.rehlds.flightrec.api; +import org.rehlds.flightrec.api.util.UtilByteBuffer; +import org.rehlds.flightrec.api.util.UtilSizeBuf; + public class FlightrecMessage { - String type; - int version; - byte[] rawData; - EntranceKind entranceKind; + public final FlightrecMessageType messageType; + public final EntranceKind entranceKind; + + public final byte[] rawData; + public final int rawDataPos; + public final int rawDataLen; DecodedExtraData decodedData; + + public FlightrecMessage(FlightrecMessageType messageType, EntranceKind entranceKind, byte[] rawData, int rawDataOffset, int rawDataLen) { + this.messageType = messageType; + this.entranceKind = entranceKind; + this.rawData = rawData; + this.rawDataPos = rawDataOffset; + this.rawDataLen = rawDataLen; + } + + public UtilSizeBuf getDataSizebuf() { + return new UtilSizeBuf("msg: '" + messageType + "' @" + rawDataPos, new UtilByteBuffer(rawData), rawDataPos, rawDataLen); + } + + public boolean isEnterMessage() { + return (entranceKind == EntranceKind.ENTRANCE_ENTER); + } + + public boolean isLeaveMessage() { + return (entranceKind == EntranceKind.ENTRANCE_LEAVE); + } } diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageDef.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageDef.java new file mode 100644 index 0000000..4c40a48 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageDef.java @@ -0,0 +1,22 @@ +package org.rehlds.flightrec.api; + +public class FlightrecMessageDef { + public final FlightrecMessageType msgType; + public final int opcode; + + public FlightrecMessageDef(String module, String message, long version, boolean inout, int opcode) { + msgType = new FlightrecMessageType(module, message, version, inout); + this.opcode = opcode; + } + + @Override + public String toString() { + return "FlightrecMessageDef{" + + "module='" + msgType.module + '\'' + + ", message='" + msgType.message + '\'' + + ", version=" + msgType.version + + ", inout=" + msgType.inout + + ", opcode=" + opcode + + '}'; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageType.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageType.java new file mode 100644 index 0000000..102cc49 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/FlightrecMessageType.java @@ -0,0 +1,49 @@ +package org.rehlds.flightrec.api; + +public class FlightrecMessageType { + public final String module; + public final String message; + public final long version; + public final boolean inout; + + public FlightrecMessageType(String module, String message, long version, boolean inout) { + this.module = module; + this.message = message; + this.version = version; + this.inout = inout; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FlightrecMessageType that = (FlightrecMessageType) o; + + if (inout != that.inout) return false; + if (version != that.version) return false; + if (!message.equals(that.message)) return false; + if (!module.equals(that.module)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = module.hashCode(); + result = 31 * result + message.hashCode(); + result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + (inout ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "FlightrecMessageType{" + + "module='" + module + '\'' + + ", message='" + message + '\'' + + ", version=" + version + + ", inout=" + inout + + '}'; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/MessageDecoder.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/MessageDecoder.java new file mode 100644 index 0000000..69d3f51 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/MessageDecoder.java @@ -0,0 +1,6 @@ +package org.rehlds.flightrec.api; + +public interface MessageDecoder { + FlightrecMessageType getMessageType(); + DecodedExtraData decode(FlightrecMessage msg); +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/SimpleDecoderModule.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/SimpleDecoderModule.java new file mode 100644 index 0000000..a1c3ead --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/SimpleDecoderModule.java @@ -0,0 +1,35 @@ +package org.rehlds.flightrec.api; + +import java.util.HashMap; +import java.util.Map; + +public class SimpleDecoderModule implements DecoderModule { + Map decoders = new HashMap<>(); + + public final String description; + public final String version; + + public SimpleDecoderModule(String description, String version) { + this.description = description; + this.version = version; + } + + @Override + public MessageDecoder lookupDecoder(FlightrecMessageType msgType) { + return decoders.get(msgType); + } + + @Override + public String getDescription() { + return description; + } + + @Override + public String getVersion() { + return version; + } + + public void registerDecoder(MessageDecoder msgDecoder) { + decoders.put(msgDecoder.getMessageType(), msgDecoder); + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/Globals.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/Globals.java new file mode 100644 index 0000000..ba583b7 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/Globals.java @@ -0,0 +1,8 @@ +package org.rehlds.flightrec.api.util; + +import java.nio.charset.Charset; + +public class Globals { + public static final Charset UTF8 = Charset.forName("UTF-8"); + +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/SizebufOverflowException.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/SizebufOverflowException.java new file mode 100644 index 0000000..ad4ec97 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/SizebufOverflowException.java @@ -0,0 +1,10 @@ +package org.rehlds.flightrec.api.util; + +public class SizebufOverflowException extends RuntimeException { + public final String sizebufName; + + public SizebufOverflowException(String sizebufName) { + super(sizebufName + " overflowed"); + this.sizebufName = sizebufName; + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilByteBuffer.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilByteBuffer.java new file mode 100644 index 0000000..aa76d21 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilByteBuffer.java @@ -0,0 +1,79 @@ +package org.rehlds.flightrec.api.util; + +public class UtilByteBuffer { + byte data[]; + + public UtilByteBuffer(byte[] data) { + this.data = data; + } + + public byte[] getData() { + return data; + } + + public UtilByteBuffer cutLeft(int newStart) { + byte[] newData = new byte[data.length - newStart]; + System.arraycopy(data, newStart, newData, 0, data.length - newStart); + return new UtilByteBuffer(newData); + } + + public int getDataLength() { + return data.length; + } + + public int readUInt8(int pos) { + return data[pos] & 0xFF; + } + + public boolean readBool(int pos) { + return data[pos] != 0; + } + + public int readUInt16(int pos) { + return (data[pos] & 0xFF) | ((data[pos + 1] & 0xFF) << 8); + } + + public long readUInt32(int pos) { + return (data[pos] & 0xFF) | ((data[pos + 1] & 0xFF) << 8) | ((data[pos + 2] & 0xFF) << 16) | ((long)(data[pos + 3] & 0xFF) << 24); + } + + public int readInt32(int pos) { + return (data[pos] & 0xFF) | ((data[pos + 1] & 0xFF) << 8) | ((data[pos + 2] & 0xFF) << 16) | ((data[pos + 3] & 0xFF) << 24); + } + + public long readInt64(int pos) { + long lowBits = readUInt32(pos); + long highBits = readUInt32(pos + 4); + + return lowBits | (highBits << 32); + } + + public double readDouble(int pos) { + long bits = readInt64(pos); + return Double.longBitsToDouble(bits); + } + + public float readFloat(int pos) { + int bits = readInt32(pos); + return Float.intBitsToFloat(bits); + } + + public String readString(int pos) { + return readString(pos, data.length - pos, true); + } + + public String readString(int pos, int maxSize, boolean errorOnMaxSizeHit) { + int iMax = Math.min(data.length, pos + maxSize); + for (int i = pos; i < iMax; i++) { + if (data[i] == 0) { + return new String(data, pos, i - pos, Globals.UTF8); + } + } + + if (errorOnMaxSizeHit) { + return null; + } + + return new String(data, pos, iMax - pos, Globals.UTF8); + } +} diff --git a/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilSizeBuf.java b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilSizeBuf.java new file mode 100644 index 0000000..d8e52a7 --- /dev/null +++ b/flightrec/decoder_api/src/main/java/org/rehlds/flightrec/api/util/UtilSizeBuf.java @@ -0,0 +1,159 @@ +package org.rehlds.flightrec.api.util; + +public class UtilSizeBuf { + String name; + UtilByteBuffer buf; + int startPos; + int maxSize; + int curPos; + + public UtilSizeBuf(String name, UtilByteBuffer buf, int startPos, int maxSize) { + this.name = name; + this.buf = buf; + this.startPos = startPos; + this.maxSize = maxSize; + curPos = 0; + } + + public UtilSizeBuf(String name, UtilByteBuffer buf) { + this(name, buf, 0, buf.getDataLength()); + } + + public void init(int startPos, int maxSize) { + this.startPos = startPos; + this.maxSize = maxSize; + curPos = 0; + } + + public void reset() { + this.curPos = 0; + } + + public int tell() { + return curPos; + } + + public int getAbsoluteCurrentPos() { + return curPos + startPos; + } + + public void skip(int count) { + if (curPos + count > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + curPos += count; + } + + public int readUInt8() { + if (curPos + 1 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos++; + return buf.readUInt8(pos + this.startPos); + } + + public boolean readBool() { + if (curPos + 1 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos++; + return buf.readBool(pos + this.startPos); + } + + public int readUInt16() { + if (curPos + 2 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 2; + return buf.readUInt16(pos + this.startPos); + } + + public long readUInt32() { + if (curPos + 4 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 4; + return buf.readUInt32(pos + this.startPos); + } + + public int readInt32() { + if (curPos + 4 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 4; + return buf.readInt32(pos + this.startPos); + } + + public long readInt64() { + if (curPos + 8 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 8; + return buf.readInt64(pos + this.startPos); + } + + public double readDouble() { + if (curPos + 8 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 8; + return buf.readDouble(pos + this.startPos); + } + + public float readFloat() { + if (curPos + 4 > maxSize) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + int pos = curPos; + curPos += 4; + return buf.readFloat(pos + this.startPos); + } + + public String readString() { + String s = buf.readString(curPos + this.startPos); + if (s == null) { + curPos = maxSize; + throw new SizebufOverflowException(name); + } + + curPos += s.getBytes(Globals.UTF8).length + 1; + return s; + } + + public int getStartPos() { + return startPos; + } + + public int getMaxSize() { + return maxSize; + } + + public byte[] getBuffer() { + return buf.getData(); + } +} diff --git a/flightrec/decoder_api/src/test/java/org/rehlds/flightrec/util/UtilByteBufferTest.java b/flightrec/decoder_api/src/test/java/org/rehlds/flightrec/util/UtilByteBufferTest.java new file mode 100644 index 0000000..22ed565 --- /dev/null +++ b/flightrec/decoder_api/src/test/java/org/rehlds/flightrec/util/UtilByteBufferTest.java @@ -0,0 +1,127 @@ +package org.rehlds.flightrec.util; + +import org.junit.Test; +import org.rehlds.flightrec.api.util.UtilByteBuffer; + +import static org.junit.Assert.*; + +public class UtilByteBufferTest { + + @Test + public void testReadUInt8() throws Exception { + byte data[] = { 0x10, 0x00, 0x7F, (byte)0x80, (byte)0xFF }; + UtilByteBuffer bb = new UtilByteBuffer(data); + + assertEquals(0x10, bb.readUInt8(0)); + assertEquals(0x00, bb.readUInt8(1)); + assertEquals(0x7F, bb.readUInt8(2)); + assertEquals(0x80, bb.readUInt8(3)); + assertEquals(0xFF, bb.readUInt8(4)); + } + + @Test + public void testReadUInt16() throws Exception { + byte data[] = { 0x10, 0x00, 0x7F, (byte)0x80, (byte)0xFF, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + + assertEquals(0x10, bb.readUInt16(0)); + assertEquals(0x7F00, bb.readUInt16(1)); + assertEquals(0x807F, bb.readUInt16(2)); + assertEquals(0xFF80, bb.readUInt16(3)); + assertEquals(0x00FF, bb.readUInt16(4)); + } + + @Test + public void testReadUInt32() throws Exception { + byte data[] = { 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, (byte)0xFF, 0x00, 0x00, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + assertEquals(0x10000000, bb.readUInt32(0)); + assertEquals(0x100000, bb.readUInt32(1)); + assertEquals(0x1000, bb.readUInt32(2)); + assertEquals(0x10, bb.readUInt32(3)); + assertEquals(0x0, bb.readUInt32(4)); + assertEquals(0x7F000000, bb.readUInt32(5)); + assertEquals(0x7F0000, bb.readUInt32(6)); + assertEquals(0x7F00, bb.readUInt32(7)); + assertEquals(0x7F, bb.readUInt32(8)); + assertEquals(0xFF000000L, bb.readUInt32(9)); + assertEquals(0xFF0000, bb.readUInt32(10)); + assertEquals(0xFF00, bb.readUInt32(11)); + assertEquals(0xFF, bb.readUInt32(12)); + } + + @Test + public void testReadInt32() throws Exception { + byte data[] = { 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, (byte)0xFF, 0x00, 0x00, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + assertEquals(0x10000000, bb.readInt32(0)); + assertEquals(0x100000, bb.readInt32(1)); + assertEquals(0x1000, bb.readInt32(2)); + assertEquals(0x10, bb.readInt32(3)); + + assertEquals(0x0, bb.readInt32(4)); + + assertEquals(0x7F000000, bb.readInt32(5)); + assertEquals(0x7F0000, bb.readInt32(6)); + assertEquals(0x7F00, bb.readInt32(7)); + assertEquals(0x7F, bb.readInt32(8)); + + assertEquals((int)0xFF000000, bb.readInt32(9)); + assertEquals(0xFF0000, bb.readInt32(10)); + assertEquals(0xFF00, bb.readInt32(11)); + assertEquals(0xFF, bb.readInt32(12)); + } + + @Test + public void testReadInt64() throws Exception { + byte data[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte)0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + assertEquals(0x1000000000000000L, bb.readInt64(0)); + assertEquals(0x10000000000000L, bb.readInt64(1)); + assertEquals(0x100000000000L, bb.readInt64(2)); + assertEquals(0x1000000000L, bb.readInt64(3)); + assertEquals(0x10000000L, bb.readInt64(4)); + assertEquals(0x100000L, bb.readInt64(5)); + assertEquals(0x1000L, bb.readInt64(6)); + assertEquals(0x10L, bb.readInt64(7)); + + assertEquals(0x00L, bb.readInt64(8)); + + assertEquals(0x7F00000000000000L, bb.readInt64(9)); + assertEquals(0x7F000000000000L, bb.readInt64(10)); + assertEquals(0x7F0000000000L, bb.readInt64(11)); + assertEquals(0x7F00000000L, bb.readInt64(12)); + assertEquals(0x7F000000L, bb.readInt64(13)); + assertEquals(0x7F0000L, bb.readInt64(14)); + assertEquals(0x7F00L, bb.readInt64(15)); + assertEquals(0x7FL, bb.readInt64(16)); + + assertEquals(0xFF00000000000000L, bb.readInt64(17)); + assertEquals(0xFF000000000000L, bb.readInt64(18)); + assertEquals(0xFF0000000000L, bb.readInt64(19)); + assertEquals(0xFF00000000L, bb.readInt64(20)); + assertEquals(0xFF000000L, bb.readInt64(21)); + assertEquals(0xFF0000L, bb.readInt64(22)); + assertEquals(0xFF00L, bb.readInt64(23)); + assertEquals(0xFFL, bb.readInt64(24)); + } + + @Test + public void testReadString() throws Exception { + byte data[] = { 0x00, 0x41, 0x00, 0x41, 0x42, 0x00, 0x50, 0x75, 0x62, 0x6C, 0x69, 0x63, 0x69, 0x74, (byte)0xC3, (byte)0xA9, 0x00, 0x41, 0x42 }; + UtilByteBuffer bb = new UtilByteBuffer(data); + + assertEquals("", bb.readString(0)); + assertEquals("A", bb.readString(1)); + assertEquals("", bb.readString(2)); + assertEquals("AB", bb.readString(3)); + assertEquals("Publicité", bb.readString(6)); + assertNull(bb.readString(17)); + + assertEquals("Public", bb.readString(6, 6, false)); + assertNull(bb.readString(6, 6, true)); + + assertEquals("AB", bb.readString(17, 2, false)); + assertNull(bb.readString(17, 2, true)); + } +} \ No newline at end of file diff --git a/publish.gradle b/publish.gradle index 7b6fbd3..b40caa3 100644 --- a/publish.gradle +++ b/publish.gradle @@ -10,50 +10,69 @@ void _copyFile(String from, String to) { GradleCppUtils.copyFile(project.file(from), project.file(to), false) } -task publishPrepareFiles << { - def pubRootDir = project.file('publish/publishRoot') - if (pubRootDir.exists()) { - if (!pubRootDir.deleteDir()) { - throw new RuntimeException("Failed to delete ${pubRootDir}") +task publishPrepareFiles { + dependsOn ':flightrec/decoder:uberjar' + doLast { + def pubRootDir = project.file('publish/publishRoot') + if (pubRootDir.exists()) { + if (!pubRootDir.deleteDir()) { + throw new RuntimeException("Failed to delete ${pubRootDir}") + } } - } - pubRootDir.mkdirs() + pubRootDir.mkdirs() - //bugfixed binaries - project.file('publish/publishRoot/bin/bugfixed').mkdirs() - _copyFileToDir('publish/releaseRehldsFixes/swds.dll', 'publish/publishRoot/bin/bugfixed/') - _copyFileToDir('publish/releaseRehldsFixes/swds.pdb', 'publish/publishRoot/bin/bugfixed/') - _copyFile('publish/releaseRehldsFixes/libengine_i486.so', 'publish/publishRoot/bin/bugfixed/engine_i486.so') + //bugfixed binaries + project.file('publish/publishRoot/bin/bugfixed').mkdirs() + _copyFileToDir('publish/releaseRehldsFixes/swds.dll', 'publish/publishRoot/bin/bugfixed/') + _copyFileToDir('publish/releaseRehldsFixes/swds.pdb', 'publish/publishRoot/bin/bugfixed/') + _copyFile('publish/releaseRehldsFixes/libengine_i486.so', 'publish/publishRoot/bin/bugfixed/engine_i486.so') - //pure binaries - project.file('publish/publishRoot/bin/pure').mkdirs() - _copyFileToDir('publish/releaseRehldsNofixes/swds.dll', 'publish/publishRoot/bin/pure/') - _copyFileToDir('publish/releaseRehldsNofixes/swds.pdb', 'publish/publishRoot/bin/pure/') - _copyFile('publish/releaseRehldsNofixes/libengine_i486.so', 'publish/publishRoot/bin/pure/engine_i486.so') + //pure binaries + project.file('publish/publishRoot/bin/pure').mkdirs() + _copyFileToDir('publish/releaseRehldsNofixes/swds.dll', 'publish/publishRoot/bin/pure/') + _copyFileToDir('publish/releaseRehldsNofixes/swds.pdb', 'publish/publishRoot/bin/pure/') + _copyFile('publish/releaseRehldsNofixes/libengine_i486.so', 'publish/publishRoot/bin/pure/engine_i486.so') - //hlsdk - project.file('publish/publishRoot/hlsdk').mkdirs() - copy { - from 'rehlds/common' - into 'publish/publishRoot/hlsdk/common' - } - copy { - from 'rehlds/dlls' - into 'publish/publishRoot/hlsdk/dlls' - } - copy { - from 'rehlds/pm_shared' - into 'publish/publishRoot/hlsdk/pm_shared' - } - copy { - from 'rehlds/public' - into 'publish/publishRoot/hlsdk/public' - include 'interface.h', 'interface.cpp', 'FileSystem.h' - } - copy { - from 'rehlds/public/rehlds' - into 'publish/publishRoot/hlsdk/engine' + //hlsdk + project.file('publish/publishRoot/hlsdk').mkdirs() + copy { + from 'rehlds/common' + into 'publish/publishRoot/hlsdk/common' + } + copy { + from 'rehlds/dlls' + into 'publish/publishRoot/hlsdk/dlls' + } + copy { + from 'rehlds/pm_shared' + into 'publish/publishRoot/hlsdk/pm_shared' + } + copy { + from 'rehlds/public' + into 'publish/publishRoot/hlsdk/public' + include 'interface.h', 'interface.cpp', 'FileSystem.h' + } + copy { + from 'rehlds/public/rehlds' + into 'publish/publishRoot/hlsdk/engine' + } + + //flightrecorder + + def flightRecJarTask = project(':flightrec/decoder').tasks.getByName('uberjar') + println flightRecJarTask + println flightRecJarTask.class.name + File flightRecJarFile = flightRecJarTask.archivePath + project.file('publish/publishRoot/flighrec').mkdirs() + copy { + from flightRecJarFile + into 'publish/publishRoot/flighrec' + } + copy { + from new File(project(':flightrec/decoder').projectDir, 'pub') + into 'publish/publishRoot/flighrec' + } } } diff --git a/rehlds/engine/host.cpp b/rehlds/engine/host.cpp index f175e6b..30aa804 100644 --- a/rehlds/engine/host.cpp +++ b/rehlds/engine/host.cpp @@ -883,6 +883,8 @@ void Host_CheckConnectionFailure(void) /* <364fd> ../engine/host.c:1350 */ void _Host_Frame(float time) { + + static double host_times[6]; if (setjmp(host_enddemo)) return; @@ -892,8 +894,9 @@ void _Host_Frame(float time) return; #ifdef REHLDS_FLIGHT_REC + static long frameCounter = 0; if (rehlds_flrec_frame.string[0] != '0') { - FR_StartFrame(); + FR_StartFrame(frameCounter); } #endif //REHLDS_FLIGHT_REC @@ -957,8 +960,9 @@ void _Host_Frame(float time) #ifdef REHLDS_FLIGHT_REC if (rehlds_flrec_frame.string[0] != '0') { - FR_EndFrame(); + FR_EndFrame(frameCounter); } + frameCounter++; #endif //REHLDS_FLIGHT_REC } diff --git a/rehlds/engine/pr_edict.cpp b/rehlds/engine/pr_edict.cpp index 295aa38..c0976c5 100644 --- a/rehlds/engine/pr_edict.cpp +++ b/rehlds/engine/pr_edict.cpp @@ -454,7 +454,7 @@ void *PvAllocEntPrivateData(edict_t *pEdict, int32 cb) #ifdef REHLDS_FLIGHT_REC if (rehlds_flrec_pvdata.string[0] != '0') { - FR_AllocEntPrivateData(pEdict->pvPrivateData); + FR_AllocEntPrivateData(pEdict->pvPrivateData, cb); } #endif //REHLDS_FLIGHT_REC diff --git a/rehlds/public/rehlds/crc32c.cpp b/rehlds/public/rehlds/crc32c.cpp new file mode 100644 index 0000000..6dd7758 --- /dev/null +++ b/rehlds/public/rehlds/crc32c.cpp @@ -0,0 +1,143 @@ +/* +Copyright (C) 2010 by Ronnie Sahlberg +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License +along with this program; if not, see . +*/ + +#include "crc32c.h" +#include "sys_shared.h" +#include "immintrin.h" + +/*****************************************************************/ +/* */ +/* CRC LOOKUP TABLE */ +/* ================ */ +/* The following CRC lookup table was generated automagically */ +/* by the Rocksoft^tm Model CRC Algorithm Table Generation */ +/* Program V1.0 using the following model parameters: */ +/* */ +/* Width : 4 bytes. */ +/* Poly : 0x1EDC6F41L */ +/* Reverse : TRUE. */ +/* */ +/* For more information on the Rocksoft^tm Model CRC Algorithm, */ +/* see the document titled "A Painless Guide to CRC Error */ +/* Detection Algorithms" by Ross Williams */ +/* (ross@guest.adelaide.edu.au.). This document is likely to be */ +/* in the FTP archive "ftp.adelaide.edu.au/pub/rocksoft". */ +/* */ +/*****************************************************************/ + +static uint32 crctable[256] = { + 0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L, + 0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL, + 0x8AD958CFL, 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL, + 0x4D43CFD0L, 0xBF284CD3L, 0xAC78BF27L, 0x5E133C24L, + 0x105EC76FL, 0xE235446CL, 0xF165B798L, 0x030E349BL, + 0xD7C45070L, 0x25AFD373L, 0x36FF2087L, 0xC494A384L, + 0x9A879FA0L, 0x68EC1CA3L, 0x7BBCEF57L, 0x89D76C54L, + 0x5D1D08BFL, 0xAF768BBCL, 0xBC267848L, 0x4E4DFB4BL, + 0x20BD8EDEL, 0xD2D60DDDL, 0xC186FE29L, 0x33ED7D2AL, + 0xE72719C1L, 0x154C9AC2L, 0x061C6936L, 0xF477EA35L, + 0xAA64D611L, 0x580F5512L, 0x4B5FA6E6L, 0xB93425E5L, + 0x6DFE410EL, 0x9F95C20DL, 0x8CC531F9L, 0x7EAEB2FAL, + 0x30E349B1L, 0xC288CAB2L, 0xD1D83946L, 0x23B3BA45L, + 0xF779DEAEL, 0x05125DADL, 0x1642AE59L, 0xE4292D5AL, + 0xBA3A117EL, 0x4851927DL, 0x5B016189L, 0xA96AE28AL, + 0x7DA08661L, 0x8FCB0562L, 0x9C9BF696L, 0x6EF07595L, + 0x417B1DBCL, 0xB3109EBFL, 0xA0406D4BL, 0x522BEE48L, + 0x86E18AA3L, 0x748A09A0L, 0x67DAFA54L, 0x95B17957L, + 0xCBA24573L, 0x39C9C670L, 0x2A993584L, 0xD8F2B687L, + 0x0C38D26CL, 0xFE53516FL, 0xED03A29BL, 0x1F682198L, + 0x5125DAD3L, 0xA34E59D0L, 0xB01EAA24L, 0x42752927L, + 0x96BF4DCCL, 0x64D4CECFL, 0x77843D3BL, 0x85EFBE38L, + 0xDBFC821CL, 0x2997011FL, 0x3AC7F2EBL, 0xC8AC71E8L, + 0x1C661503L, 0xEE0D9600L, 0xFD5D65F4L, 0x0F36E6F7L, + 0x61C69362L, 0x93AD1061L, 0x80FDE395L, 0x72966096L, + 0xA65C047DL, 0x5437877EL, 0x4767748AL, 0xB50CF789L, + 0xEB1FCBADL, 0x197448AEL, 0x0A24BB5AL, 0xF84F3859L, + 0x2C855CB2L, 0xDEEEDFB1L, 0xCDBE2C45L, 0x3FD5AF46L, + 0x7198540DL, 0x83F3D70EL, 0x90A324FAL, 0x62C8A7F9L, + 0xB602C312L, 0x44694011L, 0x5739B3E5L, 0xA55230E6L, + 0xFB410CC2L, 0x092A8FC1L, 0x1A7A7C35L, 0xE811FF36L, + 0x3CDB9BDDL, 0xCEB018DEL, 0xDDE0EB2AL, 0x2F8B6829L, + 0x82F63B78L, 0x709DB87BL, 0x63CD4B8FL, 0x91A6C88CL, + 0x456CAC67L, 0xB7072F64L, 0xA457DC90L, 0x563C5F93L, + 0x082F63B7L, 0xFA44E0B4L, 0xE9141340L, 0x1B7F9043L, + 0xCFB5F4A8L, 0x3DDE77ABL, 0x2E8E845FL, 0xDCE5075CL, + 0x92A8FC17L, 0x60C37F14L, 0x73938CE0L, 0x81F80FE3L, + 0x55326B08L, 0xA759E80BL, 0xB4091BFFL, 0x466298FCL, + 0x1871A4D8L, 0xEA1A27DBL, 0xF94AD42FL, 0x0B21572CL, + 0xDFEB33C7L, 0x2D80B0C4L, 0x3ED04330L, 0xCCBBC033L, + 0xA24BB5A6L, 0x502036A5L, 0x4370C551L, 0xB11B4652L, + 0x65D122B9L, 0x97BAA1BAL, 0x84EA524EL, 0x7681D14DL, + 0x2892ED69L, 0xDAF96E6AL, 0xC9A99D9EL, 0x3BC21E9DL, + 0xEF087A76L, 0x1D63F975L, 0x0E330A81L, 0xFC588982L, + 0xB21572C9L, 0x407EF1CAL, 0x532E023EL, 0xA145813DL, + 0x758FE5D6L, 0x87E466D5L, 0x94B49521L, 0x66DF1622L, + 0x38CC2A06L, 0xCAA7A905L, 0xD9F75AF1L, 0x2B9CD9F2L, + 0xFF56BD19L, 0x0D3D3E1AL, 0x1E6DCDEEL, 0xEC064EEDL, + 0xC38D26C4L, 0x31E6A5C7L, 0x22B65633L, 0xD0DDD530L, + 0x0417B1DBL, 0xF67C32D8L, 0xE52CC12CL, 0x1747422FL, + 0x49547E0BL, 0xBB3FFD08L, 0xA86F0EFCL, 0x5A048DFFL, + 0x8ECEE914L, 0x7CA56A17L, 0x6FF599E3L, 0x9D9E1AE0L, + 0xD3D3E1ABL, 0x21B862A8L, 0x32E8915CL, 0xC083125FL, + 0x144976B4L, 0xE622F5B7L, 0xF5720643L, 0x07198540L, + 0x590AB964L, 0xAB613A67L, 0xB831C993L, 0x4A5A4A90L, + 0x9E902E7BL, 0x6CFBAD78L, 0x7FAB5E8CL, 0x8DC0DD8FL, + 0xE330A81AL, 0x115B2B19L, 0x020BD8EDL, 0xF0605BEEL, + 0x24AA3F05L, 0xD6C1BC06L, 0xC5914FF2L, 0x37FACCF1L, + 0x69E9F0D5L, 0x9B8273D6L, 0x88D28022L, 0x7AB90321L, + 0xAE7367CAL, 0x5C18E4C9L, 0x4F48173DL, 0xBD23943EL, + 0xF36E6F75L, 0x0105EC76L, 0x12551F82L, 0xE03E9C81L, + 0x34F4F86AL, 0xC69F7B69L, 0xD5CF889DL, 0x27A40B9EL, + 0x79B737BAL, 0x8BDCB4B9L, 0x988C474DL, 0x6AE7C44EL, + 0xBE2DA0A5L, 0x4C4623A6L, 0x5F16D052L, 0xAD7D5351L +}; + +uint32 crc32c_t8_nosse(uint32 iCRC, uint8 u8) { + return (iCRC >> 8) ^ crctable[(iCRC ^ u8) & 0xFF]; +} + +uint32 crc32c_t_nosse(uint32 iCRC, const uint8 *buf, int len) { + uint32 crc = iCRC; + while (len-- > 0) { + crc = (crc >> 8) ^ crctable[(crc ^ (*buf++)) & 0xFF]; + } + return crc; +} + +uint32 crc32c_t8_sse(uint32 iCRC, uint8 u8) { + return _mm_crc32_u8(iCRC, u8); +} + +uint32 crc32c_t_sse(uint32 iCRC, const uint8 *buf, unsigned int len) { + uint32 crc32cval = iCRC; + unsigned int i = 0; + + for (; i < (len >> 2); i += 4) { + crc32cval = _mm_crc32_u32(crc32cval, *(uint32*)&buf[i]); + } + + for (; i < len; i++) { + crc32cval = _mm_crc32_u8(crc32cval, buf[i]); + } + + return crc32cval; +} + +uint32 crc32c_t(uint32 iCRC, const uint8 *buf, unsigned int len) { + return cpuinfo.sse4_2 ? crc32c_t_nosse(iCRC, buf, len) : crc32c_t_nosse(iCRC, buf, len); +} + +uint32 crc32c(const uint8 *buf, int len) { + return crc32c_t(0xffffffff, buf, len); +} diff --git a/rehlds/public/rehlds/crc32c.h b/rehlds/public/rehlds/crc32c.h new file mode 100644 index 0000000..d3a78a3 --- /dev/null +++ b/rehlds/public/rehlds/crc32c.h @@ -0,0 +1,22 @@ +/* +Copyright (C) 2010 by Ronnie Sahlberg +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License +along with this program; if not, see . +*/ +#pragma once +#include "archtypes.h" + +extern uint32 crc32c_t8_nosse(uint32 iCRC, uint8 u8); +extern uint32 crc32c_t8_sse(uint32 iCRC, uint8 u8); +extern uint32 crc32c_t_nosse(uint32 iCRC, const uint8 *buf, int len); +extern uint32 crc32c_t_sse(uint32 iCRC, const uint8 *buf, unsigned int len); +extern uint32 crc32c_t(uint32 iCRC, const uint8 *buf, unsigned int len); +extern uint32 crc32c(const uint8 *buf, int len); diff --git a/rehlds/rehlds/FlightRecorderImpl.cpp b/rehlds/rehlds/FlightRecorderImpl.cpp index 1801529..2c8cca4 100644 --- a/rehlds/rehlds/FlightRecorderImpl.cpp +++ b/rehlds/rehlds/FlightRecorderImpl.cpp @@ -59,15 +59,19 @@ CRehldsFlightRecorder::CRehldsFlightRecorder() { void CRehldsFlightRecorder::InitHeadersContent() { m_pMetaHeader->version = FLIGHT_RECORDER_VERSION; + m_pMetaHeader->regionSize = META_REGION_SIZE; m_pMetaHeader->metaRegionPos = 0; m_pMetaHeader->numMessages = 0; + m_pMetaHeader->headerCrc32 = crc32c_t(0, m_MetaRegion, ((uint8*)m_pMetaHeader - m_MetaRegion) + offsetof(meta_header, headerCrc32)); m_pRecorderState->wpos = 0; m_pRecorderState->lastMsgBeginPos = 0xFFFFFFFF; m_pRecorderState->curMessage = 0; m_pDataHeader->version = FLIGHT_RECORDER_VERSION; + m_pDataHeader->regionSize = DATA_REGION_SIZE; m_pDataHeader->prevItrLastPos = 0xFFFFFFFF; + m_pDataHeader->headerCrc32 = crc32c_t(0, m_DataRegion, ((uint8*)m_pDataHeader - m_DataRegion) + offsetof(data_header, headerCrc32)); } void CRehldsFlightRecorder::MoveToStart() { @@ -206,10 +210,11 @@ uint16 CRehldsFlightRecorder::RegisterMessage(const char* module, const char *me sizebuf_t sb; sb.buffername = "FlightRecorded Meta"; sb.cursize = m_pMetaHeader->metaRegionPos; - sb.maxsize = META_REGION_MAIN_SIZE; + sb.maxsize = META_REGION_MAIN_SIZE - META_REGION_HEADER; sb.flags = 0; sb.data = m_MetaRegionPtr; + MSG_WriteByte(&sb, MRT_MESSAGE_DEF); MSG_WriteShort(&sb, msgId); MSG_WriteString(&sb, module); MSG_WriteString(&sb, message); diff --git a/rehlds/rehlds/FlightRecorderImpl.h b/rehlds/rehlds/FlightRecorderImpl.h index 140f2bb..b503030 100644 --- a/rehlds/rehlds/FlightRecorderImpl.h +++ b/rehlds/rehlds/FlightRecorderImpl.h @@ -36,6 +36,11 @@ public: private: + enum meta_record_type_t { + MRT_MESSAGE_DEF = 1, + MRT_SHUNT = 2, + }; + #pragma pack(push, 1) struct recorder_state { unsigned int wpos; diff --git a/rehlds/rehlds/flight_recorder.cpp b/rehlds/rehlds/flight_recorder.cpp index 23ac747..4a12fff 100644 --- a/rehlds/rehlds/flight_recorder.cpp +++ b/rehlds/rehlds/flight_recorder.cpp @@ -54,28 +54,31 @@ void FR_Rehlds_Init() { Cvar_RegisterVariable(&rehlds_flrec_pvdata); Cmd_AddCommand("rehlds_flrec_dump", &FR_Dump_f); - g_FRMsg_Frame = g_FlightRecorder->RegisterMessage("rehlds", "Frame", 1, true); + g_FRMsg_Frame = g_FlightRecorder->RegisterMessage("rehlds", "Frame", 2, true); g_FRMsg_FreeEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "FreeEntPrivateData", 1, false); - g_FRMsg_AllocEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "AllocEntPrivateData", 1, false); + g_FRMsg_AllocEntPrivateData = g_FlightRecorder->RegisterMessage("rehlds", "AllocEntPrivateData", 2, false); } -void FR_StartFrame() { +void FR_StartFrame(long frameCounter) { FR_CheckInit(); g_FlightRecorder->StartMessage(g_FRMsg_Frame, true); + g_FlightRecorder->WriteInt64(frameCounter); g_FlightRecorder->WriteDouble(realtime); g_FlightRecorder->EndMessage(g_FRMsg_Frame, true); } -void FR_EndFrame() { +void FR_EndFrame(long frameCounter) { FR_CheckInit(); g_FlightRecorder->StartMessage(g_FRMsg_Frame, false); + g_FlightRecorder->WriteInt64(frameCounter); g_FlightRecorder->EndMessage(g_FRMsg_Frame, false); } -void FR_AllocEntPrivateData(void* res) { +void FR_AllocEntPrivateData(void* res, int size) { FR_CheckInit(); g_FlightRecorder->StartMessage(g_FRMsg_AllocEntPrivateData, true); g_FlightRecorder->WriteUInt32((size_t)res); + g_FlightRecorder->WriteInt32(size); g_FlightRecorder->EndMessage(g_FRMsg_AllocEntPrivateData, true); } diff --git a/rehlds/rehlds/flight_recorder.h b/rehlds/rehlds/flight_recorder.h index e7ff49e..9b80f43 100644 --- a/rehlds/rehlds/flight_recorder.h +++ b/rehlds/rehlds/flight_recorder.h @@ -36,9 +36,9 @@ extern void FR_Dump_f(); extern void FR_Rehlds_Init(); -extern void FR_StartFrame(); -extern void FR_EndFrame(); +extern void FR_StartFrame(long frameCounter); +extern void FR_EndFrame(long frameCounter); extern void FR_FreeEntPrivateData(void* data); -extern void FR_AllocEntPrivateData(void* res); +extern void FR_AllocEntPrivateData(void* res, int size); #endif //REHLDS_FLIGHT_REC diff --git a/rehlds/unittests/crc32c_tests.cpp b/rehlds/unittests/crc32c_tests.cpp new file mode 100644 index 0000000..5f5f575 --- /dev/null +++ b/rehlds/unittests/crc32c_tests.cpp @@ -0,0 +1,40 @@ +#include "precompiled.h" +#include "cppunitlite/TestHarness.h" + +TEST(CRC32C_Hash, CRC32C, 1000) { + Sys_CheckCpuInstructionsSupport(); + CHECK("SSE4.2 Support", cpuinfo.sse4_2); + + struct testdata_t { + const char* src; + uint32 hash0; + uint32 hashFF; + }; + + testdata_t testdata[] = { + { "a", 0x93AD1061, 0x3E2FBCCF }, + { "ab", 0x13C35EE4, 0x1D5DD6C9 }, + { "abc", 0x562F9CCD, 0xC9B4C048 }, + { "abcd", 0xDAAF41F6, 0x6D37F5CE }, + { "abcde", 0x8122A0A2, 0x3BAF2968 }, + { "abcdef", 0x0496937B, 0xAC43100E }, + { "abcdefg", 0x5D199E2C, 0x19D80BBE }, + { "abcdefgh", 0x86BC933D, 0xF56BDE48 }, + { "abcdefghi", 0x9639F15F, 0xD2236603 }, + { "abcdefghij", 0x0584645C, 0x19A66BC8 }, + }; + + for (int i = 0; i < ARRAYSIZE(testdata); i++) { + uint32 pureCksum0 = crc32c_t_nosse(0, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + uint32 sseCksum0 = crc32c_t_sse(0, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + + uint32 pureCksumFF = crc32c_t_nosse(-1, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + uint32 sseCksumFF = crc32c_t_sse(-1, (const uint8*)testdata[i].src, strlen(testdata[i].src)); + + UINT32_EQUALS("Pure crc32c checksum-0 mismatch", testdata[i].hash0, pureCksum0); + UINT32_EQUALS("SSE crc32c checksum-0 mismatch", testdata[i].hash0, sseCksum0); + + UINT32_EQUALS("Pure crc32c checksum-FF mismatch", testdata[i].hashFF, pureCksumFF); + UINT32_EQUALS("SSE crc32c checksum-FF mismatch", testdata[i].hashFF, sseCksumFF); + } +} \ No newline at end of file