Fixed #24: Remember fields unmarked by encoder and force set it if entity is used as a baseline

rehlds/engine/delta.cpp

@@ -735,7 +735,7 @@ qboolean DELTA_CheckDelta(unsigned char *from, unsigned char *to, delta_t *pFiel
 	qboolean sendfields;
 
 #if defined(REHLDS_OPT_PEDANTIC) || defined(REHLDS_FIXES)
-	sendfields = DELTAJit_Fields_Clear_Mark_Check(from, to, pFields);
+	sendfields = DELTAJit_Fields_Clear_Mark_Check(from, to, pFields, NULL);
 #else
 	DELTA_ClearFlags(pFields);
 	DELTA_MarkSendFields(from, to, pFields);
@@ -752,7 +752,7 @@ NOINLINE qboolean DELTA_WriteDelta(unsigned char *from, unsigned char *to, qbool
 	int bytecount;
 
 #if defined(REHLDS_OPT_PEDANTIC) || defined(REHLDS_FIXES)
-	sendfields = DELTAJit_Fields_Clear_Mark_Check(from, to, pFields);
+	sendfields = DELTAJit_Fields_Clear_Mark_Check(from, to, pFields, NULL);
 #else // REHLDS_OPT_PEDANTIC || REHLDS_FIXES
 	DELTA_ClearFlags(pFields);
 	DELTA_MarkSendFields(from, to, pFields);
@@ -763,6 +763,15 @@ NOINLINE qboolean DELTA_WriteDelta(unsigned char *from, unsigned char *to, qbool
 	return sendfields;
 }
 
+#ifdef REHLDS_FIXES //Fix for https://github.com/dreamstalker/rehlds/issues/24
+qboolean DELTA_WriteDeltaForceMask(unsigned char *from, unsigned char *to, qboolean force, delta_t *pFields, void(*callback)(void), void* pForceMask) {
+	qboolean sendfields = DELTAJit_Fields_Clear_Mark_Check(from, to, pFields, pForceMask);
+	_DELTA_WriteDelta(from, to, force, pFields, callback, sendfields);
+	return sendfields;
+}
+#endif
+
+
 qboolean _DELTA_WriteDelta(unsigned char *from, unsigned char *to, qboolean force, delta_t *pFields, void(*callback)( void ), qboolean sendfields)
 {
 	int i;

rehlds/engine/delta.h

@@ -139,6 +139,11 @@ void DELTA_SetSendFlagBits(delta_t *pFields, int *bits, int *bytecount);
 qboolean DELTA_IsFieldMarked(delta_t* pFields, int fieldNumber);
 void DELTA_WriteMarkedFields(unsigned char *from, unsigned char *to, delta_t *pFields);
 qboolean DELTA_CheckDelta(unsigned char *from, unsigned char *to, delta_t *pFields);
+
+#ifdef REHLDS_FIXES //Fix for https://github.com/dreamstalker/rehlds/issues/24
+qboolean DELTA_WriteDeltaForceMask(unsigned char *from, unsigned char *to, qboolean force, delta_t *pFields, void(*callback)(void), void* pForceMask);
+#endif
+
 qboolean DELTA_WriteDelta(unsigned char *from, unsigned char *to, qboolean force, delta_t *pFields, void(*callback)(void));
 qboolean _DELTA_WriteDelta(unsigned char *from, unsigned char *to, qboolean force, delta_t *pFields, void(*callback)(void), qboolean sendfields);
 int DELTA_ParseDelta(unsigned char *from, unsigned char *to, delta_t *pFields);

rehlds/engine/delta_jit.cpp

@@ -136,14 +136,17 @@ public:
 	int markedFieldsMaskSize;
 
 	delta_marked_mask_t marked_fields_mask;
-	int sse_highbits[2]; //High 64 bits for manipulating marked_fields_mask via SSE registers 
+	int mfm_sse_highbits[2]; //High 64 bits for manipulating marked_fields_mask via SSE registers 
+
+	delta_marked_mask_t originalMarkedFieldsMask; //mask based on data, before calling the conditional encoder
+	int omfm_sse_highbits[2]; //High 64 bits for manipulating marked_fields_mask via SSE registers 
 
 	CDeltaJit(delta_t* _delta, CDeltaClearMarkFieldsJIT* _cleanMarkCheckFunc);
 
 	virtual ~CDeltaJit();
 };
 
-class CDeltaClearMarkFieldsJIT : public jitasm::function<int, CDeltaClearMarkFieldsJIT, void*, void*, void*> {
+class CDeltaClearMarkFieldsJIT : public jitasm::function<int, CDeltaClearMarkFieldsJIT, void*, void*, void*, void*> {
 public:
 	deltajitdata_t *jitdesc;
 	deltajit_marked_count_type_t countType;
@@ -156,7 +159,7 @@ public:
 	}
 
 	void checkFieldMask(jitasm::Frontend::Reg32& mask, deltajit_memblock_field* jitField);
-	Result main(Addr src, Addr dst, Addr deltaJit);
+	Result main(Addr src, Addr dst, Addr deltaJit, Addr pForceMarkMask);
 	void processStrings(Addr src, Addr dst);
 	void callConditionalEncoder(Addr src, Addr dst, Addr deltaJit);
 	void calculateBytecount();
@@ -257,7 +260,7 @@ void CDeltaClearMarkFieldsJIT::calculateBytecount() {
 	mov(dword_ptr[ebx + delta_masksize_offset], edx);
 }
 
-CDeltaClearMarkFieldsJIT::Result CDeltaClearMarkFieldsJIT::main(Addr src, Addr dst, Addr deltaJit)
+CDeltaClearMarkFieldsJIT::Result CDeltaClearMarkFieldsJIT::main(Addr src, Addr dst, Addr deltaJit, Addr pForceMarkMask)
 {
 #ifndef REHLDS_FIXES
 	sub(esp, 12); //some local storage is required for precise DT_TIMEWINDOW marking
@@ -369,12 +372,28 @@ CDeltaClearMarkFieldsJIT::Result CDeltaClearMarkFieldsJIT::main(Addr src, Addr d
 		}
 	}
 
+	//apply 'force mark' mask if it's present
+	mov(eax, ptr[pForceMarkMask]);
+	If(eax != 0);
+		//mask for cleaning garbage in high 64 bits
+		mov(edx, -1);
+		movd(xmm0, edx);
+		movd(xmm1, edx);
+		psllq(xmm0, 32);
+		por(xmm0, xmm1);
+		
+		movdqu(xmm_tmp, xmmword_ptr[eax]);
+		pand(xmm_tmp, xmm0); //clean high 64 bits
+
+		por(marked_fields_mask, xmm_tmp); //apply the 'force' mask
+	EndIf();
+
 	size_t delta_markbits_offset = offsetof(CDeltaJit, marked_fields_mask);
 
-	//Before calling the condition encoder we have to save 'marked fields' mask 
+	//Save mask from SSE register to CDeltaJit::marked_fields_mask and CDeltaJit::originalMarkedFieldsMask
-	//from SSE register into the CDeltaJit::marked_fields_mask, because conditional encoder can modify the mask
 	mov(ebx, ptr[deltaJit]);
 	movdqu(xmmword_ptr[ebx + delta_markbits_offset], marked_fields_mask);
+	movdqu(xmmword_ptr[ebx + offsetof(CDeltaJit, originalMarkedFieldsMask)], marked_fields_mask);
 
 	processStrings(src, dst);
 
@@ -490,10 +509,10 @@ CDeltaJit* DELTAJit_LookupDeltaJit(const char* callsite, delta_t *pFields) {
 	return deltaJit;
 }
 
-NOINLINE int DELTAJit_Fields_Clear_Mark_Check(unsigned char *from, unsigned char *to, delta_t *pFields) {
+NOINLINE int DELTAJit_Fields_Clear_Mark_Check(unsigned char *from, unsigned char *to, delta_t *pFields, void* pForceMarkMask) {
 	CDeltaJit* deltaJit = DELTAJit_LookupDeltaJit(__FUNCTION__, pFields);
 	CDeltaClearMarkFieldsJIT &func = *deltaJit->cleanMarkCheckFunc;
-	return func(from, to, deltaJit);
+	return func(from, to, deltaJit, pForceMarkMask);
 }
 
 void DELTAJit_SetSendFlagBits(delta_t *pFields, int *bits, int *bytecount) {
@@ -532,6 +551,16 @@ qboolean DELTAJit_IsFieldMarked(delta_t* pFields, int fieldNumber) {
 	return deltaJit->marked_fields_mask.u32[0] & (1 << fieldNumber);
 }
 
+uint64 DELTAJit_GetOriginalMask(delta_t* pFields) {
+	CDeltaJit* deltaJit = DELTAJit_LookupDeltaJit(__FUNCTION__, pFields);
+	return deltaJit->originalMarkedFieldsMask.u64;
+}
+
+uint64 DELTAJit_GetMaskU64(delta_t* pFields) {
+	CDeltaJit* deltaJit = DELTAJit_LookupDeltaJit(__FUNCTION__, pFields);
+	return deltaJit->marked_fields_mask.u64;
+}
+
 void CDeltaJitRegistry::Cleanup() {
 #ifndef REHLDS_FIXES
 	for (auto itr = m_DeltaToJITMap.iterator(); itr.hasElement(); itr.next()) {

rehlds/engine/delta_jit.h

@@ -73,8 +73,13 @@ union delta_marked_mask_t {
 
 extern CDeltaJitRegistry g_DeltaJitRegistry;
 
-extern int DELTAJit_Fields_Clear_Mark_Check(unsigned char *from, unsigned char *to, delta_t *pFields);
+extern int DELTAJit_Fields_Clear_Mark_Check(unsigned char *from, unsigned char *to, delta_t *pFields, void* pForceMarkMask);
 extern void DELTAJit_SetSendFlagBits(delta_t *pFields, int *bits, int *bytecount);
 extern void DELTAJit_SetFieldByIndex(struct delta_s *pFields, int fieldNumber);
 extern void DELTAJit_UnsetFieldByIndex(struct delta_s *pFields, int fieldNumber);
-extern qboolean DELTAJit_IsFieldMarked(delta_t* pFields, int fieldNumber);
+extern qboolean DELTAJit_IsFieldMarked(delta_t* pFields, int fieldNumber);
+
+/* Returns original mask, before it was changed by the conditional encoder */
+extern uint64 DELTAJit_GetOriginalMask(delta_t* pFields);
+
+extern uint64 DELTAJit_GetMaskU64(delta_t* pFields);

rehlds/engine/sv_main.cpp

@@ -4151,6 +4151,12 @@ int SV_CreatePacketEntities(sv_delta_t type, client_t *client, packet_entities_t
 	int oldmax;
 	int numbase;
 
+	// fix for https://github.com/dreamstalker/rehlds/issues/24
+#ifdef REHLDS_FIXES
+	int baselineToIdx = -1; //index of the baseline in to->entities[]
+	uint64 toBaselinesForceMask[MAX_PACKET_ENTITIES];
+#endif
+
 	numbase = 0;
 	if (type == sv_packet_delta)
 	{
@@ -4253,17 +4259,50 @@ int SV_CreatePacketEntities(sv_delta_t type, client_t *client, packet_entities_t
 				_mm_prefetch(((const char*)baseline_) + 64, _MM_HINT_T0);
 				if (offset)
 					SV_SetCallback(newindex, 0, custom, &numbase, 1, offset);
+
+				// fix for https://github.com/dreamstalker/rehlds/issues/24
+#ifdef REHLDS_FIXES
+				if (offset) 
+					baselineToIdx = newnum - offset;
+#endif
 			}
 		}
 
+
+		delta_t* delta = custom ? g_pcustomentitydelta : (SV_IsPlayerIndex(newindex) ? g_pplayerdelta : g_pentitydelta);
+
+		// fix for https://github.com/dreamstalker/rehlds/issues/24
+#ifdef REHLDS_FIXES
+		DELTA_WriteDeltaForceMask(
+			(uint8 *)baseline_,
+			(uint8 *)&to->entities[newnum],
+			TRUE,
+			delta,
+			&SV_InvokeCallback,
+			baselineToIdx != -1 ? &toBaselinesForceMask[baselineToIdx] : NULL
+		);
+		baselineToIdx = -1;
+
+		uint64 origMask = DELTAJit_GetOriginalMask(delta);
+		uint64 usedMask = DELTAJit_GetMaskU64(delta);
+		uint64 diffMask = origMask ^ usedMask;
+
+		//Remember changed fields that was marked in original mask, but unmarked by the conditional encoder
+		toBaselinesForceMask[newnum] = diffMask & origMask;
+		
+		
+#else //REHLDS_FIXES
 		DELTA_WriteDelta(
 			(uint8 *)baseline_,
 			(uint8 *)&to->entities[newnum],
 			TRUE,
-			custom ? g_pcustomentitydelta : (SV_IsPlayerIndex(newindex) ? g_pplayerdelta : g_pentitydelta),
+			delta,
 			&SV_InvokeCallback
 			);
+#endif //REHLDS_FIXES
+
 		++newnum;
+		
 	}
 
 	MSG_WriteBits(0, 16);

rehlds/unittests/delta_tests.cpp

@@ -64,7 +64,7 @@ NOINLINE qboolean _DoMarkFields(void* src, void* dst, delta_t* delta, bool useJi
 	qboolean sendfields;
 	if (useJit) {
 		DELTA_ClearFlags(delta);
-		return DELTAJit_Fields_Clear_Mark_Check((unsigned char*)src, (unsigned char*)dst, delta);
+		return DELTAJit_Fields_Clear_Mark_Check((unsigned char*)src, (unsigned char*)dst, delta, NULL);
 	} else {
 		DELTA_ClearFlags(delta);
 		DELTA_MarkSendFields((unsigned char*)src, (unsigned char*)dst, delta);