mirror of
https://github.com/alliedmodders/amxmodx.git
synced 2024-10-16 23:37:04 +03:00
608 lines
13 KiB
Plaintext
Executable File
608 lines
13 KiB
Plaintext
Executable File
{
|
|
This file is translated to Delphi from the file referenced below
|
|
by Jan Martin Pettersen (hdalis@users.sourceforge.net)
|
|
23/07/2005.
|
|
|
|
Some code in this file is also taken from the SciTE (Neil Hodgson)
|
|
}
|
|
// Utf8_16.cxx
|
|
// Copyright (C) 2002 Scott Kirkwood
|
|
//
|
|
// Permission to use, copy, modify, distribute and sell this code
|
|
// and its documentation for any purpose is hereby granted without fee,
|
|
// provided that the above copyright notice appear in all copies or
|
|
// any derived copies. Scott Kirkwood makes no representations
|
|
// about the suitability of this software for any purpose.
|
|
// It is provided "as is" without express or implied warranty.
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
unit UtfFunct;
|
|
interface
|
|
|
|
uses Windows,SysUtils,Classes,Math;
|
|
const
|
|
UniBufSize=32000;
|
|
|
|
type
|
|
Utf16=Word;
|
|
Utf8=Byte;
|
|
TUtf8Array=array[0..1] of Utf8;
|
|
PUtf8=^TUtf8Array;
|
|
TUtf16Array=array[0..1] of Utf16;
|
|
PUtf16=^TUtf16Array;
|
|
|
|
//uniCookie isn't used yet..
|
|
UniMode=(uni8Bit, uni16BE, uni16LE, uniUTF8,uniCookie);
|
|
//States for the unicode Next functions..
|
|
eState=(eStart,e2Bytes2,e3Bytes2,e3Bytes3);
|
|
|
|
// Reads UTF-16 and outputs UTF-8
|
|
Utf16_Iter=class(TObject)
|
|
private
|
|
m_eEncoding : UniMode;
|
|
m_eState : eState;
|
|
m_pBuf : PByte;
|
|
m_pRead : PByte;
|
|
m_pEnd : PByte;
|
|
m_nCur : Utf8;
|
|
m_nCur16 : Utf16;
|
|
public
|
|
constructor Create;
|
|
procedure Reset;
|
|
procedure Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
|
function More : Boolean;
|
|
procedure Next;
|
|
function Get : Utf8;
|
|
end;
|
|
|
|
|
|
// Reads UTF-8 and outputs UTF-16
|
|
Utf8_Iter =class(TObject)
|
|
private
|
|
m_eEncoding : UniMode;
|
|
m_eState : eState;
|
|
m_pBuf : PByte;
|
|
m_pRead : PByte;
|
|
m_pEnd : PByte;
|
|
m_nCur16 : Utf16;
|
|
procedure toStart;
|
|
procedure Swap;
|
|
public
|
|
constructor Create;
|
|
procedure Reset;
|
|
procedure Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
|
function More : Boolean; //bool
|
|
procedure Next;
|
|
function Get : Utf16;
|
|
function canGet : Boolean;
|
|
end;
|
|
|
|
// Reads UTF16 and outputs UTF8
|
|
UtfRead=class(TObject)
|
|
private
|
|
m_eEncoding : UniMode;
|
|
m_pBuf : PByte;
|
|
m_nBufSize : Cardinal;
|
|
m_bFirstRead : Boolean;
|
|
m_pNewBuf : PByte;
|
|
m_nLen : Cardinal;
|
|
m_Iter16 : Utf16_Iter;
|
|
public
|
|
constructor Create;
|
|
destructor Destroy;override;
|
|
function getEncoding : UniMode;
|
|
function getNewBuf : PChar;
|
|
function Convert(buf : PChar; len : Cardinal) : Cardinal;
|
|
procedure Reset;
|
|
property Encoding : UniMode read m_eEncoding;
|
|
end;
|
|
|
|
// Read in a UTF-8 buffer and write out to UTF-16 or UTF-8
|
|
UtfWrite=class(TObject)
|
|
private
|
|
m_eEncoding : UniMode;
|
|
m_pBuf : PUtf16;
|
|
m_nBufSize : Cardinal;
|
|
m_bFirstWrite : Boolean;
|
|
m_pFile : TStream;
|
|
procedure SetDestStream(Value : TStream);
|
|
procedure SetEncoding(eType : UniMode);
|
|
public
|
|
constructor Create;
|
|
function Write(const Buffer; Count : Cardinal) : LongInt;
|
|
property DestStream : TStream read m_pFile write SetDestStream;
|
|
property Encoding : UniMode read m_eEncoding write SetEncoding;
|
|
end;
|
|
|
|
//Returns the UTF8 length of the buffer 'uptr'.
|
|
function UTF8Length(const wideSrc : PWideChar; wideLen : Cardinal) : Cardinal;
|
|
//Transforms UCS2 to UTF8.
|
|
procedure UTF8FromUCS2(const wideSrc : PWideChar; wideLen : Cardinal; utfDestBuf : PChar; utfDestLen : Cardinal);
|
|
function UTF8ToAnsiP(const srcBuffer : PChar;len : Integer;destBuffer : PChar) : Integer;
|
|
function DetectEncoding(buf : PByte;len : Integer;var Encoding : UniMode) : Integer;
|
|
|
|
implementation
|
|
|
|
const
|
|
k_boms : array[uni8bit..uniUTF8,0..2] of Utf8=(
|
|
($00,$00,$00),
|
|
($FE,$FF,$00),
|
|
($FF,$FE,$00),
|
|
($EF,$BB,$BF));
|
|
|
|
function UTF8ToAnsiP(const srcBuffer : PChar;len : Integer;destBuffer : PChar) : Integer;
|
|
var
|
|
tmpbuffer : String;
|
|
srcLen,i,destLen : Integer;
|
|
begin
|
|
Result:=0;
|
|
if (not assigned(srcBuffer)) or (not assigned(destBuffer)) then Exit;
|
|
if len=-1 then
|
|
srcLen:=Length(srcBuffer)
|
|
else
|
|
srcLen:=len;
|
|
tmpbuffer:=UTF8ToAnsi(Copy(srcBuffer,1,srcLen));
|
|
destLen:=Length(tmpbuffer);
|
|
for i:=1 to destLen do
|
|
destBuffer[i-1]:=tmpbuffer[i];
|
|
destBuffer[destLen]:=#0;
|
|
Result:=destLen;
|
|
end;
|
|
|
|
function UTF8Length(const wideSrc : PWideChar; wideLen : Cardinal) : Cardinal;
|
|
var
|
|
i,len : Cardinal;
|
|
uch : Cardinal;
|
|
begin
|
|
len := 0;
|
|
i:=0;
|
|
while((i<wideLen) and (Cardinal(wideSrc[i])<>0)) do
|
|
begin
|
|
uch:=Cardinal(wideSrc[i]);
|
|
if (uch < $80) then
|
|
Inc(len)
|
|
else if (uch < $800) then
|
|
Inc(len,2)
|
|
else
|
|
Inc(len,3);
|
|
Inc(i);
|
|
end;
|
|
Result:=len;
|
|
end;
|
|
|
|
procedure UTF8FromUCS2(const wideSrc : PWideChar; wideLen : Cardinal; utfDestBuf : PChar; utfDestLen : Cardinal);
|
|
var
|
|
k : Integer;
|
|
i : Cardinal;
|
|
uch : Cardinal;
|
|
begin
|
|
k:= 0;
|
|
i:=0;
|
|
while((i<wideLen)and(Cardinal(wideSrc[i])<>0)) do
|
|
begin
|
|
uch:=Cardinal(wideSrc[i]);
|
|
if uch<$80 then
|
|
begin
|
|
utfDestBuf[k] := Char(uch);
|
|
Inc(k);
|
|
end else
|
|
if (uch<$800) then
|
|
begin
|
|
utfDestBuf[k]:=Char($C0 or (uch shr 6));
|
|
Inc(k);
|
|
utfDestBuf[k] := Char($80 or (uch and $3f));
|
|
Inc(k);
|
|
end else
|
|
begin
|
|
utfDestBuf[k] := Char($E0 or (uch shr 12));
|
|
Inc(k);
|
|
utfDestBuf[k] := Char($80 or ((uch shr 6) and $3f));
|
|
Inc(k);
|
|
utfDestBuf[k] := Char($80 or (uch and $3f));
|
|
Inc(k);
|
|
end;
|
|
end;
|
|
utfDestBuf[utfDestLen]:=#0;
|
|
end;
|
|
|
|
function DetectEncoding(buf : PByte;len : Integer;var Encoding : UniMode) : Integer;
|
|
var
|
|
nRet : Integer;
|
|
pbTmp : PByteArray;
|
|
begin
|
|
Encoding := uni8bit;
|
|
pbTmp:=PByteArray(buf);
|
|
nRet := 0;
|
|
if (len > 1) then
|
|
begin
|
|
if ((pbTmp[0]=k_Boms[uni16BE][0]) and (pbTmp[1]=k_Boms[uni16BE][1])) then
|
|
begin
|
|
Encoding := uni16BE;
|
|
nRet := 2;
|
|
end else
|
|
if ((pbTmp[0]=k_Boms[uni16LE][0]) and (pbTmp[1]=k_Boms[uni16LE][1])) then
|
|
begin
|
|
Encoding := uni16LE;
|
|
nRet := 2;
|
|
end else
|
|
if ((len>2) and (pbTmp[0]=k_Boms[uniUTF8][0]) and (pbTmp[1]=k_Boms[uniUTF8][1]) and (pbTmp[2]=k_Boms[uniUTF8][2])) then
|
|
begin
|
|
Encoding := uniUTF8;
|
|
nRet := 3;
|
|
end;
|
|
end;
|
|
Result:=nRet;
|
|
end;
|
|
|
|
|
|
constructor Utf16_Iter.Create;
|
|
begin
|
|
Reset;
|
|
end;
|
|
procedure Utf16_Iter.Reset;
|
|
begin
|
|
m_pBuf := nil;
|
|
m_pRead := nil;
|
|
m_pEnd := nil;
|
|
m_eState := eStart;
|
|
m_nCur := 0;
|
|
m_nCur16 := 0;
|
|
m_eEncoding := uni8bit;
|
|
end;
|
|
procedure Utf16_Iter.Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
|
begin
|
|
m_pBuf := pBuf;
|
|
m_pRead := pBuf;
|
|
m_pEnd := pBuf;
|
|
Inc(m_pEnd,nLen);
|
|
m_eEncoding := eEncoding;
|
|
Next;
|
|
end;
|
|
|
|
procedure Utf16_Iter.Next;
|
|
begin
|
|
case m_eState of
|
|
eStart:
|
|
begin
|
|
if (m_eEncoding = uni16LE) then
|
|
begin
|
|
m_nCur16 := Utf16(m_pRead^);
|
|
Inc(m_pRead);
|
|
m_nCur16 := m_nCur16 or Utf16((m_pRead^ shl 8));
|
|
end else
|
|
begin
|
|
m_nCur16 := Utf16(m_pRead^ shl 8);
|
|
Inc(m_pRead);
|
|
m_nCur16 := m_nCur16 or m_pRead^;
|
|
end;
|
|
Inc(m_pRead);
|
|
|
|
if (m_nCur16 < $80) then
|
|
begin
|
|
m_nCur := Byte(m_nCur16 and $FF);
|
|
m_eState := eStart;
|
|
end else
|
|
if (m_nCur16 < $800) then
|
|
begin
|
|
m_nCur := Byte($C0 or (m_nCur16 shr 6));
|
|
m_eState := e2Bytes2;
|
|
end else
|
|
begin
|
|
m_nCur := Byte($E0 or (m_nCur16 shr 12));
|
|
m_eState := e3Bytes2;
|
|
end;
|
|
|
|
end;
|
|
e2Bytes2:
|
|
begin
|
|
m_nCur := Byte($80 or (m_nCur16 and $3F));
|
|
m_eState := eStart;
|
|
end;
|
|
e3Bytes2:
|
|
begin
|
|
m_nCur := Byte($80 or ((m_nCur16 shr 6) and $3F));
|
|
m_eState := e3Bytes3;
|
|
end;
|
|
e3Bytes3:
|
|
begin
|
|
m_nCur := Byte($80 or (m_nCur16 and $3F));
|
|
m_eState := eStart;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
function Utf16_Iter.More : Boolean;
|
|
begin
|
|
Result:=Cardinal(m_pRead) <= Cardinal(m_pEnd);
|
|
end;
|
|
|
|
function Utf16_Iter.Get : Utf8;
|
|
begin
|
|
Result:=m_nCur;
|
|
end;
|
|
|
|
constructor Utf8_Iter.Create;
|
|
begin
|
|
Reset;
|
|
end;
|
|
|
|
procedure Utf8_Iter.Reset;
|
|
begin
|
|
m_pBuf := nil;
|
|
m_pRead := nil;
|
|
m_pEnd := nil;
|
|
m_eState := eStart;
|
|
m_nCur16 := 0;
|
|
m_eEncoding := uni8bit;
|
|
end;
|
|
|
|
procedure Utf8_Iter.Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
|
begin
|
|
m_pBuf := pBuf;
|
|
m_pRead := pBuf;
|
|
m_pEnd := pBuf;
|
|
Inc(m_pEnd,nLen);
|
|
m_eEncoding := eEncoding;
|
|
Next;
|
|
end;
|
|
|
|
procedure Utf8_Iter.Next;
|
|
begin
|
|
case (m_eState) of
|
|
eStart:
|
|
begin
|
|
if (($E0 and m_pRead^) = $E0) then
|
|
begin
|
|
m_nCur16 := Utf16(((not $E0) and m_pRead^) shl 12);
|
|
m_eState := e3Bytes2;
|
|
end else if (($C0 and m_pRead^) = $C0) then
|
|
begin
|
|
m_nCur16 := Utf16((not $C0 and m_pRead^) shl 6);
|
|
m_eState := e2Bytes2;
|
|
end else
|
|
begin
|
|
m_nCur16 := m_pRead^;
|
|
toStart;
|
|
end;
|
|
end;
|
|
e2Bytes2:
|
|
begin
|
|
m_nCur16 :=m_nCur16 or utf8($3F and m_pRead^);
|
|
toStart;
|
|
end;
|
|
e3Bytes2:
|
|
begin
|
|
m_nCur16 :=m_nCur16 or utf16(($3F and m_pRead^) shl 6);
|
|
m_eState := e3Bytes3;
|
|
end;
|
|
e3Bytes3:
|
|
begin
|
|
m_nCur16 :=m_nCur16 or utf8($3F and m_pRead^);
|
|
toStart;
|
|
end;
|
|
end;
|
|
Inc(m_pRead);
|
|
end;
|
|
|
|
function Utf8_Iter.More : Boolean;
|
|
begin
|
|
Result:=Cardinal(m_pRead) <= Cardinal(m_pEnd);
|
|
end;
|
|
|
|
function Utf8_Iter.Get : Utf16;
|
|
begin
|
|
Result:=m_nCur16;
|
|
end;
|
|
function Utf8_Iter.canGet : Boolean;
|
|
begin
|
|
Result:=m_eState = eStart;
|
|
end;
|
|
procedure Utf8_Iter.toStart;
|
|
begin
|
|
m_eState := eStart;
|
|
if (m_eEncoding = uni16BE) then
|
|
Swap;
|
|
end;
|
|
procedure Utf8_Iter.Swap;
|
|
var
|
|
p : PUtf8;
|
|
swapbyte : Utf8;
|
|
begin
|
|
p := PUtf8(@m_nCur16);
|
|
swapbyte := p[0];
|
|
p[0]:= p[1];
|
|
p[1]:=swapbyte;
|
|
end;
|
|
|
|
constructor UtfRead.Create;
|
|
begin
|
|
m_eEncoding := uni8bit;
|
|
m_nBufSize := 0;
|
|
m_pNewBuf := nil;
|
|
m_bFirstRead := True;
|
|
end;
|
|
|
|
destructor UtfRead.Destroy;
|
|
begin
|
|
if ((m_eEncoding <> uni8bit) and (m_eEncoding <> uniUTF8)) then
|
|
begin
|
|
if assigned(m_pNewBuf) then FreeMem(m_pNewBuf);
|
|
end;
|
|
inherited;
|
|
end;
|
|
|
|
function UtfRead.getEncoding : UniMode;
|
|
begin
|
|
Result:=m_eEncoding;
|
|
end;
|
|
function UtfRead.getNewBuf : PChar;
|
|
begin
|
|
Result:=PChar(m_pNewBuf);
|
|
end;
|
|
|
|
procedure UtfRead.Reset;
|
|
begin
|
|
m_bFirstRead:=True;
|
|
m_nBufSize:=0;
|
|
m_eEncoding :=uni8Bit;
|
|
end;
|
|
|
|
function UtfRead.Convert(buf : PChar; len : Cardinal) : Cardinal;
|
|
var
|
|
nSkip : Cardinal;
|
|
newSize : Cardinal;
|
|
pCur,pTemp : PByte;
|
|
begin
|
|
m_Iter16:=Utf16_Iter.Create;
|
|
try
|
|
m_pBuf := PByte(buf);
|
|
m_nLen := len;
|
|
|
|
nSkip := 0;
|
|
if (m_bFirstRead) then
|
|
begin
|
|
nSkip := DetectEncoding(m_pBuf,m_nLen,m_eEncoding);
|
|
m_bFirstRead := False;
|
|
end;
|
|
|
|
if (m_eEncoding = uni8bit) then
|
|
begin
|
|
// Do nothing, pass through
|
|
m_nBufSize := 0;
|
|
m_pNewBuf := m_pBuf;
|
|
Result:=len;
|
|
Exit;
|
|
end;
|
|
|
|
if (m_eEncoding = uniUTF8) then
|
|
begin
|
|
// Pass through after BOM
|
|
m_nBufSize := 0;
|
|
m_pNewBuf := m_pBuf;
|
|
Inc(m_pNewBuf,nSkip);
|
|
Result:=len - nSkip;
|
|
Exit;
|
|
end;
|
|
|
|
// Else...
|
|
//newSize := len + len div 2 + 1;
|
|
newSize:=len*2+1;
|
|
if (m_nBufSize <> newSize) then
|
|
begin
|
|
FreeMem(m_pNewBuf);
|
|
m_pNewBuf:=nil;
|
|
GetMem(m_pNewBuf,newSize);
|
|
m_nBufSize := newSize;
|
|
end;
|
|
|
|
pCur := m_pNewBuf;
|
|
pTemp:=m_pBuf;
|
|
Inc(pTemp,nSkip);
|
|
m_Iter16.Set_(pTemp, len - nSkip, m_eEncoding);
|
|
while(m_Iter16.More) do
|
|
begin
|
|
pCur^:=m_Iter16.Get;
|
|
Inc(PCur);
|
|
m_Iter16.Next;
|
|
end;
|
|
// Return number of bytes writen out
|
|
finally
|
|
FreeAndNil(m_Iter16);
|
|
end;
|
|
Result:=Cardinal(pCur) - Cardinal(m_pNewBuf);
|
|
end;
|
|
|
|
constructor UtfWrite.Create;
|
|
begin
|
|
m_eEncoding := uni8bit;
|
|
m_pFile := nil;
|
|
m_pBuf := nil;
|
|
m_bFirstWrite := true;
|
|
m_nBufSize := 0;
|
|
end;
|
|
|
|
procedure UtfWrite.SetEncoding(eType : UniMode);
|
|
begin
|
|
m_eEncoding := eType;
|
|
end;
|
|
procedure UtfWrite.SetDestStream(Value : TStream);
|
|
begin
|
|
m_pFile:=Value;
|
|
m_bFirstWrite:=True;
|
|
end;
|
|
|
|
function UtfWrite.Write(const Buffer; Count : Cardinal) : LongInt;
|
|
var
|
|
iter8 : Utf8_Iter;
|
|
pCur : ^Utf16;
|
|
ret : LongInt;
|
|
pTemp : PChar;
|
|
begin
|
|
if Count=0 then
|
|
begin
|
|
Result:=0;
|
|
Exit;
|
|
end;
|
|
iter8:=Utf8_Iter.Create;
|
|
try
|
|
if (not assigned(m_pFile)) then
|
|
begin
|
|
Result:=0;
|
|
Exit;
|
|
end;
|
|
if (m_eEncoding = uni8bit) then
|
|
begin
|
|
// Normal write
|
|
m_bFirstWrite:=False;
|
|
Result:=m_pFile.Write(PChar(Buffer)^, Count);
|
|
Exit;
|
|
end;
|
|
|
|
if (m_eEncoding = uniUTF8) then
|
|
begin
|
|
pTemp:=PChar(Buffer);
|
|
if (m_bFirstWrite) then
|
|
begin
|
|
m_pFile.Write(k_Boms[m_eEncoding], 3);
|
|
m_bFirstWrite := false;
|
|
end;
|
|
Result:=m_pFile.Write(pTemp^, Count);
|
|
Exit;
|
|
end;
|
|
|
|
if (Count > m_nBufSize) then
|
|
begin
|
|
m_nBufSize := Count;
|
|
if assigned(m_pBuf) then FreeMem(m_pBuf);
|
|
m_pBuf := nil;
|
|
GetMem(m_pBuf,SizeOf(Utf16)*(Count+1));
|
|
end;
|
|
|
|
if (m_bFirstWrite) then
|
|
begin
|
|
if ((m_eEncoding = uni16BE) or (m_eEncoding = uni16LE)) then
|
|
begin
|
|
// Write the BOM
|
|
m_pFile.Write(k_Boms[m_eEncoding],2);
|
|
end;
|
|
m_bFirstWrite := false;
|
|
end;
|
|
|
|
iter8.set_(PByte(Buffer), Count, m_eEncoding);
|
|
pCur := @m_pBuf[0];
|
|
while(iter8.More) do
|
|
begin
|
|
if (iter8.canGet) then
|
|
begin
|
|
pCur^ := iter8.Get;
|
|
Inc(pCur);
|
|
end;
|
|
iter8.Next;
|
|
end;
|
|
ret := m_pFile.Write(m_pBuf^,Cardinal(pCur)-Cardinal(m_pBuf));
|
|
finally
|
|
if assigned(iter8) then FreeAndNil(iter8);
|
|
end;
|
|
Result:=ret;
|
|
end;
|
|
|
|
end. |