Thursday, March 5, 2009

Conversion between Utf-8 and GB2312

Some codes about conversion between utf-8 and gb2312 when handling different text format.

// ChineseCodeLib.h: interface for the CChineseCodeLib class.
//
//////////////////////////////////////////////////////////////////////
#include<string>
using namespace std;

/*
功?能?:?汉?字?GB2312与?UTF-8编?码?互?转?
作?者?:?litz
Email:mycro@163.com
参?考?:?吴?康?彬?先?生?的?文?章?《?UTF-8与?GB2312之?间?的?互?换?》?
http://www.vckbase.com/document/viewdoc/?id=1397
*/


#if !defined(__CCHINESECODELIB_H_)
#define __CCHINESECODELIB_H_


class CChineseCodeLib
{
public:
static void UTF_8ToGB2312(string& pOut,char *pText, int pLen);
static void GB2312ToUTF_8(string& pOut,char *pText, int pLen);
// Unicode 转?换?成?UTF-8
static void UnicodeToUTF_8(char* pOut,wchar_t* pText);
// GB2312 转?换?成? ?Unicode
static void Gb2312ToUnicode(wchar_t* pOut,char *gbBuffer);
// 把?Unicode 转?换?成?GB2312
static void UnicodeToGB2312(char* pOut,unsigned short uData);
// 把?UTF-8转?换?成?Unicode
static void UTF_8ToUnicode(wchar_t* pOut,char* pText);

CChineseCodeLib();
virtual ~CChineseCodeLib();
};

#endif // !defined(__CCHINESECODELIB_H_)




///////////////////////////////////////////////////



// ChineseCodeLib.cpp: implementation of the CChineseCodeLib class.
//
//////////////////////////////////////////////////////////////////////

#include <stdio.h>
#include <tchar.h>
#include <windows.h>
#include "ChineseCodeLib.h"

//////////////////////////////////////////////////////////////////////
// Construction/Destruction
//////////////////////////////////////////////////////////////////////

CChineseCodeLib::CChineseCodeLib()
{

}

CChineseCodeLib::~CChineseCodeLib()
{

}


void CChineseCodeLib::UTF_8ToUnicode(wchar_t* pOut,char *pText)
{
char* uchar = (char *)pOut;

uchar[1] = ((pText[0] & 0x0F) << 4) + ((pText[1] >> 2) & 0x0F);
uchar[0] = ((pText[1] & 0x03) << 6) + (pText[2] & 0x3F);

return;
}

void CChineseCodeLib::UnicodeToGB2312(char* pOut,unsigned short uData)
{
WideCharToMultiByte(CP_ACP,NULL,(LPCWSTR)&uData,1,pOut,sizeof(wchar_t),NULL,NULL);
return;
}

void CChineseCodeLib::Gb2312ToUnicode(wchar_t* pOut,char *gbBuffer)
{
::MultiByteToWideChar(CP_ACP,MB_PRECOMPOSED,gbBuffer,2,pOut,1);
return;
}

void CChineseCodeLib::UnicodeToUTF_8(char* pOut,wchar_t* pText)
{
// 注?意?wchar_t高?低?字?的?顺?序?,低?字?节?在?前?,?高?字?节?在?后?
char* pchar = (char *)pText;

pOut[0] = (0xE0 | ((pchar[1] & 0xF0) >> 4));
pOut[1] = (0x80 | ((pchar[1] & 0x0F) << 2)) + ((pchar[1] & 0xC0) >> 6);
pOut[2] = (0x80 | (pchar[0] & 0x3F));

return;
}

void CChineseCodeLib::GB2312ToUTF_8(string& pOut,char *pText, int pLen)
{
char buf[4];
char* rst = new char[pLen + (pLen >> 2) + 2];

memset(buf,0,4);
memset(rst,0,pLen + (pLen >> 2) + 2);

int i = 0;
int j = 0;
while(i < pLen)
{
//如?果?是?英?文?直?接?复?制?就?可?以?
if( *(pText + i) >= 0)
{
rst[j++] = pText[i++];
}
else
{
wchar_t pbuffer;
Gb2312ToUnicode(&pbuffer,pText+i);

UnicodeToUTF_8(buf,&pbuffer);

unsigned short int tmp = 0;
tmp = rst[j] = buf[0];
tmp = rst[j+1] = buf[1];
tmp = rst[j+2] = buf[2];


j += 3;
i += 2;
}
}
rst[j] = '\0';

//返?回?结?果?
pOut = rst;
delete []rst;

return;
}

void CChineseCodeLib::UTF_8ToGB2312(string &pOut, char *pText, int pLen)
{
char * newBuf = new char[pLen];
char Ctemp[4];
memset(Ctemp,0,4);

int i =0;
int j = 0;

while(i < pLen)
{
if(pText[i] > 0)
{
newBuf[j++] = pText[i++];
}
else
{
wchar_t Wtemp;
UTF_8ToUnicode(&Wtemp,pText + i);

UnicodeToGB2312(Ctemp,Wtemp);

newBuf[j] = Ctemp[0];
newBuf[j + 1] = Ctemp[1];

i += 3;
j += 2;
}
}
newBuf[j] = '\0';

pOut = newBuf;
delete []newBuf;

return;
}


///////////////////////////////////////////////////



//A test main program:



//Input utf.txt in the same folder with the source files.



//Output resutl.txt



// Decode1.cpp : Defines the entry point for the console application.


#include "ChineseCodeLib.h"

#include <iostream>
#include <fstream>
#include <string>
using namespace std;

#define LEN 100000

char str[LEN];
string res;


int main(int argc, char* argv[])
{
ifstream fin("utf.txt");
ofstream fout("result.txt");

while(!fin.eof())
{
fin.getline(str,LEN);
if(strlen(str)<10)
continue;
CChineseCodeLib::UTF_8ToGB2312(res,str,strlen(str));
//cout<<res<<endl;
fout<<res;
}

fin.close();
fout.close();

return 0;
}




From: http://blog.csdn.net/Mycro/archive/2005/12/06/544637.aspx

No comments:

Post a Comment

Google+