如何逐行读取unicode(utf-8)/二进制文件

Question

嗨程序员,

我想逐行读取由记事本创建的Unicode(UTF-8)文本文件,我不想在屏幕上显示Unicode字符串,我只想阅读并比较字符串!

此代码逐行读取ANSI文件,并比较字符串

我想要的是

逐行阅读test_ansi.txt

如果line ="b"打印"YES!"

别打印"不!"

read_ansi_line_by_line.c

#include <stdio.h>

int main()
{
    char *inname = "test_ansi.txt";
    FILE *infile;
    char line_buffer[BUFSIZ]; /* BUFSIZ is defined if you include stdio.h */
    char line_number;

    infile = fopen(inname, "r");
    if (!infile) {
        printf("\nfile '%s' not found\n", inname);
        return 0;
    }
    printf("\n%s\n\n", inname);

    line_number = 0;
    while (fgets(line_buffer, sizeof(line_buffer), infile)) {
        ++line_number;
        /* note that the newline is in the buffer */
        if (strcmp("b\n", line_buffer) == 0 ){
            printf("%d: YES!\n", line_number);
        }else{
            printf("%d: NO!\n", line_number,line_buffer);
        }
    }
    printf("\n\nTotal: %d\n", line_number);
    return 0;
}

test_ansi.txt

a
b
c

编译

gcc -o read_ansi_line_by_line read_ansi_line_by_line.c

产量

test_ansi.txt

1: NO!
2: YES!
3: NO!


Total: 3

现在我需要读取由记事本创建的Unicode(UTF-8)文件,经过6个多月的时间我没有发现C中的任何好的代码/库可以读取以UTF-8编码的文件!,我不知道为什么但是我认为标准C不支持Unicode!

读取Unicode二进制文件OK!,但问题是二进制文件已经以二进制模式创建!这意味着如果我们想要读取由记事本创建的Unicode(UTF-8)文件,我们需要从UTF-8翻译它文件到BINARY文件!

此代码将Unicode字符串写入二进制文件,注意C文件以UTF-8编码并由GCC编译

我想要的是

将Unicode字符"ب"写入test_bin.dat

create_bin.c

#define UNICODE
#ifdef UNICODE
#define _UNICODE
#else
#define _MBCS
#endif

#include <stdio.h>
#include <wchar.h>

int main()
{
     /*Data to be stored in file*/
     wchar_t line_buffer[BUFSIZ]=L"?";
     /*Opening file for writing in binary mode*/
     FILE *infile=fopen("test_bin.dat","wb");
     /*Writing data to file*/
     fwrite(line_buffer, 1, 13, infile);
     /*Closing File*/
     fclose(infile);

    return 0;
}

编译

gcc -o create_bin create_bin.c

产量

create test_bin.dat

现在我想逐行读取二进制文件并进行比较!

我想要的是

如果line ="ب"打印"YES!",请逐行读取test_bin.dat.别打印"不!"

read_bin_line_by_line.c

#define UNICODE
#ifdef UNICODE
#define _UNICODE
#else
#define _MBCS
#endif

#include <stdio.h>
#include <wchar.h>

int main()
{
    wchar_t *inname = L"test_bin.dat";
    FILE *infile;
    wchar_t line_buffer[BUFSIZ]; /* BUFSIZ is defined if you include stdio.h */

    infile = _wfopen(inname,L"rb");
    if (!infile) {
        wprintf(L"\nfile '%s' not found\n", inname);
        return 0;
    }
    wprintf(L"\n%s\n\n", inname);

    /*Reading data from file into temporary buffer*/
    while (fread(line_buffer,1,13,infile)) {
        /* note that the newline is in the buffer */
        if ( wcscmp ( L"?" , line_buffer ) == 0 ){
             wprintf(L"YES!\n");
        }else{
             wprintf(L"NO!\n", line_buffer);
        }
    }
    /*Closing File*/
    fclose(infile);
    return 0;
}

产量

test_bin.dat

YES!

---

问题

这种方法非常长!并且没有力量(我是软件工程的初学者)

请任何人知道如何阅读Unicode文件？(我知道它不容易!)请知道如何将Unicode文件转换为二进制文件？(简单方法)请知道如何在二进制模式下读取Unicode文件？(我不确定)

谢谢.

Answer 1

UTF-8的一个很好的特性是,你并不需要为了进行比较来解码.无论您是先解码,strcmp返回的顺序都是相同的.所以只需将其作为原始字节读取并运行strcmp.

Answer 2

我找到了解决问题的方法,我想与任何有兴趣在C99中读取UTF-8文件的人分享解决方案.

void ReadUTF8(FILE* fp)
{
    unsigned char iobuf[255] = {0};
    while( fgets((char*)iobuf, sizeof(iobuf), fp) )
    {
            size_t len = strlen((char *)iobuf);
            if(len > 1 &&  iobuf[len-1] == '\n')
                iobuf[len-1] = 0;
            len = strlen((char *)iobuf);
            printf("(%d) \"%s\"  ", len, iobuf);
            if( iobuf[0] == '\n' )
                printf("Yes\n");
            else
                printf("No\n");
    }
}

void ReadUTF16BE(FILE* fp)
{
}

void ReadUTF16LE(FILE* fp)
{
}

int main()
{
    FILE* fp = fopen("test_utf8.txt", "r");
    if( fp != NULL)
    {
        // see http://en.wikipedia.org/wiki/Byte-order_mark for explaination of the BOM
        // encoding
        unsigned char b[3] = {0};
        fread(b,1,2, fp);
        if( b[0] == 0xEF && b[1] == 0xBB)
        {
            fread(b,1,1,fp); // 0xBF
            ReadUTF8(fp);
        }
        else if( b[0] == 0xFE && b[1] == 0xFF)
        {
            ReadUTF16BE(fp);
        }
        else if( b[0] == 0 && b[1] == 0)
        {
            fread(b,1,2,fp); 
            if( b[0] == 0xFE && b[1] == 0xFF)
                ReadUTF16LE(fp);
        }
        else
        {
            // we don't know what kind of file it is, so assume its standard
            // ascii with no BOM encoding
            rewind(fp);
            ReadUTF8(fp);
        }
    }        

    fclose(fp);
}