为什么mbtowc不按预期计算字符集？

Question

我想要计算文件中的字符(在各种字符集中)并且我使用函数'mbtowc'来检测字符.我无法弄清楚为什么字符和结果值不同.这是我的例子:

char buf[BUFFER_SIZE + MB_LEN_MAX];

int fd = open ("chinese_test", O_RDONLY);

unsigned int bytes, chars;

int bytes_read;

bytes = chars = 0;

while((bytes_read = read(fd, buf, BUFFER_SIZE)) > 0) {
    wchar_t wc_buf[BUFFER_SIZE], *wcp;
    char *p;
    int n = 0;

    bytes += bytes_read;

    p = buf;
    wcp = wc_buf;

    while((n = mbtowc(wcp, p, MB_LEN_MAX)) > 0) {
        p += n;
        wcp++;

        chars++;
    }

}

printf("chars: %d\tbytes: %d\n", chars, bytes);

我用一些带有GB2312字符的文本测试函数,但字符和字节的值太大了.

我的测试返回 - >字符:4638 | bytes:17473但'wc'linux命令返回:chars:16770 | 字节:17473

为何如此区别？我做错了什么？

---

现在我已经有了这段代码,但结果仍然存在差异.

char buf[BUFFER_SIZE * MB_LEN_MAX];

int fd = open ("test_chinese", O_RDONLY), filled = 0;

unsigned int bytes, chars;

int bytes_read;

bytes = chars = 0;

while((bytes_read = read(fd, buf, BUFFER_SIZE)) > 0) {
    wchar_t wc_buf[BUFFER_SIZE], *wcp;
    char *p;
    int n = 0;

    bytes += bytes_read;

    p = buf;
    wcp = wc_buf;



    while(bytes_read > 0) {
        n = mbtowc(NULL, p, MB_LEN_MAX);

        if (n <= 0) {
            p++;
            bytes_read--;
            continue;
        }
        p += n;

        bytes_read -= n;

        chars++;
    }

}

printf("\n\nchars: %d\tbytes: %d\n", chars, bytes);

Answer 1

问题是你BUFFER_SIZE的文件大小chinese_test和字节对齐的组合wchar_t.作为证据,尝试大幅增加BUFFER_SIZE- 你应该开始得到你想要的答案.

发生的事情是你的程序适用于它收到的第一个文本块.但是,如果字符在第一个和第二个块之间分开,请考虑代码中发生的情况,如下所示:

  | First Block                 | Second Block      |
  | [wchar_t] [wchar_t] ... [wchar_t] [wchar_t] ... |
  | [1,2,3,4] [1,2,3,4] ... [1,2,3,4] [1,2,3,4] ... |

您的代码将在第一个字符的第3个字节开始第二个块,并且不会被识别为有效字符.由于在找不到有效字符时mbtowc将返回-1,因此您的循环将立即结束并将为整个块计算零个字符.这同样适用于以下块.

编辑:
我注意到的另一个问题是你需要设置语言环境才能mbtowc正常工作.考虑到所有这些问题,我编写了以下内容,它返回与wc我相同的字符数:

#include <stdlib.h>
#include <stdio.h>
#include <locale.h>

int BUFFER_SIZE = 1024;
const char *DEFAULT_F_IN = "chinese_test";

struct counts {
    int bytes;
    int chars;
};

int count_block(struct counts *c, char *buf, int buf_size)
{
    int offset = 0;
    while (offset < buf_size) {
        int n = mbtowc(NULL, buf + offset, MB_CUR_MAX);
        if (n <= 0) {
            break;
        }

        offset += n;
        c->bytes += n;
        c->chars++;
    }

    return buf_size - offset;
}

void get_counts(struct counts *c, FILE *fd)
{
    char buf[BUFFER_SIZE];
    c->bytes = 0;
    c->chars = 0;

    int bytes_read;
    while((bytes_read = fread(buf, sizeof(*buf), BUFFER_SIZE, fd)) > 0) {
        int remaining = count_block(c, buf, bytes_read);
        if (remaining == 0) {
            continue;
        } else if (remaining < MB_CUR_MAX) {
            fseek(fd, -remaining, SEEK_CUR);
        } else {
            perror("Error");
            exit(1);
        }
    }
}

int main(int argc, char *argv[]) {
    FILE *fd;
    if (argc > 1) {
        fd = fopen(argv[1], "rb");
    } else {
        fd = fopen(DEFAULT_F_IN, "rb");
    }

    setlocale(LC_ALL, "");
    struct counts c;
    get_counts(&c, fd);
    printf("chars: %d\tbytes: %d\n", c.chars, c.bytes);

    return 0;
}