From 5c767c5e3e47c0d41447822649f37816f87d172a Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 26 Apr 2022 16:01:14 +0300 Subject: [PATCH 1/4] linux mb test --- libs/main/core/pistring.cpp | 53 ++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/libs/main/core/pistring.cpp b/libs/main/core/pistring.cpp index 9395c78d..c3317a02 100644 --- a/libs/main/core/pistring.cpp +++ b/libs/main/core/pistring.cpp @@ -27,7 +27,7 @@ #ifdef WINDOWS # include #endif -#include +#include //! \addtogroup Core //! \{ @@ -201,11 +201,11 @@ llong PIString::toNumberBase(const PIString & value, int base, bool * ok) { void PIString::appendFromChars(const char * c, int s, const char * codepage) { if (s == 0) return; int old_sz = size_s(); + if (s == -1) s = strlen(c); #ifdef PIP_ICU UErrorCode e((UErrorCode)0); UConverter * cc = ucnv_open(codepage, &e); if (cc) { - if (s == -1) s = strlen(c); enlarge(s); // UChar * ucs = new UChar[s]; // memset(ucs, 0, s * sizeof(UChar)); @@ -223,7 +223,6 @@ void PIString::appendFromChars(const char * c, int s, const char * codepage) { } #else # ifdef WINDOWS - if (s == -1) s = strlen(c); int sz = MultiByteToWideChar((uint)(uintptr_t)codepage, MB_ERR_INVALID_CHARS, c, s, 0, 0); if (sz <= 0) return; enlarge(sz); @@ -231,25 +230,37 @@ void PIString::appendFromChars(const char * c, int s, const char * codepage) { return; //printf("request %d\n", sz); # else - mbstate_t state; - memset(&state, 0, sizeof(state)); - const char ** pc; - char * c_ = nullptr; - if (s > 0) { - c_ = (char*)malloc(s+1); - memcpy(c_, c, s); - c_[s] = '\0'; - pc = (const char **)&c_; - } else { - pc = &c; + std::mbstate_t state{}; + PIChar c16; + int sz = 0; + while(std::size_t rc = std::mbrtoc16(&c16, c+sz, s - sz, &state)) { + if(rc == (std::size_t)-3) + d.append(c16); + else if(rc == (std::size_t)-2) + break; + else if(rc == (std::size_t)-1) + break; + else { + sz += rc; + d.append(c16); } - size_t len = mbsrtowcs(NULL, pc, 0, &state); - wchar_t wstr[len+1]; - mbsrtowcs(&wstr[0], pc, len+1, &state); - if (c_) free(c_); - enlarge(len); - for (size_t i=0; i 0) { +// c_ = (char*)malloc(s+1); +// memcpy(c_, c, s); +// c_[s] = '\0'; +// pc = (const char **)&c_; +// } else { +// pc = &c; +// } +// size_t len = mbsrtowcs(NULL, pc, 0, &state); +// wchar_t wstr[len+1]; +// mbsrtowcs(&wstr[0], pc, len+1, &state); +// if (c_) free(c_); +// d.enlarge(len); +// for (size_t i=0; i Date: Tue, 26 Apr 2022 16:03:37 +0300 Subject: [PATCH 2/4] linux mb test --- libs/main/core/pistring.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/main/core/pistring.cpp b/libs/main/core/pistring.cpp index c3317a02..c8edb2db 100644 --- a/libs/main/core/pistring.cpp +++ b/libs/main/core/pistring.cpp @@ -233,16 +233,16 @@ void PIString::appendFromChars(const char * c, int s, const char * codepage) { std::mbstate_t state{}; PIChar c16; int sz = 0; - while(std::size_t rc = std::mbrtoc16(&c16, c+sz, s - sz, &state)) { + while(std::size_t rc = std::mbrtoc16((char16_t*)&c16, c+sz, s - sz, &state)) { if(rc == (std::size_t)-3) - d.append(c16); + append(c16); else if(rc == (std::size_t)-2) break; else if(rc == (std::size_t)-1) break; else { sz += rc; - d.append(c16); + append(c16); } } // const char ** pc; From 06c8e6af103b350eb7d0be481c3217ddf8c8f8b9 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 26 Apr 2022 16:30:26 +0300 Subject: [PATCH 3/4] linux mb test --- libs/main/core/pistring.cpp | 49 ++++++------------------------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/libs/main/core/pistring.cpp b/libs/main/core/pistring.cpp index c8edb2db..3bfdf3c3 100644 --- a/libs/main/core/pistring.cpp +++ b/libs/main/core/pistring.cpp @@ -27,7 +27,9 @@ #ifdef WINDOWS # include #endif -#include +#include +#include +#include //! \addtogroup Core //! \{ @@ -207,17 +209,9 @@ void PIString::appendFromChars(const char * c, int s, const char * codepage) { UConverter * cc = ucnv_open(codepage, &e); if (cc) { enlarge(s); -// UChar * ucs = new UChar[s]; -// memset(ucs, 0, s * sizeof(UChar)); e = (UErrorCode)0; int sz = ucnv_toUChars(cc, (UChar*)(PIDeque::data(old_sz)), s, c, s, &e); - //printf("appendFromChars %d -> %d\n", s, sz); - //printf("PIString %d -> %d\n", c[0], ucs[0]); resize(old_sz+sz); -// for (int i = 0; i < sz; ++i) { -// push_back(PIChar((ushort)ucs[i])); -// } -// delete[] ucs; ucnv_close(cc); return; } @@ -227,40 +221,11 @@ void PIString::appendFromChars(const char * c, int s, const char * codepage) { if (sz <= 0) return; enlarge(sz); MultiByteToWideChar((uint)(uintptr_t)codepage, MB_ERR_INVALID_CHARS, c, s, (LPWSTR)PIDeque::data(old_sz), sz); - return; - //printf("request %d\n", sz); # else - std::mbstate_t state{}; - PIChar c16; - int sz = 0; - while(std::size_t rc = std::mbrtoc16((char16_t*)&c16, c+sz, s - sz, &state)) { - if(rc == (std::size_t)-3) - append(c16); - else if(rc == (std::size_t)-2) - break; - else if(rc == (std::size_t)-1) - break; - else { - sz += rc; - append(c16); - } - } -// const char ** pc; -// char * c_ = nullptr; -// if (s > 0) { -// c_ = (char*)malloc(s+1); -// memcpy(c_, c, s); -// c_[s] = '\0'; -// pc = (const char **)&c_; -// } else { -// pc = &c; -// } -// size_t len = mbsrtowcs(NULL, pc, 0, &state); -// wchar_t wstr[len+1]; -// mbsrtowcs(&wstr[0], pc, len+1, &state); -// if (c_) free(c_); -// d.enlarge(len); -// for (size_t i=0; i, char16_t> ucs2conv; + std::u16string ucs2 = ucs2conv.from_bytes(c, c+s); + enlarge(ucs2.size()); + ucs2.copy((char16_t *)PIDeque::data(old_sz), ucs2.size()); # endif #endif } From 186c71d9730de0a729bce74ec9550ede46a27816 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 26 Apr 2022 16:41:04 +0300 Subject: [PATCH 4/4] linux mb test --- libs/main/core/pistring.cpp | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/libs/main/core/pistring.cpp b/libs/main/core/pistring.cpp index 3bfdf3c3..11d88b10 100644 --- a/libs/main/core/pistring.cpp +++ b/libs/main/core/pistring.cpp @@ -324,21 +324,8 @@ void PIString::buildData(const char * cp) const { const size_t len = MB_CUR_MAX*size()+1; data_ = (char *)malloc(len); sz = ucnv_fromUChars(cc, data_, len, (const UChar*)(PIDeque::data()), size_s(), &e); -// char uc[8]; -// data_.reserve(size_s()); -// for (int i = 0; i < size_s(); ++i) { -// if (at(i).isAscii()) { -// data_.push_back(uchar(at(i).unicode16Code())); -// } else { -// e = (UErrorCode)0; -// for (int j = 0; j < sz; ++j) { -// data_.push_back(uc[j]); -// } -// } -// } ucnv_close(cc); data_[sz] = '\0'; -// data_.push_back('\0'); return; } #else @@ -357,23 +344,10 @@ void PIString::buildData(const char * cp) const { data_[sz] = '\0'; return; # else - wchar_t wc; - //char tc[MB_CUR_MAX]; - mbstate_t state; - memset(&state, 0, sizeof(state)); - data_ = (char *)malloc(MB_CUR_MAX*size()+1); - char *p = data_; - for (int i = 0; i < size_s(); ++i) { -// if (at(i).isAscii()) { -// data_.push_back(uchar(at(i).toAscii())); -// continue; -// } - wc = at(i).toWChar(); - sz = wcrtomb(p, wc, &state); - if (sz < 0) break; - p += sz; - } - p[0] = '\0'; + std::wstring_convert, char16_t> ucs2conv; + std::string u8str = ucs2conv.to_bytes((char16_t*)PIDeque::data(), (char16_t*)PIDeque::data()+size()); + data_ = (char *)malloc(u8str.size()+1); + strcpy(data_, u8str.c_str()); # endif #endif }