SHS/pip
4
3
Fork 0
Code Issues 67 Pull Requests 1 Actions Releases Wiki Activity

binlog fixes

Browse Source 
PIBinaryStream doc
remove makePIPair
rename bytesAvailable
This commit is contained in:
Бычков Андрей
2022-07-28 14:46:58 +03:00
parent 00d06f71ba
commit 1b09ad5c27
19 changed files with 95 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
libs/main/io_devices/pibinarylog.cpp
View File

@@ -550,15 +550,33 @@ void PIBinaryLog::parseLog(PIFile * f, PIBinaryLog::BinLogInfo * info, PIVector<
}
uchar read_sig[PIBINARYLOG_SIGNATURE_SIZE];
for (uint i=0; i<PIBINARYLOG_SIGNATURE_SIZE; i++) read_sig[i] = 0;
bool ok = true;
if (f->read(read_sig, PIBINARYLOG_SIGNATURE_SIZE) < 0) {if (info) info->records_count = -1; ok = false;}
for (uint i=0; i<PIBINARYLOG_SIGNATURE_SIZE; i++)
if (read_sig[i] != binlog_sig[i]) {if (info) info->records_count = -2; ok = false;}
if (f->read(read_sig, PIBINARYLOG_SIGNATURE_SIZE) < 0) {
if (info) info->records_count = -1;
return;
}
for (uint i=0; i<PIBINARYLOG_SIGNATURE_SIZE; i++) {
if (read_sig[i] != binlog_sig[i]) {
if (info) info->records_count = -2;
return;
}
}
uchar read_version = 0;
if (f->read(&read_version, 1) < 0) {if (info) info->records_count = -3; ok = false;}
if (read_version == 0) {if (info) info->records_count = -4; ok = false;}
if (read_version < PIBINARYLOG_VERSION_OLD) {if (info) info->records_count = -5; ok = false;}
if (read_version > PIBINARYLOG_VERSION) {if (info) info->records_count = -6; ok = false;}
if (f->read(&read_version, 1) < 0) {
if (info) info->records_count = -3;
return;
}
if (read_version == 0) {
if (info) info->records_count = -4;
return;
}
if (read_version < PIBINARYLOG_VERSION_OLD) {
if (info) info->records_count = -5;
return;
}
if (read_version > PIBINARYLOG_VERSION) {
if (info) info->records_count = -6;
return;
}
if (read_version == PIBINARYLOG_VERSION) {
uint32_t sz = 0;
f->read(&sz, 4);
@@ -566,7 +584,6 @@ void PIBinaryLog::parseLog(PIFile * f, PIBinaryLog::BinLogInfo * info, PIVector<
info->user_header = f->read(sz);
}
}
if (!ok) return;
PIByteArray ba;
BinLogRecord br;
br.id = 0;
@@ -580,11 +597,15 @@ void PIBinaryLog::parseLog(PIFile * f, PIBinaryLog::BinLogInfo * info, PIVector<
if (f->read(ba.data(), ba.size()) > 0) {
ba >> br.id >> br.size >> br.timestamp;
} else break;
if (info->log_size - f->pos() >= br.size) f->seek(f->pos() + br.size);
if (info) {
if (info->log_size - f->pos() >= br.size) {
f->seek(f->pos() + br.size);
}
}
else break;
}
if (br.id > 0) {
if (info) {
if (index) {
BinLogIndex bl_ind;
bl_ind.id = br.id;
bl_ind.data_size = br.size;
@@ -674,15 +695,9 @@ bool PIBinaryLog::cutBinLog(const PIBinaryLog::BinLogInfo & src, const PIString
}
bool PIBinaryLog::joinBinLogsSerial(const PIStringList & src, const PIString & dst) {
bool PIBinaryLog::joinBinLogsSerial(const PIStringList & src, const PIString & dst, std::function<bool (const PIString &, PISystemTime)> progress) {
PIBinaryLog slog;
PIBinaryLog dlog;
dlog.createNewFile(dst);
if (!dlog.isOpened()) {
piCout << "[PIBinaryLog]" << "Error, can't create" << dst;
return false;
}
piCout << "[PIBinaryLog]" << "Start join binlogs to" << dst;
PISystemTime dtime;
PISystemTime lt;
PITimeMeasurer tm;
@@ -692,8 +707,16 @@ bool PIBinaryLog::joinBinLogsSerial(const PIStringList & src, const PIString & d
piCout << "[PIBinaryLog]" << "Error, can't open" << fn;
return false;
}
if (first) first = false;
else {
if (first) {
first = false;
dlog.setHeader(slog.getHeader());
dlog.createNewFile(dst);
if (!dlog.isOpened()) {
piCout << "[PIBinaryLog]" << "Error, can't create" << dst;
return false;
}
piCout << "[PIBinaryLog]" << "Start join binlogs to" << dst;
} else {
dtime = lt;
}
tm.reset();
@@ -701,18 +724,29 @@ bool PIBinaryLog::joinBinLogsSerial(const PIStringList & src, const PIString & d
PISystemTime st;
while (!slog.isEnd()) {
br = slog.readRecord();
if (br.data.isEmpty() || br.id < 1) continue;
st = br.timestamp;
lt = dtime + br.timestamp;
if (dlog.writeBinLog_raw(br.id, lt, br.data) <= 0) {
piCout << "[PIBinaryLog]" << "Error, can't write to file" << dst;
return false;
}
if (tm.elapsed_s() > 1) {
if (tm.elapsed_s() > 0.1) {
tm.reset();
piCout << "[PIBinaryLog]" << "process" << PITime::fromSystemTime(lt).toString();
if (progress) {
if (!progress(fn, lt)) {
slog.close();
dlog.close();
PIFile::remove(dlog.path());
return false;
}
} else {
piCout << "[PIBinaryLog]" << "process" << PITime::fromSystemTime(lt).toString();
}
}
}
piCout << "[PIBinaryLog]" << "complete" << fn;
slog.close();
//piCout << "[PIBinaryLog]" << "complete" << fn;
}
piCout << "[PIBinaryLog]" << "Finish join binlogs, total time" << PITime::fromSystemTime(lt).toString();
return true;

2
libs/main/io_devices/pibinarylog.h
View File

@@ -298,7 +298,7 @@ public:
static bool cutBinLog(const BinLogInfo & src, const PIString & dst, int from, int to);
//! Create new binlog from serial splitted binlogs "src"
static bool joinBinLogsSerial(const PIStringList & src, const PIString & dst);
static bool joinBinLogsSerial(const PIStringList & src, const PIString & dst, std::function<bool (const PIString &, PISystemTime)> progress = nullptr);
protected:
PIString constructFullPathDevice() const override;

2
libs/main/io_devices/pifile.h
View File

@@ -227,7 +227,7 @@ public:
//! \~russian Возвращает размер файла в байтах
llong size() const;
ssize_t bytesAvailible() const override {return size() - pos();}
ssize_t bytesAvailable() const override {return size() - pos();}
//! \~english Returns read/write position
//! \~russian Возвращает позицию чтения/записи

2
libs/main/io_devices/piiobytearray.h
View File

@@ -84,7 +84,7 @@ public:
//! \~russian Вставляет данные "ba" в содержимое буфера в текущую позицию
int writeByteArray(const PIByteArray & ba);
ssize_t bytesAvailible() const override {
ssize_t bytesAvailable() const override {
if (data_) return data_->size();
else return 0;
}

4
libs/main/io_devices/piiodevice.h
View File

@@ -285,7 +285,7 @@ public:
PIByteArray read(int max_size);
//! \~english Returns the number of bytes that are available for reading.
//! \~russian Возвращает количество байт
//! \~russian Возвращает количество байт доступных для чтения
//! \~\details
//! \~english This function is commonly used with sequential devices
//! to determine the number of bytes to allocate in a buffer before reading.
@@ -293,7 +293,7 @@ public:
//! \~russian Эта функция как правило используется чтобы знать какой
//! размер буфера нужен в памяти для чтения.
//! Если функция возвращает -1 это значит что количество байт для чтения не известно.
virtual ssize_t bytesAvailible() const {return -1;}
virtual ssize_t bytesAvailable() const {return -1;}
//! \~english Write maximum "max_size" bytes of "data" to device
//! \~russian Пишет в устройство не более "max_size" байт из "data"

2
libs/main/io_devices/piiostream.h
View File

@@ -57,7 +57,7 @@ public:
ssize_t binaryStreamSizeImp() const {
if (!dev) return 0;
return dev->bytesAvailible();
return dev->bytesAvailable();
}
private:

2
libs/main/io_devices/piiostring.h
View File

@@ -88,7 +88,7 @@ public:
//! \~russian Вставляет строку "string" в содержимое буфера в текущую позицию
int writeString(const PIString & string);
ssize_t bytesAvailible() const override {
ssize_t bytesAvailable() const override {
if (str) return str->size() - pos;
else return 0;
}

2
libs/main/io_devices/pipeer.cpp
View File

@@ -924,7 +924,7 @@ void PIPeer::changeName(const PIString &new_name) {
}
ssize_t PIPeer::bytesAvailible() const {
ssize_t PIPeer::bytesAvailable() const {
ssize_t ret = 0;
read_buffer_mutex.lock();
if (!read_buffer.isEmpty()) ret = read_buffer.back().size();

2
libs/main/io_devices/pipeer.h
View File

@@ -116,7 +116,7 @@ public:
void setTrustPeerName(const PIString & peer_name) {trust_peer = peer_name;}
void setTcpServerIP(const PIString & ip) {server_ip = ip; tcpClientReconnect();}
ssize_t bytesAvailible() const override;
ssize_t bytesAvailable() const override;
EVENT2(dataReceivedEvent, const PIString &, from, const PIByteArray &, data);

2
libs/main/io_devices/pispi.cpp
View File

@@ -88,7 +88,7 @@ bool PISPI::isParameterSet(PISPI::Parameters parameter) const {
}
ssize_t PISPI::bytesAvailible() const {
ssize_t PISPI::bytesAvailable() const {
return recv_buf.size();
}

2
libs/main/io_devices/pispi.h
View File

@@ -60,7 +60,7 @@ public:
//! Returns parameters
PIFlags<PISPI::Parameters> parameters() const {return spi_mode;}
ssize_t bytesAvailible() const override;
ssize_t bytesAvailable() const override;
protected:
bool openDevice() override;

2
libs/main/io_devices/pitransparentdevice.cpp
View File

@@ -49,7 +49,7 @@ PITransparentDevice::~PITransparentDevice() {
}
ssize_t PITransparentDevice::bytesAvailible() const {
ssize_t PITransparentDevice::bytesAvailable() const {
ssize_t ret = 0;
que_mutex.lock();
if (que.isNotEmpty()) ret = que.back().size();

2
libs/main/io_devices/pitransparentdevice.h
View File

@@ -44,7 +44,7 @@ public:
virtual ~PITransparentDevice();
ssize_t bytesAvailible() const override;
ssize_t bytesAvailable() const override;
protected:
bool openDevice() override;