pip/libs/main/io_devices/pibinarylog.h

//! \addtogroup IO
//! \{
//! \file pibinarylog.h
//! \brief
//! \~english Binary log
//! \~russian Бинарный лог
//! \details
//! \~english Class for writing and reading binary data to/from log files, with support for playback in different modes
//! \~russian Класс для записи и чтения бинарных данных в/из файлов логов с поддержкой воспроизведения в различных режимах
//! \~\}
/*
    PIP - Platform Independent Primitives
    Class for write binary data to logfile, and read or playback this data
    Andrey Bychkov work.a.b@yandex.ru

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef PIBINARYLOG_H
#define PIBINARYLOG_H

#include "pichunkstream.h"
#include "pifile.h"

//! \~english Class for writing and reading binary data to/from log files, with support for playback in different modes
//! \~russian Класс для записи и чтения бинарных данных в/из файлов логов с поддержкой воспроизведения в различных режимах
class PIP_EXPORT PIBinaryLog: public PIIODevice {
	PIIODEVICE(PIBinaryLog, "binlog");

public:
	explicit PIBinaryLog();
	virtual ~PIBinaryLog();

 	//! \~english Play modes for \a PIBinaryLog
 	//! \~russian Режимы воспроизведения для \a PIBinaryLog
 	enum PlayMode {
 		PlayRealTime /*! \~english Play in system realtime, default mode \~russian В системном реальном времени, режим по умолчанию */,
 		PlayVariableSpeed /*! \~english Play in software realtime with speed, set by \a setSpeed \~russian В программном реальном времени со скоростью, задаваемой через \a setSpeed */,
 		PlayStaticDelay /*! \~english Play with custom static delay, ignoring timestamp \~russian С пользовательской статической задержкой, игнорируя метку времени */
 	};

 	//! \~english Different split modes for writing \a PIBinaryLog, which can separate files by size, by time or by records count
 	//! \~russian Различные режимы разделения для записи \a PIBinaryLog, позволяющие разделять файлы по размеру, времени или количеству записей
 	enum SplitMode {
 		SplitNone /*! \~english Without separate, default mode \~russian Без разделения, режим по умолчанию */,
 		SplitTime /*! \~english Separate files by record time \~russian Разделение файлов по времени записи */,
 		SplitSize /*! \~english Separate files by size \~russian Разделение файлов по размеру */,
 		SplitCount /*! \~english Separate files by records count \~russian Разделение файлов по количеству записей */
 	};

#pragma pack(push, 8)

 	//! \~english Struct contains information about all records with same ID
 	//! \~russian Структура содержит информацию обо всех записях с одинаковым ID
 	struct PIP_EXPORT BinLogRecordInfo {
 		//! \~english Constructor, initializes all fields to zero
 		//! \~russian Конструктор, инициализирует все поля нулями
 		BinLogRecordInfo() {
 			id = count   = 0;
 			minimum_size = maximum_size = 0;
 		}
 		//! \~english Record ID
 		//! \~russian ID записи
 		int id;
 		//! \~english Records count
 		//! \~russian Количество записей
 		int count;
 		//! \~english Minimum record size
 		//! \~russian Минимальный размер записи
 		int minimum_size;
 		//! \~english Maximum record size
 		//! \~russian Максимальный размер записи
 		int maximum_size;
 		//! \~english Start time of records
 		//! \~russian Время начала записей
 		PISystemTime start_time;
 		//! \~english End time of records
 		//! \~russian Время окончания записей
 		PISystemTime end_time;
 	};

 	//! \~english Struct contains position, ID and timestamp of record in file
 	//! \~russian Структура содержит позицию, ID и метку времени записи в файле
 	struct PIP_EXPORT BinLogIndex {
 		//! \~english Record ID
 		//! \~russian ID записи
 		int id;
 		//! \~english Data size
 		//! \~russian Размер данных
 		int data_size;
 		//! \~english Position in file
 		//! \~russian Позиция в файле
 		llong pos;
 		//! \~english Timestamp
 		//! \~russian Метка времени
 		PISystemTime timestamp;
 	};

#pragma pack(pop)

 	//! \~english Struct contains full information about Binary Log file and about all Records using map of \a BinLogRecordInfo
 	//! \~russian Структура содержит полную информацию о файле бинарного лога и обо всех записях с использованием карты \a BinLogRecordInfo
 	struct PIP_EXPORT BinLogInfo {
 		//! \~english Path to the log file
 		//! \~russian Путь к файлу лога
 		PIString path;
 		//! \~english Total records count
 		//! \~russian Общее количество записей
 		int records_count = 0;
 		//! \~english Log file size
 		//! \~russian Размер файла лога
 		llong log_size    = 0L;
 		//! \~english Start time of logging
 		//! \~russian Время начала логирования
 		PISystemTime start_time;
 		//! \~english End time of logging
 		//! \~russian Время окончания логирования
 		PISystemTime end_time;
 		//! \~english Map of record information by ID
 		//! \~russian Карта информации о записях по ID
 		PIMap<int, BinLogRecordInfo> records;
 		//! \~english User-defined header data
 		//! \~russian Пользовательские данные заголовка
 		PIByteArray user_header;
 	};


 	//! \~english Returns current \a PlayMode
 	//! \~russian Возвращает текущий \a PlayMode
 	PlayMode playMode() const { return play_mode; }

 	//! \~english Returns current \a SplitMode
 	//! \~russian Возвращает текущий \a SplitMode
 	SplitMode splitMode() const { return split_mode; }

 	//! \~english Returns current directory where binlogs will be saved
 	//! \~russian Возвращает текущий каталог, куда будут сохраняться бинарные логи
 	PIString logDir() const { return property("logDir").toString(); }

 	//! \~english Returns current file prefix
 	//! \~russian Возвращает текущий префикс файла
 	PIString filePrefix() const { return property("filePrefix").toString(); }

 	//! \~english Returns default ID, used in \a write function
 	//! \~russian Возвращает ID по умолчанию, используемый в функции \a write
 	int defaultID() const { return default_id; }

 	//! \~english Returns current play speed
 	//! \~russian Возвращает текущую скорость воспроизведения
 	double playSpeed() const { return play_speed > 0 ? 1. / play_speed : 0.; }

 	//! \~english Returns current play delay
 	//! \~russian Возвращает текущую задержку воспроизведения
 	PISystemTime playDelay() const { return play_delay; }

 	//! \~english Returns current binlog file split time
 	//! \~russian Возвращает текущее время разделения файла бинарного лога
 	PISystemTime splitTime() const { return split_time; }

 	//! \~english Returns current binlog file split size
 	//! \~russian Возвращает текущий размер разделения файла бинарного лога
 	llong splitFileSize() const { return split_size; }

 	//! \~english Returns current binlog file split records count
 	//! \~russian Возвращает текущее количество записей для разделения файла бинарного лога
 	int splitRecordCount() const { return split_count; }

 	//! \~english Returns if rapid start enabled
 	//! \~russian Возвращает, включено ли быстрое начало
 	bool rapidStart() const { return rapid_start; }

 	//! \~english Returns if index creates while writing
 	//! \~russian Возвращает, создается ли индекс во время записи
 	bool createIndexOnFly() const { return create_index_on_fly; }

	//! Create binlog file with Filename = path
	void createNewFile(const PIString & path);

	//! Set \a PlayMode
	void setPlayMode(PlayMode mode) { setProperty("playMode", (int)mode); }

	//! Set \a SplitMode
	void setSplitMode(SplitMode mode) { setProperty("splitMode", (int)mode); }

	//! Set path to directory where binlogs will be saved
	void setLogDir(const PIString & path) { setProperty("logDir", path); }

	//! Set file prefix, used to
	void setFilePrefix(const PIString & prefix) { setProperty("filePrefix", prefix); }

	//! Set defaultID, used in \a write function
	void setDefaultID(int id) { setProperty("defaultID", id); }

	//! If enabled BinLog \a ThreadedRead starts without delay for first record, i.e. first record will be readed immediately
	void setRapidStart(bool enabled) { setProperty("rapidStart", enabled); }

	//! Set index creation while writing
	void setCreateIndexOnFly(bool yes);

	//! Set play speed to "speed", default value is 1.0x
	//! Also this function set \a playMode to \a PlayVariableSpeed
	void setPlaySpeed(double speed) {
		setPlayMode(PlayVariableSpeed);
		setProperty("playSpeed", speed);
	}

	//! Setting static delay between records, default value is 1 sec
	//! Also this function set \a playMode to \a PlayStaticDelay
	void setPlayDelay(const PISystemTime & delay) {
		setPlayMode(PlayStaticDelay);
		setProperty("playDelay", delay);
	}

	//! Set \a playMode to \a PlayRealTime
	void setPlayRealTime() { setPlayMode(PlayRealTime); }

	//! Set binlog file split time
	//! Also this function set \a splitMode to \a SplitTime
	void setSplitTime(const PISystemTime & time) {
		setSplitMode(SplitTime);
		setProperty("splitTime", time);
	}

	//! Set binlog file split size
	//! Also this function set \a splitMode to \a SplitSize
	void setSplitFileSize(llong size) {
		setSplitMode(SplitSize);
		setProperty("splitFileSize", size);
	}

	//! Set binlog file split records count
	//! Also this function set \a splitMode to \a SplitCount
	void setSplitRecordCount(int count) {
		setSplitMode(SplitCount);
		setProperty("splitRecordCount", count);
	}

	//! Set pause while playing via \a threadedRead or writing via write
	void setPause(bool pause);

	//! Set function wich returns new binlog file path when using split mode.
	//! Overrides internal file path generator (logdir() + prefix() + current_time()).
	//! To restore internal file path generator set this function to "nullptr".
	void setFuncGetNewFilePath(std::function<PIString()> f) { f_new_path = f; }

	//! Write one record to BinLog file, with ID = id, id must be greather than 0
	int writeBinLog(int id, PIByteArray data) { return writeBinLog(id, data.data(), data.size_s()); }

	//! Write one record to BinLog file, with ID = id, id must be greather than 0
	int writeBinLog(int id, const void * data, int size);

	//! Write one RAW record to BinLog file, with ID = id, Timestamp = time
	int writeBinLog_raw(int id, const PISystemTime & time, const PIByteArray & data) {
		return writeBinLog_raw(id, time, data.data(), data.size_s());
	}
	int writeBinLog_raw(int id, const PISystemTime & time, const void * data, int size);

	//! Returns count of writed records
	int writeCount() const { return write_count; }

	//! Read one record from BinLog file, with ID = id, if id = 0 than any id will be readed
	PIByteArray readBinLog(int id = 0, PISystemTime * time = 0, int * readed_id = 0);

	//! Read one record from BinLog file, with ID = id, if id = 0 than any id will be readed
	int readBinLog(int id, void * read_to, int max_size, PISystemTime * time = 0, int * readed_id = 0);

	//! Returns binary log file size
	llong logSize() const { return log_size; }

	//! Return position in current binlog file
	llong logPos() const { return file.pos(); }

	//! Return true, if position at the end of BinLog file
	bool isEnd() const {
		if (isClosed()) return true;
		return file.isEnd();
	}

	//! Returns if BinLog file is empty
	bool isEmpty() const;

	//! Returns BinLog pause status
	bool isPause() const { return is_pause; }

	//! Returns id of last readed record
	int lastReadedID() const { return lastrecord.id; }

	//! Returns timestamp of last readed record
	PISystemTime lastReadedTimestamp() const { return lastrecord.timestamp; }

	//! Returns timestamp of log start
	PISystemTime logStartTimestamp() const { return startlogtime; }

	//! Set custom file header, you can get it back when read this binlog
	void setHeader(const PIByteArray & header);

	//! Get custom file header
	PIByteArray getHeader() const;

#ifdef DOXYGEN
	//! Read one message from binlog file, with ID contains in "filterID" or any ID, if "filterID" is empty
	int read(void * read_to, int max_size);

	//! Write one record to BinLog file, with ID = "defaultID"
	int write(const void * data, int size);
#endif

	//! Array of ID, that BinLog can read from binlog file, when use \a read function, or in \a ThreadedRead
	PIVector<int> filterID;

	//! Go to begin of BinLog file
	void restart();

	//! Get binlog info \a BinLogInfo
	BinLogInfo logInfo() const {
		if (is_indexed) return index.info;
		return getLogInfo(path());
	}

	//! Get binlog index \a BinLogIndex, need \a createIndex before getting index
	const PIVector<BinLogIndex> & logIndex() const { return index.index; }

	//! Create index of current binlog file
	bool createIndex();

	//! Return if current binlog file is indexed
	bool isIndexed() { return is_indexed; }

	//! Find nearest record of time \"time\". Returns -1 if not indexed or time less than first record
	int posForTime(const PISystemTime & time);

	//! Go to record #index
	void seekTo(int rindex);

	//! Go to nearest record
	bool seek(const PISystemTime & time);

	//! Set position in file to reading/playing
	bool seek(llong filepos);

	//! Get current record index (position record in file)
	int pos() const;

	PIByteArray saveIndex() const;
	bool loadIndex(PIByteArray saved);

	//! \handlers
	//! \{

	//! \fn PIString createNewFile()
	//! \brief Create new binlog file in \a logDir, if successful returns filename, else returns empty string.
	//! Filename is like  \a filePrefix + "yyyy_MM_dd__hh_mm_ss.binlog"

	//! \}
	//! \events
	//! \{

	//! \fn void fileEnd()
	//! \brief Raise on file end while reading

	//! \fn void fileError()
	//! \brief Raise on file creation error

	//! \fn void newFile(const PIString & filename)
	//! \brief Raise on new file created

	//! \}

	EVENT_HANDLER(PIString, createNewFile);
	EVENT(fileEnd);
	EVENT(fileError);
	EVENT1(newFile, const PIString &, filename);
	EVENT1(posChanged, int, pos);
	EVENT3(threadedReadRecord, PIByteArray, data, int, id, PISystemTime, time);

	//! Get binlog info and statistic
	static BinLogInfo getLogInfo(const PIString & path);

	//! Create new binlog from part of "src" with allowed IDs and "from" to "to" file position
	static bool cutBinLog(const BinLogInfo & src, const PIString & dst, int from, int to);

	//! Create new binlog from serial splitted binlogs "src"
	static bool joinBinLogsSerial(const PIStringList & src,
	                              const PIString & dst,
	                              std::function<bool(const PIString &, PISystemTime)> progress = nullptr);

protected:
	PIString constructFullPathDevice() const override;
	void configureFromFullPathDevice(const PIString & full_path) override;
	PIPropertyStorage constructVariantDevice() const override;
	void configureFromVariantDevice(const PIPropertyStorage & d) override;
	ssize_t readDevice(void * read_to, ssize_t max_size) override;
	ssize_t writeDevice(const void * data, ssize_t size) override;
	bool openDevice() override;
	bool closeDevice() override;
	void propertyChanged(const char * s) override;
	bool threadedRead(const uchar * readed, ssize_t size) override;
	DeviceInfoFlags deviceInfoFlags() const override { return PIIODevice::Reliable; }

private:
	struct Record {
		int id;
		int size;
		PISystemTime timestamp;
		PIByteArray data;
	};
	struct CompleteIndex {
		void clear();
		void makeIndexPos();
		BinLogInfo info;
		PIVector<BinLogIndex> index;
		PIMap<llong, int> index_pos;
	};
	BINARY_STREAM_FRIEND(CompleteIndex)

	bool writeFileHeader();
	bool checkFileHeader();
	Record readRecord();
	int writeRecord(int id, PISystemTime time, const void * data, int size);
	static void parseLog(PIFile * f, BinLogInfo * info, PIVector<BinLogIndex> * index);
	void moveIndex(int i);
	static PIString getLogfilePath(const PIString & log_dir, const PIString & prefix);

	CompleteIndex index;
	PlayMode play_mode;
	SplitMode split_mode;
	PIFile file;
	Record lastrecord;
	PISystemTime startlogtime, play_delay, split_time, pause_time;
	mutable PIMutex logmutex, pausemutex;
	double play_time, play_speed;
	llong split_size, log_size;
	int write_count, split_count, default_id, current_index;
	bool is_started, is_thread_ok, is_indexed, rapid_start, is_pause, create_index_on_fly;
	PIByteArray user_header;
	std::function<PIString()> f_new_path;
};


BINARY_STREAM_WRITE(PIBinaryLog::BinLogInfo) {
	PIChunkStream cs;
	cs.add(1, v.path)
		.add(2, v.records_count)
		.add(3, v.log_size)
		.add(4, v.start_time)
		.add(5, v.end_time)
		.add(6, v.records)
		.add(7, v.user_header);
	s << cs.data();
	return s;
}

BINARY_STREAM_READ(PIBinaryLog::BinLogInfo) {
	PIByteArray csba;
	s >> csba;
	PIChunkStream cs(csba);
	while (!cs.atEnd()) {
		switch (cs.read()) {
		case 1: cs.get(v.path); break;
		case 2: cs.get(v.records_count); break;
		case 3: cs.get(v.log_size); break;
		case 4: cs.get(v.start_time); break;
		case 5: cs.get(v.end_time); break;
		case 6: cs.get(v.records); break;
		case 7: cs.get(v.user_header); break;
		}
	}
	return s;
}


BINARY_STREAM_WRITE(PIBinaryLog::CompleteIndex) {
	s << v.info << v.index;
	return s;
}

BINARY_STREAM_READ(PIBinaryLog::CompleteIndex) {
	s >> v.info >> v.index;
	return s;
}


//! \relatesalso PICout \brief Output operator PIBinaryLog::BinLogInfo to PICout
inline PICout operator<<(PICout s, const PIBinaryLog::BinLogInfo & bi) {
	s.space();
	s.saveAndSetControls(0);
	s << "[PIBinaryLog] " << bi.path << "\n";
	if (bi.log_size < 0) {
		s << "invalid file path";
		s.restoreControls();
		return s;
	}
	if (bi.log_size == 0) {
		s << "Invalid empty file";
		s.restoreControls();
		return s;
	}
	if (bi.records_count < 0 && bi.records_count > -4) {
		s << "Invalid file or corrupted signature";
		s.restoreControls();
		return s;
	}
	if (bi.records_count < -3) {
		s << "Invalid binlog version";
		s.restoreControls();
		return s;
	}
	s << "read records " << bi.records_count << " in " << bi.records.size() << " types, log size " << bi.log_size;
	s << "\nlog start " << bi.start_time << " , log end " << bi.end_time;
	const auto keys = bi.records.keys();
	for (int i: keys) {
		const auto & bri(bi.records.at(i));
		s << "\n  record id " << bri.id << " , count " << bri.count;
		s << "\n    record start " << bri.start_time << " , end " << bri.end_time;
		s << "\n    record size " << bri.minimum_size << " - " << bri.maximum_size;
	}
	s.restoreControls();
	return s;
}

#endif // PIBINARYLOG_H