nosql-stream.js

// Total.js NoSQL FileStream
// The MIT License
// Copyright 2020-2023 (c) Peter Širka <petersirka@gmail.com>

'use strict';

const BUFFERSIZE = 1024 * 32;
const BUFFERDOCS = 15;
const NEWLINEBUFFER = Buffer.from('\n', 'utf8');
const DEFSTATS = { size: 0 };

function NoSQLStream(filename) {
	this.filename = filename;
	this.fd = null;
	this.stats = DEFSTATS;
	this.type = null;
	this.bytesread = 0;
	this.ticks = 0;
	this.position = 0;
	this.cache = [null, null];
	this.buffer = null;
	this.divider = ',';
	this.remchar = '-';
	this.buffercount = BUFFERDOCS;
	this.buffersize = BUFFERSIZE;
	this.linesize = 0;
	this.start = 0;
	this.indexer = 0;
	// this.canceled = false;
	// this.docs = '';
	// this.docscount = 0;
}

// Because of performance
NoSQLStream.prototype.readhelpers = function() {

	var self = this;

	self.cb_read = () => self.read();

	self.cb_readbuffer = function(err, size, chunk) {

		self.position += size;

		var beg = 0;

		if (self.buffer) {
			self.cache[0] = self.buffer;
			self.cache[1] = chunk;

			beg = self.buffer.length - 1;

			if (beg < 0)
				beg = 0;

			self.buffer = Buffer.concat(self.cache);

		} else
			self.buffer = chunk;

		var index = self.buffer.lastIndexOf(NEWLINEBUFFER);
		if (index === -1) {
			self.read();
			return;
		}

		var tmp = self.buffer.toString('utf8', 0, index).split('\n');
		for (var i = 0; i < tmp.length; i++) {
			if (tmp[i][0] !== self.remchar) {
				if (self.array)
					self.docs.push(tmp[i]);
				else
					self.docs += (self.docs ? self.divider : '') + tmp[i];
				self.docscount++;
				self.indexer++;
			}
		}

		self.buffer = self.buffer.slice(index + 1);

		if (self.ondocuments() === false)
			self.canceled = true;

		self.docs = self.array ? [] : '';
		self.docscount = 0;
		self.ticks++;

		if (self.ticks % 5 === 0)
			setImmediate(self.cb_readticks);
		else
			self.read();
	};

	self.cb_readticks = () => self.read();
	self.cb_readreverse = () => self.readreverse2();

	self.cb_readreversebuffer = function(err, size, chunk) {

		self.bytesread += size;

		if (self.buffer) {
			self.cache[0] = chunk;
			self.cache[1] = self.buffer;
			self.buffer = Buffer.concat(self.cache);
		} else
			self.buffer = chunk;

		var index = self.buffer.indexOf(NEWLINEBUFFER);
		if (index === -1) {
			self.readreverse2();
			return;
		}

		var tmp = self.buffer.toString('utf8', index).trim().split('\n');
		for (var i = 0; i < tmp.length; i++) {
			if (tmp[i][0] !== self.remchar) {
				if (self.array)
					self.docs.push(tmp[i]);
				else
					self.docs += (self.docs ? self.divider : '') + tmp[i];
				self.docscount++;
				self.indexer++;
			}
		}

		if (self.ondocuments() === false)
			self.canceled = true;

		self.buffer = self.buffer.slice(0, index + 1);
		self.docs = self.array ? [] : '';
		self.docscount = 0;
		self.ticks++;

		if (self.ticks % 5 === 0)
			setImmediate(self.cb_readreverseticks);
		else
			self.readreverse2();
	};

	self.cb_readreverseticks = () => self.readreverse2();

	self.cb_readstream = function(chunk) {

		if (self.canceled)
			return;

		var beg = 0;

		if (self.buffer) {

			self.cache[0] = self.buffer;
			self.cache[1] = chunk;
			self.buffer = Buffer.concat(self.cache);
			beg = self.cache[0].length - 1;

			if (beg < 0)
				beg = 0;
		} else
			self.buffer = chunk;

		var index = self.buffer.lastIndexOf(NEWLINEBUFFER);
		if (index === -1)
			return;

		var tmp = self.buffer.toString('utf8', 0, index).split('\n');
		for (var i = 0; i < tmp.length; i++) {
			if (tmp[i][0] !== self.remchar) {
				if (self.array)
					self.docs.push(tmp[i]);
				else
					self.docs += (self.docs ? self.divider : '') + tmp[i];
				self.docscount++;
				self.indexer++;
			}
		}

		self.buffer = self.buffer.slice(index + 1);

		if (self.ondocuments() === false)
			self.canceled = true;

		if (self.canceled) {
			self.stream.destroy && self.stream.destroy();
		} else {
			self.docs = self.array ? [] : '';
			self.docscount = 0;
			self.ticks++;
		}
	};

};

// Because of performance
NoSQLStream.prototype.writehelpers = function() {

	var self = this;

	self.cb_writeAddUpdAdd = function(err, size) {
		if (err) {
			console.log('ERROR --> TextDBStreamer.writer (add)', err);
			self.canceled = true;
			self.bufferstacknew.length = 0;
			self.bufferstack.length = 0;
			self.writing = false;
		} else {
			self.positionappend += size;
			var item = self.bufferstack.shift();
			F.Fs.write(self.fd, item.data, item.position, 'utf8', self.cb_writeAddUpdUpd);
		}
	};

	self.cb_writeAddUpdUpd = function(err) {

		self.writing = false;

		if (err) {
			console.log('ERROR --> TextDBStreamer.writer (upd)', err);
			self.canceled = true;
			self.bufferstack.length = 0;
			self.bufferstacknew.length = 0;
		} else
			self.$write();
	};

	self.cb_writeAdd = function(err, size) {

		self.writing = false;
		self.positionappend += size;

		if (err) {
			console.log('ERROR --> TextDBStreamer.writer (add)', err);
			self.canceled = true;
		} else
			self.$write();
	};

	self.cb_writeUpd = function(err) {

		self.writing = false;

		if (err) {
			console.log('ERROR --> TextDBStreamer.writer (upd)', err);
			self.canceled = true;
		} else
			self.$write();
	};

	self.cb_flush = () => self.flush();

	self.cb_readwritebuffer = function(err, size, chunk) {

		self.position += size;

		var beg = 0;
		var index;

		if (self.buffer) {
			self.cache[0] = self.buffer;
			self.cache[1] = chunk;
			beg = self.buffer.length - 1;
			if (beg < 0)
				beg = 0;
			self.buffer = Buffer.concat(self.cache);
		} else
			self.buffer = chunk;


		var index = self.buffer.lastIndexOf(NEWLINEBUFFER);
		if (index === -1) {
			self.readupdate();
			return;
		}

		var tmp = self.buffer.toString('utf8', 0, index).split('\n');
		for (var i = 0; i < tmp.length; i++) {
			var size = Buffer.byteLength(tmp[i], 'utf8');
			if (tmp[i][0] !== self.remchar) {
				if (self.array)
					self.docs.push(tmp[i]);
				else
					self.docs += (self.docs ? self.divider : '') + tmp[i];
				self.docsbuffer.push({ length: size, doc: tmp[i], position: self.positionupdate });
				self.docscount++;
				self.indexer++;
			}
			self.positionupdate += size + 1;
		}

		if (self.ondocuments() === false)
			self.canceled = true;

		self.docsbuffer = [];
		self.docs = self.array ? [] : '';
		self.buffer = self.buffer.slice(index + 1);

		if (self.bufferstack.length || self.bufferstacknew.length)
			setImmediate(self.cb_writeticks);
		else
			self.readupdate();
	};

	self.cb_writeticks = function() {
		if (self.bufferstack.length || self.bufferstacknew.length)
			setImmediate(self.cb_writeticks);
		else
			self.readupdate();
	};
};

NoSQLStream.prototype.openread = function() {
	var self = this;
	self.type = 'r';
	self.position = self.start;
	self.open();
	return self;
};

NoSQLStream.prototype.openreadreverse = function() {
	var self = this;
	self.type = 'r';
	self.position = self.start;
	self.$reverse = true;
	self.open();
	return self;
};

NoSQLStream.prototype.openupdate = function() {
	var self = this;
	self.type = 'r+';
	F.Fs.open(self.filename, self.type, function(err, fd) {

		if (err) {
			self.$callback(err);
			return;
		}

		F.Fs.fstat(fd, function(err, stats) {

			self.docs = self.array ? [] : '';
			self.docscount = 0;

			if (err) {
				F.Fs.close(fd, NOOP);
				self.$callback(err);
				return;
			}

			self.docs = self.array ? [] : '';
			self.docscount = 0;
			self.fd = fd;
			self.stats = stats;
			self.position = self.start;
			self.positionappend = self.stats.size;
			self.positionupdate = self.start;
			self.bufferstack = [];
			self.bufferstacknew = [];
			self.docsbuffer = [];
			self.writehelpers();
			self.readupdate();
		});
	});

	return self;
};

// For e.g. files on URL address
NoSQLStream.prototype.openstream = function(stream) {

	var self = this;

	var close = function() {
		if (self.docscount) {
			self.ondocuments();
			self.docscount = 0;
			self.docs = self.array ? [] : '';
		}
		self.$callback && self.$callback();
		self.$callback = null;
	};

	self.docs = self.array ? [] : '';
	self.docscount = 0;
	self.readhelpers();
	self.stream = stream;
	self.stream.on('error', close);
	self.stream.on('end', close);
	self.stream.on('data', self.cb_readstream);
	return self;
};

NoSQLStream.prototype.open = function() {
	var self = this;
	F.Fs.open(self.filename, self.type, function(err, fd) {

		if (err) {
			self.$callback(err);
			return;
		}

		F.Fs.fstat(fd, function(err, stats) {
			self.docs = self.array ? [] : '';
			self.docscount = 0;
			self.fd = fd;
			self.stats = stats;
			self.position = self.start;

			if (err) {
				F.Fs.close(fd, NOOP);
				self.$callback(err);
				return;
			}

			self.readhelpers();

			if (self.$reverse)
				self.readreverse();
			else
				self.read();
		});
	});
};

NoSQLStream.prototype.close = function() {

	var self = this;

	if (self.fd) {

		self.stream = null;

		F.Fs.close(self.fd, function(err) {
			err && F.error(err);
			self.$callback && self.$callback();
		});

		if (self.buffer) {
			self.buffer = null;
			self.cache[0] = null;
			self.cache[1] = null;
			self.bytesread = 0;
		}

		self.canceled = false;
		self.fd = null;
		self.type = null;
		self.docscache = null;
		self.docs = null;

	} else if (self.$callback)
		self.$callback && self.$callback();

	return self;
};

NoSQLStream.prototype.replace = function(index, value, divider) {

	var self = this;
	var rec = self.docsbuffer[index];

	if (rec.doc === value)
		return;

	var was = true;
	var sep = (divider || self.divider);

	if (rec.doc.length === value.length) {
		var b = Buffer.byteLength(value);
		if (rec.length === b) {
			self.write(value + sep, rec.position);
			was = false;
		}
	}

	if (was) {
		var tmp = self.remchar + rec.doc.substring(1) + sep;
		self.write(tmp, rec.position);
		self.write2(value + sep);
	}

	return self;
};

NoSQLStream.prototype.write = function(doc, position) {
	var self = this;
	self.bufferstack.push({ position: position, data: doc });
	!self.writing && self.$write();
	return self;
};

NoSQLStream.prototype.write2 = function(doc) {
	var self = this;
	self.bufferstacknew.push(Buffer.from(doc));
	!self.writing && self.$write();
	return self;
};

NoSQLStream.prototype.$write = function() {
	var self = this;
	if (self.bufferstacknew.length && self.bufferstack.length) {
		self.writing = true;
		var buf = self.bufferstacknew.splice(0, 5);
		buf = buf.length > 1 ? Buffer.concat(buf) : buf[0];
		F.Fs.write(self.fd, buf, 0, buf.length, self.positionappend, self.cb_writeAddUpdAdd);
	} else if (self.bufferstacknew.length) {
		self.writing = true;
		var buf = self.bufferstacknew.splice(0, 5);
		buf = buf.length > 1 ? Buffer.concat(buf) : buf[0];
		F.Fs.write(self.fd, buf, 0, buf.length, self.positionappend, self.cb_writeAdd);
	} else if (self.bufferstack.length) {
		self.writing = true;
		var item = self.bufferstack.shift();
		F.Fs.write(self.fd, item.data, item.position, 'utf8', self.cb_writeUpd);
	}
};

NoSQLStream.prototype.flush = function() {
	var self = this;
	if (self.writing)
		setTimeout(self.cb_flush, 100);
	else
		self.close();
	return self;
};

NoSQLStream.prototype.read = function() {

	var self = this;
	var size = self.stats.size - self.position;


	if (!self.fd || size <= 0 || self.canceled) {

		if (!self.canceled && self.buffer && self.buffer.length) {
			self.cb_readbuffer(null, 1, NEWLINEBUFFER);
			self.buffer = Buffer.alloc(0);
			return;
		}

		if (self.docscount) {
			self.ondocuments();
			self.docscount = 0;
			self.docs = self.array ? [] : '';
		}

		self.close();

	} else {
		size = size < self.buffersize ? size : self.buffersize;
		var buffer = Buffer.alloc(size);
		F.Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readbuffer);
	}
};

NoSQLStream.prototype.readreverse = function() {
	var self = this;
	self.position = self.stats.size;
	self.readreverse2();
	return self;
};

NoSQLStream.prototype.readreverse2 = function() {
	var self = this;

	if (!self.fd || self.position <= self.start || self.canceled) {

		if (!self.canceled && self.buffer && self.buffer.length) {
			self.cb_readreversebuffer(null, 1, NEWLINEBUFFER);
			self.buffer = Buffer.alloc(0);
			return;
		}

		if (self.docscount) {
			self.ondocuments();
			self.docs = self.array ? [] : '';
			self.docscount = 0;
		}

		self.close();

	} else {
		var size = self.stats.size - self.bytesread;
		size = size < self.buffersize ? size : self.buffersize;
		self.position -= size;
		var buffer = Buffer.alloc(size);
		F.Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readreversebuffer);
	}
};

NoSQLStream.prototype.readupdate = function() {

	var self = this;
	var size = self.stats.size - self.position;

	if (!self.fd || size <= 0 || self.canceled) {

		if (!self.canceled && self.buffer && self.buffer.length) {
			self.positionappend++;
			self.cb_readwritebuffer(null, 1, NEWLINEBUFFER);
			self.buffer = Buffer.alloc(0);
			return;
		}

		if (self.docsbuffer.length) {
			self.ondocuments();
			self.docsbuffer = [];
			self.docs = self.array ? [] : '';
		}

		self.flush();
	} else {
		size = size < self.buffersize ? size : self.buffersize;
		var buffer = Buffer.alloc(size);
		F.Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readwritebuffer);
	}
};

module.exports = NoSQLStream;