From 7e2a8f04e6e878d74b6ece399d31702972ac8f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Feb 2018 10:06:55 +0100 Subject: [PATCH 0001/1669] Added async components rendering. --- changes.txt | 4 ++ index.js | 196 ++++++++++++++++++++++++++++++++++++++++----------- internal.js | 36 +++++++++- package.json | 2 +- 4 files changed, 194 insertions(+), 44 deletions(-) mode change 100755 => 100644 index.js diff --git a/changes.txt b/changes.txt index 997765649..91cf07f85 100755 --- a/changes.txt +++ b/changes.txt @@ -1,3 +1,7 @@ +======= 3.0.0 + +- added: (IMPORTANT) Total.js components can have async delegate + ======= 2.9.3 (HOTFIX) - added: `String.arg(obj)` for a simple templating `Hello {variable}!` diff --git a/index.js b/index.js old mode 100755 new mode 100644 index 6fa58a8d6..9ad945f44 --- a/index.js +++ b/index.js @@ -526,8 +526,8 @@ var PERF = {}; function Framework() { this.$id = null; // F.id ==> property - this.version = 2920; - this.version_header = '2.9.2'; + this.version = 3000; + this.version_header = '3.0.0'; this.version_node = process.version.toString().replace('v', '').replace(/\./g, '').parseFloat(); this.config = { @@ -3637,6 +3637,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe obj.$owner = _owner; F.temporary.owners[_owner] = true; _controller = ''; + obj.name = name; F.components.instances[name] = obj; obj && typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); } catch(e) { @@ -3648,6 +3649,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe _owner = (packageName ? packageName + '@' : '') + type + '#' + name; F.temporary.owners[_owner] = true; obj = require(js); + obj.name = name; obj.$owner = _owner; _controller = ''; F.components.instances[name] = obj; @@ -9780,14 +9782,15 @@ Controller.prototype.getSchema = function() { * Renders component * @param {String} name A component name * @param {Object} settings Optional, settings. + * @model {Object} settings Optional, model for the component. * @return {String} */ -Controller.prototype.component = function(name, settings) { +Controller.prototype.component = function(name, settings, model) { var filename = F.components.views[name]; if (filename) { var generator = framework_internal.viewEngine(name, filename, this); if (generator) - return generator.call(this, this, this.repository, this.$model, this.session, this.query, this.body, this.url, F.global, F.helpers, this.user, this.config, F.functions, 0, this.outputPartial, this.req.cookie, this.req.files, this.req.mobile, settings || EMPTYOBJECT); + return generator.call(this, this, this.repository, model || this.$model, this.session, this.query, this.body, this.url, F.global, F.helpers, this.user, this.config, F.functions, 0, this.outputPartial, this.req.files, this.req.mobile, settings || EMPTYOBJECT); } return ''; }; @@ -9800,8 +9803,16 @@ Controller.prototype.$components = function(group, settings) { for (var i = 0, length = keys.length; i < length; i++) { var component = F.components.instances[keys[i]]; if (component.group === group) { - var tmp = this.component(keys[i], settings); - tmp && output.push(tmp); + if (component.render) { + !this.$viewasync && (this.$viewasync = []); + $VIEWASYNC++; + var name = '@{-' + $VIEWASYNC + '-}'; + this.$viewasync.push({ replace: name, name: component.name, settings: settings }); + output.push(name); + } else { + var tmp = this.component(keys[i], settings); + tmp && output.push(tmp); + } } } return output.join('\n'); @@ -10515,22 +10526,6 @@ Controller.prototype.$templateToggle = function(visible, name, model, expire, ke }; Controller.prototype.$view = function(name, model, expire, key) { - return this.$viewToggle(true, name, model, expire, key); -}; - -Controller.prototype.$viewCompile = function(body, model, key) { - var self = this; - var layout = self.layoutName; - self.layoutName = ''; - var value = self.viewCompile(body, model, null, true, key); - self.layoutName = layout; - return value || ''; -}; - -Controller.prototype.$viewToggle = function(visible, name, model, expire, key) { - - if (!visible) - return ''; var self = this; var cache; @@ -10538,22 +10533,34 @@ Controller.prototype.$viewToggle = function(visible, name, model, expire, key) { if (expire) { cache = '$view.' + name + '.' + (key || ''); var output = self.cache.read2(cache); - if (output) - return output; + if (output) { + if (output.components) + self.$hasComponents = true; + return output.body; + } } - var layout = self.layoutName; - self.layoutName = ''; - var value = self.view(name, model, null, true); - self.layoutName = layout; - + var value = self.view(name, model, null, true, true, cache); if (!value) return ''; - expire && self.cache.add(cache, value, expire, false); + expire && self.cache.add(cache, { components: value instanceof Function, body: value instanceof Function ? '' : value }, expire, false); return value; }; +Controller.prototype.$viewCompile = function(body, model, key) { + var self = this; + var layout = self.layoutName; + self.layoutName = ''; + var value = self.viewCompile(body, model, null, true, key); + self.layoutName = layout; + return value || ''; +}; + +Controller.prototype.$viewToggle = function(visible, name, model, expire, key, async) { + return visible ? this.$view(name, model, expire, key, async) : ''; +}; + /** * Adds a place into the places. * @param {String} name A place name. @@ -12249,7 +12256,7 @@ Controller.prototype.proxy2 = function(url, callback, headers, timeout) { * @param {Boolean} isPartial When is `true` the method returns rendered HTML as `String` * @return {Controller/String} */ -Controller.prototype.view = function(name, model, headers, partial) { +Controller.prototype.view = function(name, model, headers, partial, noasync, cachekey) { var self = this; @@ -12360,7 +12367,7 @@ Controller.prototype.view = function(name, model, headers, partial) { } } - return self.$viewrender(filename, framework_internal.viewEngine(name, filename, self), model, headers, partial, isLayout); + return self.$viewrender(filename, framework_internal.viewEngine(name, filename, self), model, headers, partial, isLayout, noasync, cachekey); }; Controller.prototype.viewCompile = function(body, model, headers, partial, key) { @@ -12380,7 +12387,7 @@ Controller.prototype.viewCompile = function(body, model, headers, partial, key) return this.$viewrender('[dynamic view]', framework_internal.viewEngineCompile(body, this.language, this, key), model, headers, partial); }; -Controller.prototype.$viewrender = function(filename, generator, model, headers, partial, isLayout) { +Controller.prototype.$viewrender = function(filename, generator, model, headers, partial, isLayout, noasync, cachekey) { var self = this; var err; @@ -12412,7 +12419,7 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, var helpers = F.helpers; try { - value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.cookie, self.req.files, self.req.mobile, EMPTYOBJECT); + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); } catch (ex) { err = new Error('View "' + filename + '": ' + ex.message); @@ -12433,7 +12440,105 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, return value; } - if (!isLayout && self.precache && (!self.status || self.status === 200) && !partial) + // noasync = true --> rendered inline view in view + + if (self.$viewasync && self.$viewasync.length) { + + var can = ((isLayout || !self.layoutName) && noasync !== true) || !!cachekey; + if (can) { + var done = {}; + var obj = {}; + + obj.repository = self.repository; + obj.model = self.$model; + obj.user = self.user; + obj.session = self.session; + obj.controller = self; + obj.query = self.query; + obj.body = self.body; + obj.files = self.files; + + self.$viewasync.waitFor(function(item, next) { + + if (item.value) { + value = value.replace(item.replace, item.value); + if (isLayout && self.precache) + self.output = self.output.replace(item.replace, item.value); + return next(); + } + + obj.options = obj.settings = item.settings; + obj.next = obj.callback = function(model) { + item.value = self.component(item.name, item.settings, model); + value = value.replace(item.replace, item.value); + if (isLayout && self.precache) + self.output = self.output.replace(item.replace, item.value); + next(); + }; + + F.components.instances[item.name].render(obj); + + }, function() { + + if (cachekey && F.cache.items[cachekey]) { + var cache = F.cache.items[cachekey].value; + cache.body = value; + cache.components = true; + } + + if (isLayout && self.precache && (!self.status || self.status === 200) && !partial) + self.precache(self.output, CT_HTML, headers, true); + + if (isLayout || !self.layoutName) { + + self.outputPartial = ''; + self.output = ''; + isLayout = false; + + if (partial) { + done.callback && done.callback(null, value); + return; + } + + self.req.$total_success(); + + if (!self.isConnected) + return self; + + var res = self.res; + res.options.body = value; + res.options.code = self.status || 200; + res.options.type = CT_HTML; + res.options.headers = headers; + res.$text(); + F.stats.response.view++; + return self; + } + + if (partial) + self.outputPartial = value; + else + self.output = value; + + if (!cachekey && !noasync) { + self.isLayout = true; + value = self.view(self.layoutName, self.$model, headers, partial); + } + + // Async + if (partial) { + self.outputPartial = ''; + self.isLayout = false; + done.callback && done.callback(null, value); + } + + }, 3); + + return cachekey ? value : (partial ? (fn => done.callback = fn) : self); + } + } + + if (!isLayout && self.precache && (!self.status || self.status === 200) && !partial && !self.$viewasync) self.precache(value, CT_HTML, headers, true); if (isLayout || !self.layoutName) { @@ -12465,9 +12570,12 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, else self.output = value; - self.isLayout = true; - value = self.view(self.layoutName, self.$model, headers, partial); + if (!cachekey && !noasync) { + self.isLayout = true; + value = self.view(self.layoutName, self.$model, headers, partial); + } + // Async if (partial) { self.outputPartial = ''; self.isLayout = false; @@ -12510,6 +12618,9 @@ Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { self.layoutName = output.layout; self.themeName = output.theme; + if (output.components) + self.$hasComponents = true; + var res = self.res; res.options.code = self.status || 200; @@ -12576,7 +12687,7 @@ Controller.prototype.$memorize_prepare = function(key, expires, disabled, fnTo, return; } - var options = { content: value, type: contentType || CT_TEXT, layout: self.layoutName, theme: self.themeName }; + var options = { content: value, type: contentType || CT_TEXT, layout: self.layoutName, theme: self.themeName, components: (self.$viewasync ? true : false) || (self.$hasComponents ? true : false) }; if (headers) options.headers = headers; @@ -15837,8 +15948,13 @@ function parseComponent(body, filename) { while (true) { beg = body.indexOf('', beg); if (end === -1) break; diff --git a/internal.js b/internal.js index da54f7e06..8de0104bc 100755 --- a/internal.js +++ b/internal.js @@ -85,6 +85,7 @@ global.$STRING = function(value) { }; global.$VIEWCACHE = []; +global.$VIEWASYNC = 0; exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { @@ -1927,7 +1928,7 @@ function view_parse(content, minify, filename, controller) { builder += '}$output+=$EMPTY'; } else { - tmp = view_prepare(command.command, newCommand, functionsName, controller); + tmp = view_prepare(command.command, newCommand, functionsName, controller, filename); var can = false; // Inline rendering is supported only in release mode @@ -1979,7 +1980,10 @@ function view_parse(content, minify, filename, controller) { } else if (tmp) { if (view_parse_plus(builder)) builder += '+'; - builder += wrapTryCatch(tmp, command.command, command.line); + if (tmp.substring(1, 4) !== '@{-' && tmp.substring(0, 11) !== 'self.$view') + builder += wrapTryCatch(tmp, command.command, command.line); + else + builder += tmp; } } @@ -1996,7 +2000,7 @@ function view_parse(content, minify, filename, controller) { if (RELEASE) builder = builder.replace(/(\+\$EMPTY\+)/g, '+').replace(/(\$output=\$EMPTY\+)/g, '$output=').replace(/(\$output\+=\$EMPTY\+)/g, '$output+=').replace(/(\}\$output\+=\$EMPTY)/g, '}').replace(/(\{\$output\+=\$EMPTY;)/g, '{').replace(/(\+\$EMPTY\+)/g, '+').replace(/(>'\+'<)/g, '><').replace(/'\+'/g, ''); - var fn = '(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,cookie,files,mobile,settings){var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;var cookie=function(name){return self.req.cookie(name)};' + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'; + var fn = '(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;var cookie=function(name){return self.req.cookie(name)};' + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'; try { fn = eval(fn); } catch (e) { @@ -2211,7 +2215,33 @@ function view_prepare(command, dynamicCommand, functions, controller) { return '(' + command + ')'; case 'component': + controller.$hasComponents = true; + tmp = command.indexOf('\''); + + var is = false; + if (tmp !== -1) { + name = command.substring(tmp + 1, command.indexOf('\'', tmp + 1)); + tmp = F.components.instances[name]; + if (tmp && tmp.render) + is = true; + } else { + tmp = command.indexOf('"'); + name = command.substring(tmp + 1, command.indexOf('"', tmp + 1)); + tmp = F.components.instances[name]; + if (tmp && tmp.render) + is = true; + } + + + if (is) { + var settings = command.substring(11 + name.length + 2, command.length - 1).trim(); + if (settings === ')') + settings = ''; + $VIEWASYNC++; + return '\'@{-{0}-}\'+(function(index){!controller.$viewasync&&(controller.$viewasync=[]);controller.$viewasync.push({replace:\'@{-{0}-}\',name:\'{1}\',settings:{2}});return $EMPTY})({0})'.format($VIEWASYNC, name, settings || 'null'); + } + return 'self.' + command; case 'routeJS': diff --git a/package.json b/package.json index 7029e822b..707a66823 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "2.9.3-6", + "version": "3.0.0-1", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From e8d951c8b35d0174d5fc54be2480fd63293c479e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Feb 2018 18:44:58 +0100 Subject: [PATCH 0002/1669] Improve `callback` in rendering components. --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 9ad945f44..c00e499dd 100644 --- a/index.js +++ b/index.js @@ -21,7 +21,7 @@ /** * @module Framework - * @version 2.9.3 + * @version 3.0.0 */ 'use strict'; @@ -12469,6 +12469,10 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, obj.options = obj.settings = item.settings; obj.next = obj.callback = function(model) { + + if (arguments.length > 1) + model = arguments[1]; + item.value = self.component(item.name, item.settings, model); value = value.replace(item.replace, item.value); if (isLayout && self.precache) From 6377ec5965bcb8958e845fa3af87fcefba7e3832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Feb 2018 18:45:03 +0100 Subject: [PATCH 0003/1669] Update version. --- internal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal.js b/internal.js index 8de0104bc..aadc2caf2 100755 --- a/internal.js +++ b/internal.js @@ -21,7 +21,7 @@ /** * @module FrameworkInternal - * @version 2.9.2 + * @version 3.0.0 */ 'use strict'; From 3f2f807b6af7abe161affa4299fdc56c859fe050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Feb 2018 09:06:45 +0100 Subject: [PATCH 0004/1669] Add new schema operations: `.setInsert()` and `.setUpdate()`. --- builders.js | 113 +++++++++++++++++++++++++++++++++++++++++++++++----- changes.txt | 1 + index.js | 26 ++++++++++++ 3 files changed, 130 insertions(+), 10 deletions(-) diff --git a/builders.js b/builders.js index 4c89b6de6..0fb3559cd 100755 --- a/builders.js +++ b/builders.js @@ -21,7 +21,7 @@ /** * @module FrameworkBuilders - * @version 2.9.3 + * @version 3.0.0 */ 'use strict'; @@ -195,6 +195,8 @@ function SchemaBuilderEntity(parent, name) { this.$onDefault; // Array of functions for inherits this.onValidate = F.onValidate; this.onSave; + this.onInsert; + this.onUpdate; this.onGet; this.onRemove; this.onQuery; @@ -360,6 +362,12 @@ SchemaBuilderEntity.prototype.inherit = function(group, name) { if (!self.onSave && schema.onSave) self.onSave = schema.onSave; + if (!self.onInsert && schema.onInsert) + self.onInsert = schema.onInsert; + + if (!self.onUpdate && schema.onUpdate) + self.onUpdate = schema.onUpdate; + if (!self.onGet && schema.onGet) self.onGet = schema.onGet; @@ -750,6 +758,29 @@ SchemaBuilderEntity.prototype.setSave = function(fn, description) { return this; }; +/** + * Set insert handler + * @param {Function(error, model, helper, next(value), controller)} fn + * @return {SchemaBuilderEntity} + */ +SchemaBuilderEntity.prototype.setInsert = function(fn, description) { + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); + this.onInsert = fn; + this.meta.insert = description; + return this; +}; + +/** + * Set update handler + * @param {Function(error, model, helper, next(value), controller)} fn + * @return {SchemaBuilderEntity} + */ +SchemaBuilderEntity.prototype.setUpdate = function(fn, description) { + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); + this.onUpdate = fn; + this.meta.update = description; + return this; +}; /** * Set error handler @@ -912,6 +943,8 @@ SchemaBuilderEntity.prototype.destroy = function() { this.$onDefault = undefined; this.onValidate = undefined; this.onSave = undefined; + this.onInsert = undefined; + this.onUpdate = undefined; this.onRead = undefined; this.onGet = undefined; this.onRemove = undefined; @@ -943,6 +976,37 @@ SchemaBuilderEntity.prototype.destroy = function() { * @return {SchemaBuilderEntity} */ SchemaBuilderEntity.prototype.save = function(model, options, callback, controller, skip) { + return this.execute('onSave', model, options, callback, controller, skip); +}; + +/** + * Execute onInsert delegate + * @param {Object} model + * @param {Object} options Custom options object, optional + * @param {Function(err, result)} callback + * @param {Controller} controller + * @param {Boolean} skip Skips preparing and validation, optional + * @return {SchemaBuilderEntity} + */ +SchemaBuilderEntity.prototype.insert = function(model, options, callback, controller, skip) { + return this.execute('onInsert', model, options, callback, controller, skip); +}; + +/** + * Execute onUpdate delegate + * @param {Object} model + * @param {Object} options Custom options object, optional + * @param {Function(err, result)} callback + * @param {Controller} controller + * @param {Boolean} skip Skips preparing and validation, optional + * @return {SchemaBuilderEntity} + */ +SchemaBuilderEntity.prototype.update = function(model, options, callback, controller, skip) { + return this.execute('onUpdate', model, options, callback, controller, skip); +}; + + +SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, controller, skip) { if (typeof(callback) === 'boolean') { skip = callback; @@ -963,7 +1027,19 @@ SchemaBuilderEntity.prototype.save = function(model, options, callback, controll callback = function(){}; var self = this; - var $type = 'save'; + var $type; + + switch (TYPE) { + case 'onInsert': + $type = 'insert'; + break; + case 'onUpdate': + $type = 'update'; + break; + default: + $type = 'save'; + break; + } self.$prepare(model, function(err, model) { @@ -979,13 +1055,13 @@ SchemaBuilderEntity.prototype.save = function(model, options, callback, controll self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); - if (!isGenerator(self, $type, self.onSave)) { - if (self.onSave.$newversion) - self.onSave(new SchemaOptions(builder, model, options, function(res) { + if (!isGenerator(self, $type, self[TYPE])) { + if (self[TYPE].$newversion) + self[TYPE](new SchemaOptions(builder, model, options, function(res) { self.$process(arguments, model, $type, undefined, builder, res, callback); }, controller)); else - self.onSave(builder, model, options, function(res) { + self[TYPE](builder, model, options, function(res) { self.$process(arguments, model, $type, undefined, builder, res, callback); }, controller, skip !== true); return self; @@ -1018,15 +1094,16 @@ SchemaBuilderEntity.prototype.save = function(model, options, callback, controll callback(has ? builder : null, res === undefined ? model : res); }; - if (self.onSave.$newversion) - async.call(self, self.onSave)(onError, new SchemaOptions(builder, model, options, onCallback, controller)); + if (self[TYPE].$newversion) + async.call(self, self[TYPE])(onError, new SchemaOptions(builder, model, options, onCallback, controller)); else - async.call(self, self.onSave)(onError, builder, model, options, onCallback, controller, skip !== true); + async.call(self, self[TYPE])(onError, builder, model, options, onCallback, controller, skip !== true); }); return self; }; + function isGenerator(obj, name, fn) { return obj.gcache[name] ? obj.gcache[name] : obj.gcache[name] = fn.toString().substring(0, 9) === 'function*'; } @@ -2407,7 +2484,7 @@ SchemaInstance.prototype.$push = function(type, name, helper, first) { var self = this; var fn; - if (type === 'save') { + if (type === 'save' || type === 'insert' || type === 'update') { helper = name; name = undefined; @@ -2499,6 +2576,22 @@ SchemaInstance.prototype.$save = function(helper, callback) { return this; }; +SchemaInstance.prototype.$insert = function(helper, callback) { + if (this.$$can && this.$$async) + this.$push('insert', helper); + else + this.$$schema.insert(this, helper, callback, this.$$controller); + return this; +}; + +SchemaInstance.prototype.$update = function(helper, callback) { + if (this.$$can && this.$$async) + this.$push('update', helper); + else + this.$$schema.update(this, helper, callback, this.$$controller); + return this; +}; + SchemaInstance.prototype.$query = function(helper, callback) { if (this.$$can && this.$$async) this.$push('query', helper); diff --git a/changes.txt b/changes.txt index 91cf07f85..85d8f5b71 100755 --- a/changes.txt +++ b/changes.txt @@ -1,6 +1,7 @@ ======= 3.0.0 - added: (IMPORTANT) Total.js components can have async delegate +- added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` ======= 2.9.3 (HOTFIX) diff --git a/index.js b/index.js index c00e499dd..7ae9e8b64 100644 --- a/index.js +++ b/index.js @@ -9662,6 +9662,32 @@ Controller.prototype.$save = function(helper, callback) { return self; }; +Controller.prototype.$insert = function(helper, callback) { + var self = this; + if (framework_builders.isSchema(self.body)) { + self.body.$$controller = self; + self.body.$insert(helper, callback); + } else { + var model = self.getSchema().default(); + model.$$controller = self; + model.$insert(helper, callback); + } + return self; +}; + +Controller.prototype.$update = function(helper, callback) { + var self = this; + if (framework_builders.isSchema(self.body)) { + self.body.$$controller = self; + self.body.$update(helper, callback); + } else { + var model = self.getSchema().default(); + model.$$controller = self; + model.$update(helper, callback); + } + return self; +}; + Controller.prototype.$remove = function(helper, callback) { var self = this; self.getSchema().remove(helper, callback, self); From 866ad077a62db56dde6bfaf9627f6a858175c5da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Feb 2018 08:47:45 +0100 Subject: [PATCH 0005/1669] Improved code. --- index.js | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/index.js b/index.js index 7ae9e8b64..353011f8f 100644 --- a/index.js +++ b/index.js @@ -378,30 +378,14 @@ global.$OPERATION = function(schema, name, options, callback, controller) { return !!o; }; -global.DB = global.DATABASE = function() { - return typeof(F.database) === 'object' ? F.database : F.database.apply(framework, arguments); -}; - -global.ON = function() { - return F.on.apply(F, arguments); +global.DB = global.DATABASE = function(a, b, c, d) { + return typeof(F.database) === 'object' ? F.database : F.database(a, b, c, d); }; global.OFF = function() { return arguments.length > 1 ? F.removeListener.apply(F, arguments) : F.removeAllListeners.apply(F, arguments); }; -global.EMIT = function() { - return F.emit.apply(F, arguments); -}; - -global.LOG = function() { - return F.log.apply(F, arguments); -}; - -global.LOGGER = function() { - return F.logger.apply(F, arguments); -}; - global.MAKE = global.TRANSFORM = function(transform, fn) { if (typeof(transform) === 'function') { @@ -868,7 +852,7 @@ F.prototypes = function(fn) { return F; }; -F.on = function(name, fn) { +global.ON = F.on = function(name, fn) { if (name === 'init' || name === 'ready' || name === 'load') { if (this.isLoaded) { @@ -908,7 +892,8 @@ F.on = function(name, fn) { return this; }; -F.emit = function(name, a, b, c, d, e, f, g) { +global.EMIT = F.emit = function(name, a, b, c, d, e, f, g) { + var evt = F.$events[name]; if (evt) { var clean = false; @@ -5131,7 +5116,7 @@ F.onMeta = function() { }; // @arguments {Object params} -F.log = function() { +global.LOG = F.log = function() { F.datetime = new Date(); var filename = F.datetime.getFullYear() + '-' + (F.datetime.getMonth() + 1).toString().padLeft(2, '0') + '-' + F.datetime.getDate().toString().padLeft(2, '0'); @@ -5155,7 +5140,7 @@ F.log = function() { return F; }; -F.logger = function() { +global.LOGGER = F.logger = function() { F.datetime = new Date(); var dt = F.datetime.getFullYear() + '-' + (F.datetime.getMonth() + 1).toString().padLeft(2, '0') + '-' + F.datetime.getDate().toString().padLeft(2, '0') + ' ' + F.datetime.getHours().toString().padLeft(2, '0') + ':' + F.datetime.getMinutes().toString().padLeft(2, '0') + ':' + F.datetime.getSeconds().toString().padLeft(2, '0'); var str = ''; @@ -10109,12 +10094,12 @@ Controller.prototype.cancel = function() { }; Controller.prototype.log = function() { - F.log.apply(framework, arguments); + F.log.apply(F, arguments); return this; }; Controller.prototype.logger = function() { - F.logger.apply(framework, arguments); + F.logger.apply(F, arguments); return this; }; From bbc6565e9e49ccf8cf74a051b5aaa336fc1362d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Feb 2018 09:01:20 +0100 Subject: [PATCH 0006/1669] Fixed `context`. --- index.js | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/index.js b/index.js index 353011f8f..46d81cb58 100644 --- a/index.js +++ b/index.js @@ -406,7 +406,7 @@ global.MAKE = global.TRANSFORM = function(transform, fn) { }; global.NEWTRANSFORM = function() { - return TransformBuilder.addTransform.apply(this, arguments); + return TransformBuilder.addTransform.apply(F, arguments); }; global.NEWSCHEMA = function(group, name) { @@ -855,31 +855,31 @@ F.prototypes = function(fn) { global.ON = F.on = function(name, fn) { if (name === 'init' || name === 'ready' || name === 'load') { - if (this.isLoaded) { - fn.call(this); + if (F.isLoaded) { + fn.call(F); return; } } else if (name.indexOf('#') !== -1) { var arr = name.split('#'); switch (arr[0]) { case 'middleware': - F.temporary.ready[name] && fn.call(this); + F.temporary.ready[name] && fn.call(F); break; case 'component': - F.temporary.ready[name] && fn.call(this); + F.temporary.ready[name] && fn.call(F); break; case 'model': - F.temporary.ready[name] && fn.call(this, F.models[arr[1]]); + F.temporary.ready[name] && fn.call(F, F.models[arr[1]]); break; case 'source': - F.temporary.ready[name] && fn.call(this, F.sources[arr[1]]); + F.temporary.ready[name] && fn.call(F, F.sources[arr[1]]); break; case 'package': case 'module': - F.temporary.ready[name] && fn.call(this, F.modules[arr[1]]); + F.temporary.ready[name] && fn.call(F, F.modules[arr[1]]); break; case 'controller': - F.temporary.ready[name] && fn.call(this, F.controllers[arr[1]]); + F.temporary.ready[name] && fn.call(F, F.controllers[arr[1]]); break; } } @@ -889,7 +889,7 @@ global.ON = F.on = function(name, fn) { else F.$events[name] = [fn]; - return this; + return F; }; global.EMIT = F.emit = function(name, a, b, c, d, e, f, g) { @@ -1527,9 +1527,9 @@ global.CORS = F.cors = function(url, flags, credentials) { F.group = function(flags, fn) { _flags = flags; - fn.call(this); + fn.call(F); _flags = undefined; - return this; + return F; }; /** @@ -2336,7 +2336,7 @@ global.MAP = F.map = function(url, filename, filter) { }); }, isPackage ? 500 : 1); - return this; + return F; }; /** @@ -3322,7 +3322,7 @@ F.$startup = function(callback) { }); }, callback); - return this; + return F; }; F.uptodate = function(type, url, options, interval, callback, next) { @@ -5162,7 +5162,7 @@ global.LOGGER = F.logger = function() { return F; }; -F.logmail = function(address, subject, body, callback) { +global.LOGMAIL = F.logmail = function(address, subject, body, callback) { if (typeof(body) === FUNCTION) { callback = body; @@ -7412,7 +7412,7 @@ F.include = function(name, options, callback) { * @param {String} language Optional. * @return {MailMessage} */ -F.mail = function(address, subject, view, model, callback, language) { +global.MAIL = F.mail = function(address, subject, view, model, callback, language) { if (typeof(callback) === 'string') { var tmp = language; @@ -16046,6 +16046,4 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; -global.LOGMAIL = F.logmail; -global.MAIL = F.mail; +global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file From 7ae73f358253d2a2d3120852e5ab624d7da655d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Feb 2018 20:35:49 +0100 Subject: [PATCH 0007/1669] Add `RESTBuilder.patch()`. --- builders.js | 10 ++++++++++ changes.txt | 1 + 2 files changed, 11 insertions(+) diff --git a/builders.js b/builders.js index 0fb3559cd..517a441f0 100755 --- a/builders.js +++ b/builders.js @@ -3984,6 +3984,16 @@ RESTBuilder.prototype.post = function(data) { return this; }; +RESTBuilder.prototype.patch = function(data) { + if (this.$method !== 'patch') { + this.$flags = null; + this.$method = 'patch'; + this.$type = 1; + } + data && this.raw(data); + return this; +}; + RESTBuilder.prototype.json = function(data) { if (this.$type !== 1) diff --git a/changes.txt b/changes.txt index 85d8f5b71..cf9bac111 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: (IMPORTANT) Total.js components can have async delegate - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` +- added: `RESTBuilder.patch([data])` ======= 2.9.3 (HOTFIX) From 1ee5a075f78db48d6cdd53c183f7a232e9526342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Feb 2018 20:59:50 +0100 Subject: [PATCH 0008/1669] Add `CONVERT()` alias. --- changes.txt | 1 + index.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index cf9bac111..d83412ca8 100755 --- a/changes.txt +++ b/changes.txt @@ -3,6 +3,7 @@ - added: (IMPORTANT) Total.js components can have async delegate - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` +- added: `CONVERT()` alias to `F.convert()` ======= 2.9.3 (HOTFIX) diff --git a/index.js b/index.js index 46d81cb58..8e7159150 100644 --- a/index.js +++ b/index.js @@ -950,7 +950,7 @@ F.isSuccess = function(obj) { return obj === SUCCESSHELPER; }; -F.convert = function(value, convertor) { +global.CONVERT = F.convert = function(value, convertor) { if (convertor) { if (F.convertors.findIndex('name', value) !== -1) @@ -985,7 +985,7 @@ F.convert = function(value, convertor) { } for (var i = 0, length = F.convertors.length; i < length; i++) { - if (value[F.convertors[i].name]) + if (value[F.convertors[i].name] != null) value[F.convertors[i].name] = F.convertors[i].convertor(value[F.convertors[i].name]); } From 54a6a96e4ceb8b998d7ebc5fc9b0565601a1ed5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Feb 2018 21:45:01 +0100 Subject: [PATCH 0009/1669] Added uninstalling of convertors. --- index.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 8e7159150..88080f0b5 100644 --- a/index.js +++ b/index.js @@ -953,8 +953,12 @@ F.isSuccess = function(obj) { global.CONVERT = F.convert = function(value, convertor) { if (convertor) { - if (F.convertors.findIndex('name', value) !== -1) + + if (F.convertors.findIndex('name', value) !== -1) { + if (convertor == null) + F.convertors = F.convertors.remove('name', value); return false; + } if (convertor === Number) convertor = U.parseFloat; @@ -4448,6 +4452,12 @@ F.uninstall = function(type, name, options, skipEmit, packageName) { return F; } + if (type === 'convertor') { + F.convertor(name, null); + F.consoledebug('uninstall', type + '#' + name); + return F; + } + if (type === 'schedule') { F.clearSchedule(name); F.consoledebug('uninstall', type + '#' + name); From 2090b4b6b2d7d18bf37f728e4f66745dae44a958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 9 Feb 2018 13:01:47 +0100 Subject: [PATCH 0010/1669] Add `CONVERT()` method. --- builders.js | 183 +++++++++++++++++++++++++++++++++++++++++- changes.txt | 2 +- index.js | 3 +- test/test-builders.js | 6 ++ 4 files changed, 190 insertions(+), 4 deletions(-) diff --git a/builders.js b/builders.js index 517a441f0..ab1e849c4 100755 --- a/builders.js +++ b/builders.js @@ -1767,7 +1767,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { // ARRAY: if (!(val instanceof Array)) { - item[property] = (defaults ? isUndefined(self.$ondefault(property, false, self.name), []) : []); + item[property] = (defaults ? isUndefined(self.$ondefault(property, false, self.name), EMPTYARRAY) : []); continue; } @@ -2917,7 +2917,7 @@ exports.prepare = function(name, model) { }; function isUndefined(value, def) { - return value === undefined ? def : value; + return value === undefined ? (def === EMPTYARRAY ? [] : def) : value; } // ====================================================== @@ -4366,6 +4366,185 @@ OperationOptions.prototype.invalid = function(name, error, path, index) { return this; }; +global.CONVERT = function(value, schema) { + var key = schema; + if (key.length > 50) + key = key.hash(); + var fn = F.convertors2 && F.convertors2[key]; + return fn ? fn(value) : convertorcompile(schema, value, key); +}; + +function convertorcompile(schema, data, key) { + var prop = schema.split(','); + var cache = []; + for (var i = 0, length = prop.length; i < length; i++) { + var arr = prop[i].split(':'); + var obj = {}; + + var type = arr[1].toLowerCase().trim(); + var size = 0; + var isarr = type[0] === '['; + if (isarr) + type = type.substring(1, type.length - 1); + + var index = type.indexOf('('); + if (index !== -1) { + size = +type.substring(index + 1, type.length - 1).trim(); + type = type.substring(0, index); + } + + obj.name = arr[0].trim(); + obj.size = size; + obj.type = type; + obj.array = isarr; + + switch (type) { + case 'string': + obj.fn = $convertstring; + break; + case 'number': + obj.fn = $convertnumber; + break; + case 'boolean': + obj.fn = $convertboolean; + break; + case 'date': + obj.fn = $convertdate; + break; + case 'uid': + obj.fn = $convertuid; + break; + case 'upper': + obj.fn = (val, obj) => $convertstring(val, obj).toUpperCase(); + break; + case 'lower': + obj.fn = (val, obj) => $convertstring(val, obj).toLowerCase(); + break; + case 'capitalize': + obj.fn = (val, obj) => $convertstring(val, obj).capitalize(); + break; + case 'email': + obj.fn = function(val, obj) { + var tmp = $convertstring(val, obj); + return tmp.isEmail() ? tmp : ''; + }; + break; + case 'zip': + obj.fn = function(val, obj) { + var tmp = $convertstring(val, obj); + return tmp.isZIP() ? tmp : ''; + }; + break; + case 'phone': + obj.fn = function(val, obj) { + var tmp = $convertstring(val, obj); + return tmp.isPhone() ? tmp : ''; + }; + break; + case 'url': + obj.fn = function(val, obj) { + var tmp = $convertstring(val, obj); + return tmp.isURL() ? tmp : ''; + }; + break; + case 'json': + obj.fn = function(val, obj) { + var tmp = $convertstring(val, obj); + return tmp.isJSON() ? tmp : ''; + }; + break; + case 'object': + return val => val; + case 'search': + obj.fn = (val, obj) => $convertstring(val, obj).toSearch(); + break; + default: + obj.fn = val => val; + break; + } + + if (isarr) { + obj.fn2 = obj.fn; + obj.fn = function(val, obj) { + if (!(val instanceof Array)) + val = [val]; + var output = []; + for (var i = 0, length = val.length; i < length; i++) { + var o = obj.fn2(val[i], obj); + switch (obj.type) { + case 'email': + case 'phone': + case 'zip': + case 'json': + case 'url': + case 'uid': + case 'date': + o && output.push(o); + break; + default: + output.push(o); + break; + } + } + return output; + }; + } + + cache.push(obj); + } + + var fn = function(data) { + var output = {}; + for (var i = 0, length = cache.length; i < length; i++) { + var item = cache[i]; + output[item.name] = item.fn(data[item.name], item); + } + return output; + }; + if (!F.convertors2) + F.convertors2 = {}; + F.convertors2[key] = fn; + return fn(data); +} + +function $convertstring(value, obj) { + return value == null ? '' : typeof(value) !== 'string' ? obj.size ? value.toString().max(obj.size) : value.toString() : obj.size ? value.max(obj.size) : value; +} + +function $convertnumber(value) { + if (value == null) + return 0; + if (typeof(value) === 'number') + return value; + var num = +value.toString().replace(',', '.'); + return isNaN(num) ? 0 : num; +} + +function $convertboolean(value) { + return value == null ? false : value === true || value == '1' || value === 'true' || value === 'on'; +} + +function $convertuid(value) { + return value == null ? '' : typeof(value) === 'string' ? value.isUID() ? value : '' : ''; +} + +function $convertdate(value) { + + if (value == null) + return null; + + if (value instanceof Date) + return value; + + switch (typeof(value)) { + case 'string': + case 'number': + return value.parseDate(); + } + + return null; +} + // ====================================================== // EXPORTS // ====================================================== diff --git a/changes.txt b/changes.txt index d83412ca8..8d037856a 100755 --- a/changes.txt +++ b/changes.txt @@ -3,7 +3,7 @@ - added: (IMPORTANT) Total.js components can have async delegate - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` -- added: `CONVERT()` alias to `F.convert()` +- added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) ======= 2.9.3 (HOTFIX) diff --git a/index.js b/index.js index 88080f0b5..acc6178bf 100644 --- a/index.js +++ b/index.js @@ -663,6 +663,7 @@ function Framework() { this.isomorphic = {}; this.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {} }; this.convertors = []; + this.convertors2 = null; this.tests = []; this.errors = []; this.problems = []; @@ -950,7 +951,7 @@ F.isSuccess = function(obj) { return obj === SUCCESSHELPER; }; -global.CONVERT = F.convert = function(value, convertor) { +F.convert = function(value, convertor) { if (convertor) { diff --git a/test/test-builders.js b/test/test-builders.js index c3d6353e8..f8ff50932 100755 --- a/test/test-builders.js +++ b/test/test-builders.js @@ -608,6 +608,11 @@ function test_ErrorBuilder() { } +function test_Convertors() { + var a = CONVERT({ page: 5, age: 3, money: '-100', tags: 'Total.js' }, 'page:Number,age:Number, money:Number, tags:[String], empty: Boolean'); + assert.ok(a.page === 5 && a.age === 3 && a.money === -100 && a.tags[0] === 'Total.js' && a.empty === false, 'Problem in convertor'); +} + function test_Operations() { NEWOPERATION('testA', function(error, value, callback) { @@ -635,6 +640,7 @@ test_UrlBuilder(); test_Schema(); test_ErrorBuilder(); test_Operations(); +test_Convertors(); console.log('================================================'); console.log('success - OK'); From 52072654d3dbfe31a9c1b36a89321adff0aa4eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 9 Feb 2018 13:08:54 +0100 Subject: [PATCH 0011/1669] Update version. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index fd52eff1a..8b161dec0 100755 --- a/utils.js +++ b/utils.js @@ -21,7 +21,7 @@ /** * @module FrameworkUtils - * @version 2.9.2 + * @version 2.9.3 */ 'use strict'; From 50632e006fe18c87ebf64c3cc188d4982d92a622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Feb 2018 21:48:24 +0100 Subject: [PATCH 0012/1669] Fix themes in mail sending. --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index acc6178bf..d476d9981 100644 --- a/index.js +++ b/index.js @@ -7437,8 +7437,8 @@ global.MAIL = F.mail = function(address, subject, view, model, callback, languag if (controller.themeName) view = prepare_viewname(view); - else if (this.onTheme) - controller.themeName = this.onTheme(controller); + else if (F.onTheme) + controller.themeName = F.onTheme(controller); else controller.themeName = ''; From e2d890e5b4711a4fd0818721ab601bcf7d4101d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Feb 2018 21:49:14 +0100 Subject: [PATCH 0013/1669] Fix `message.manually()`. --- changes.txt | 2 ++ mail.js | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/changes.txt b/changes.txt index 8d037856a..f0d785011 100755 --- a/changes.txt +++ b/changes.txt @@ -5,6 +5,8 @@ - added: `RESTBuilder.patch([data])` - added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) +- fixed: mail `message.manually()` + ======= 2.9.3 (HOTFIX) - added: `String.arg(obj)` for a simple templating `Hello {variable}!` diff --git a/mail.js b/mail.js index 809d9a05f..910c3cd94 100755 --- a/mail.js +++ b/mail.js @@ -21,7 +21,7 @@ /** * @module FrameworkMail - * @version 2.8.0 + * @version 3.0.0 */ 'use strict'; @@ -257,7 +257,7 @@ Message.prototype.attachment = function(filename, name) { * @return {Message} */ Message.prototype.manually = function() { - this.$sending && clearTimeout(this.$sending); + this.$sending && clearImmediate(this.$sending); return this; }; @@ -353,6 +353,7 @@ Mailer.prototype.destroy = function(obj) { }; const ATTACHMENT_SO = { encoding: 'base64' }; + Mailer.prototype.$writeattachment = function(obj) { var attachment = obj.files ? obj.files.shift() : false; @@ -647,7 +648,6 @@ Mailer.prototype.$send = function(obj, options, autosend) { var socket = obj.socket2 ? obj.socket2 : obj.socket; var host = obj.host; var line = null; - var isAttach = !options.tls || (obj.tls && options.tls); isAttach && mailer.$events.send && mailer.emit('send', obj); @@ -740,11 +740,11 @@ Mailer.prototype.$send = function(obj, options, autosend) { if (obj.messages.length) { mailer.$writemessage(obj, buffer); mailer.$writeline(obj, buffer.shift()); - return; + } else { + // end + mailer.$writeline(obj, 'QUIT'); } - // end - mailer.$writeline(obj, 'QUIT'); return; case 221: // BYE @@ -762,16 +762,16 @@ Mailer.prototype.$send = function(obj, options, autosend) { } var value = auth.shift(); - if (!value) { + if (value) { + mailer.$writeline(obj, value); + } else { var err = new Error('Forbidden.'); mailer.destroy(obj); obj.callback && obj.callback(err); obj.callback = null; mailer.$events.error && !obj.try && mailer.emit('error', err, obj); - return; } - mailer.$writeline(obj, value); return; case 354: @@ -796,12 +796,12 @@ Mailer.prototype.$send = function(obj, options, autosend) { buffer = []; obj.count--; socket.emit('line', '999 TRY NEXT MESSAGE'); - return; + } else { + mailer.destroy(obj); + obj.callback && obj.callback(err); + obj.callback = null; } - mailer.destroy(obj); - obj.callback && obj.callback(err); - obj.callback = null; return; } }); From 9ee42aa29051475f4968da3879448b1ec6a0f606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Feb 2018 19:58:17 +0100 Subject: [PATCH 0014/1669] Added `Capitalize2` schema type. --- builders.js | 15 +++++++++++++++ changes.txt | 1 + test/test-builders.js | 2 ++ 3 files changed, 18 insertions(+) diff --git a/builders.js b/builders.js index ab1e849c4..c2f49141f 100755 --- a/builders.js +++ b/builders.js @@ -585,6 +585,12 @@ SchemaBuilderEntity.prototype.$parse = function(name, value, required, custom) { return parseLength(lower, result); } + if ((/^(capitalize2)+(\(\d+\))?$/).test(lower)) { + result.type = 3; + result.subtype = 'capitalize2'; + return parseLength(lower, result); + } + if ((/^(capitalize|camelcase|camelize)+(\(\d+\))?$/).test(lower)) { result.type = 3; result.subtype = 'capitalize'; @@ -1673,6 +1679,9 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { case 'capitalize': tmp = tmp.capitalize(); break; + case 'capitalize2': + tmp = tmp.capitalize(true); + break; case 'lowercase': tmp = tmp.toLowerCase(); break; @@ -1819,6 +1828,9 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { case 'capitalize': tmp = tmp.capitalize(); break; + case 'capitalize2': + tmp = tmp.capitalize(true); + break; case 'lowercase': tmp = tmp.toLowerCase(); break; @@ -4423,6 +4435,9 @@ function convertorcompile(schema, data, key) { case 'capitalize': obj.fn = (val, obj) => $convertstring(val, obj).capitalize(); break; + case 'capitalize2': + obj.fn = (val, obj) => $convertstring(val, obj).capitalize(true); + break; case 'email': obj.fn = function(val, obj) { var tmp = $convertstring(val, obj); diff --git a/changes.txt b/changes.txt index f0d785011..8ecf902ba 100755 --- a/changes.txt +++ b/changes.txt @@ -4,6 +4,7 @@ - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` - added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) +- added: `Capitalize2` schema type which converts only the first char - fixed: mail `message.manually()` diff --git a/test/test-builders.js b/test/test-builders.js index f8ff50932..e0d1413cc 100755 --- a/test/test-builders.js +++ b/test/test-builders.js @@ -366,6 +366,7 @@ function test_Schema() { var NewTypes = NEWSCHEMA('NewTypes').make(function(schema) { schema.define('capitalize', 'Capitalize'); schema.define('capitalize10', 'Capitalize(10)'); + schema.define('capitalize2', 'Capitalize2'); schema.define('lower', 'Lower'); schema.define('lower10', 'Lower(10)'); schema.define('upper', 'Upper'); @@ -385,6 +386,7 @@ function test_Schema() { var res = schema.make(obj); assert.ok(res.capitalize === 'Total FraMEWOrk', 'SchemaBuilder: Capitalize'); assert.ok(res.capitalize10 === 'Total FraM', 'SchemaBuilder: Capitalize(10)'); + assert.ok(res.capitalize2 === 'Total fraMEWOrk', 'SchemaBuilder: Capitalize2'); assert.ok(res.lower === 'total framework', 'SchemaBuilder: Lower'); assert.ok(res.lower10 === 'total fram', 'SchemaBuilder: Lower(10)'); assert.ok(res.upper === 'TOTAL FRAMEWORK', 'SchemaBuilder: Upper'); From 66b057780a3fc4d67b48529e2caae0db7aedf80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 16 Feb 2018 10:09:27 +0100 Subject: [PATCH 0015/1669] Fixed error handling in `SUCCESS()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index d476d9981..be1c9ef7f 100644 --- a/index.js +++ b/index.js @@ -438,7 +438,7 @@ global.SUCCESS = function(success, value) { var err; if (success instanceof Error) { - err = success; + err = success.toString(); success = false; } else if (success instanceof framework_builders.ErrorBuilder) { if (success.hasError()) { From 3ab19837e65a0dd469a258a95a1b4c176bcbb9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 16 Feb 2018 22:15:20 +0100 Subject: [PATCH 0016/1669] Updated `GROUP()` by adding `prefixes`. --- changes.txt | 2 ++ index.js | 37 ++++++++++++++++++++++++++++++++---- package.json | 2 +- test/controllers/default.js | 12 ++++++++++++ test/test-framework-debug.js | 18 ++++++++++++++++++ 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index 8ecf902ba..ce05a5040 100755 --- a/changes.txt +++ b/changes.txt @@ -6,6 +6,8 @@ - added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) - added: `Capitalize2` schema type which converts only the first char +- updated: `GROUP()` by adding a new argument `url_prefix` + - fixed: mail `message.manually()` ======= 2.9.3 (HOTFIX) diff --git a/index.js b/index.js index be1c9ef7f..6e6af54c0 100644 --- a/index.js +++ b/index.js @@ -269,6 +269,7 @@ var IMAGEMAGICK = false; var _controller = ''; var _owner = ''; var _flags; +var _prefix; // GO ONLINE MODE !global.framework_internal && (global.framework_internal = require('./internal')); @@ -305,7 +306,6 @@ global.FINISHED = framework_internal.onFinished; global.DESTROY = framework_internal.destroyStream; global.UID = () => UIDGENERATOR.date + (++UIDGENERATOR.index).padLeft(4, '0') + UIDGENERATOR.instance + (UIDGENERATOR.index % 2 ? 1 : 0); global.ROUTE = (a, b, c, d, e) => F.route(a, b, c, d, e); -global.GROUP = (a, b) => F.group(a, b); global.WEBSOCKET = (a, b, c, d) => F.websocket(a, b, c, d); global.FILE = (a, b, c) => F.file(a, b, c); global.REDIRECT = (a, b, c, d) => F.redirect(a, b, c, d); @@ -1530,9 +1530,35 @@ global.CORS = F.cors = function(url, flags, credentials) { return F; }; -F.group = function(flags, fn) { - _flags = flags; - fn.call(F); +global.GROUP = F.group = function() { + + var fn = null; + + _flags = null; + _prefix = null; + + for (var i = 0; i < arguments.length; i++) { + var o = arguments[i]; + + if (o instanceof Array) { + _flags = o; + continue; + } + + switch (typeof(o)) { + case 'string': + if (o.endsWith('/')) + o = o.substring(0, o.length - 1); + _prefix = o; + break; + case 'function': + fn = o; + break; + } + } + + fn && fn.call(F); + _prefix = undefined; _flags = undefined; return F; }; @@ -1610,6 +1636,9 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (url[0] !== '[' && url[0] !== '/') url = '/' + url; + if (_prefix) + url = _prefix + url; + if (url.endsWith('/')) url = url.substring(0, url.length - 1); diff --git a/package.json b/package.json index 707a66823..9d427dedc 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "3.0.0-1", + "version": "3.0.0-2", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/test/controllers/default.js b/test/controllers/default.js index fbab051ba..5b4946e50 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -26,6 +26,18 @@ exports.install = function() { this.plain('ROBOT'); }, ['robot']); + GROUP(['get'], '/prefix1/', function() { + ROUTE('/test/', function() { + this.plain('PREFIX1TEST'); + }); + }); + + GROUP('prefix2', ['get'], function() { + ROUTE('/test/', function() { + this.plain('PREFIX2TEST'); + }); + }); + F.route('#route'); F.route('/view-in-modules/', '.' + F.path.modules('someview')); F.route('/options/', plain_options, ['options']); diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 8e2d4ad99..83c93ddf8 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -397,6 +397,24 @@ function test_routing(next) { }); }); + async.await('prefix -- 1', function(complete) { + utils.request(url + 'prefix1/test/', ['get'], function(error, data) { + if (error) + throw error; + assert.ok(data === 'PREFIX1TEST', 'Group + Prefix 1'); + complete(); + }); + }); + + async.await('prefix -- 2', function(complete) { + utils.request(url + 'prefix2/test/', ['get'], function(error, data) { + if (error) + throw error; + assert.ok(data === 'PREFIX2TEST', 'Group + Prefix 2'); + complete(); + }); + }); + async.await('package/', function(complete) { utils.request(url + 'package/', ['get'], function(error, data, code, headers) { if (error) From c32f666bce33088c5c7ff6382e91229552955669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 19 Feb 2018 13:38:28 +0100 Subject: [PATCH 0017/1669] Added new methods: `MailMessage.high()`, `MailMessage.low()` and `MailMessage.confidential()` --- changes.txt | 3 +++ mail.js | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/changes.txt b/changes.txt index ce05a5040..f5e6bff82 100755 --- a/changes.txt +++ b/changes.txt @@ -5,6 +5,9 @@ - added: `RESTBuilder.patch([data])` - added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) - added: `Capitalize2` schema type which converts only the first char +- added: `MailMessage.high()` sets `high` priority of the email messsage +- added: `MailMessage.low()` sets `low` priority of the email messsage +- added: `MailMessage.confidential()` sets `Sensitivity` header with `confidential` value - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/mail.js b/mail.js index 910c3cd94..5324d4c1b 100755 --- a/mail.js +++ b/mail.js @@ -182,6 +182,21 @@ Message.prototype.from = function(address, name) { return this; }; +Message.prototype.high = function() { + this.$priority = 1; + return this; +}; + +Message.prototype.low = function() { + this.$priority = 5; + return this; +}; + +Message.prototype.confidential = function() { + this.$confidential = true; + return this; +}; + Message.prototype.to = function(address, name, clear) { if (typeof(name) === 'boolean') { @@ -537,6 +552,10 @@ Mailer.prototype.$writemessage = function(obj, buffer) { buffer.push('MAIL FROM: <' + msg.addressFrom.address + '>'); message.push('Message-ID: '); message.push('MIME-Version: 1.0'); + + self.$priority && message.push('X-Priority: ' + self.$priority); + self.$confidential && message.push('Sensitivity: Company-Confidential'); + message.push('From: ' + (msg.addressFrom.name ? unicode_encode(msg.addressFrom.name) + ' <' + msg.addressFrom.address + '>' : msg.addressFrom.address)); var length; From 6d52d67d11ab738dbc8220d3fafefb70f2fc9bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 20 Feb 2018 09:47:02 +0100 Subject: [PATCH 0018/1669] Added new SchemaOptions methods. --- builders.js | 25 +++++++++++++++++++++++++ changes.txt | 5 +++++ 2 files changed, 30 insertions(+) diff --git a/builders.js b/builders.js index c2f49141f..189e75cb3 100755 --- a/builders.js +++ b/builders.js @@ -91,6 +91,26 @@ SchemaOptions.prototype = { } }; +SchemaOptions.prototype.clean = function() { + return this.model.$clean(); +}; + +SchemaOptions.prototype.push = function(type, name, helper, first) { + return this.model.$push(type, name, helper, first); +}; + +SchemaOptions.prototype.next = function(type, name, helper) { + return this.model.$next(type, name, helper); +}; + +SchemaOptions.prototype.output = function() { + return this.model.$output(); +}; + +SchemaOptions.prototype.stop = function() { + return this.model.$stop(); +}; + SchemaOptions.prototype.DB = function() { return F.database(this.error); }; @@ -2491,6 +2511,11 @@ SchemaInstance.prototype.$response = SchemaInstance.prototype.$output = function return this; }; +SchemaInstance.prototype.$stop = function() { + this.$$async.length = 0; + return this; +}; + SchemaInstance.prototype.$push = function(type, name, helper, first) { var self = this; diff --git a/changes.txt b/changes.txt index f5e6bff82..5637636c3 100755 --- a/changes.txt +++ b/changes.txt @@ -8,6 +8,11 @@ - added: `MailMessage.high()` sets `high` priority of the email messsage - added: `MailMessage.low()` sets `low` priority of the email messsage - added: `MailMessage.confidential()` sets `Sensitivity` header with `confidential` value +- added: `SchemaBuilderEntity.$stop()` stops the async list +- added: `SchemaOptions.stop()` alias to `$.model.$stop()` +- added: `SchemaOptions.next()` alias to `$.model.$next()` +- added: `SchemaOptions.output()` alias to `$.model.$output()` +- added: `SchemaOptions.clean()` alias to `$.model.$clean()` - updated: `GROUP()` by adding a new argument `url_prefix` From f2867ed80aed3ce0fababe1788282df6b399e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 21 Feb 2018 09:48:36 +0100 Subject: [PATCH 0019/1669] Fixed email attachments. --- changes.txt | 1 + mail.js | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 5637636c3..7bd5749f3 100755 --- a/changes.txt +++ b/changes.txt @@ -16,6 +16,7 @@ - updated: `GROUP()` by adding a new argument `url_prefix` +- fixed: mail attachments - fixed: mail `message.manually()` ======= 2.9.3 (HOTFIX) diff --git a/mail.js b/mail.js index 5324d4c1b..f318eb311 100755 --- a/mail.js +++ b/mail.js @@ -263,7 +263,7 @@ Message.prototype.attachment = function(filename, name) { !name && (name = framework_utils.getName(filename)); var extension = framework_utils.getExtension(name); !this.files && (this.files = []); - this.files.push({ name: name, filename: filename, contentType: framework_utils.getContentType(extension), extension: extension }); + this.files.push({ name: name, filename: filename, type: framework_utils.getContentType(extension), extension: extension }); return this; }; @@ -291,7 +291,7 @@ Message.prototype.attachmentInline = function(filename, name, contentId) { !name && (name = framework_utils.getName(filename)); !this.files && (this.files = []); var extension = framework_utils.getExtension(name); - this.files.push({ name: name, filename: filename, contentType: framework_utils.getContentType(extension), disposition: 'inline', contentId: contentId, extension: extension }); + this.files.push({ name: name, filename: filename, type: framework_utils.getContentType(extension), disposition: 'inline', contentId: contentId, extension: extension }); return this; }; @@ -395,7 +395,7 @@ Mailer.prototype.$writeattachment = function(obj) { message.push('Content-Disposition: attachment; filename="' + name + '"'); } - message.push('Content-Type: ' + extension + ';' + (isCalendar ? ' charset="utf-8"; method=REQUEST' : '')); + message.push('Content-Type: ' + attachment.type + ';' + (isCalendar ? ' charset="utf-8"; method=REQUEST' : '')); message.push('Content-Transfer-Encoding: base64'); message.push(CRLF); mailer.$writeline(obj, message.join(CRLF)); From ff60d8817663b658d6a951baf358a757cbb22a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 21 Feb 2018 09:48:59 +0100 Subject: [PATCH 0020/1669] Improved `controller.invalid()`. --- index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 6e6af54c0..208dbcebd 100644 --- a/index.js +++ b/index.js @@ -10019,12 +10019,18 @@ Controller.prototype.error = function(err) { }; Controller.prototype.invalid = function(status) { + var self = this; + var type = typeof(status); - if (status) + if (type === 'number') self.status = status; var builder = new ErrorBuilder(); + + if (type === 'string') + builder.push(status); + setImmediate(next_controller_invalid, self, builder); return builder; }; From db384959dbf102ffe249b48e9d2e4f2d780ae808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 22 Feb 2018 20:27:20 +0100 Subject: [PATCH 0021/1669] Fixed `tests` directory. --- test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.js b/test.js index 717dc054e..ff551cd62 100644 --- a/test.js +++ b/test.js @@ -21,7 +21,7 @@ /** * @module FrameworkTest - * @version 2.9.0 + * @version 3.0.0 */ var T = F.tests = {}; @@ -73,7 +73,7 @@ function NEXT() { T.current = test; T.current.results = []; - console.log('[ TEST: ' + test.filename.substring(F.path.root('/tests/').length) + (T.current.priority ? ' ({0}) ]'.format(T.current.priority) : ' ]')); + console.log('[ TEST: ' + test.filename.substring(F.path.tests().length) + (T.current.priority ? ' ({0}) ]'.format(T.current.priority) : ' ]')); console.log(''); NEXT(); From dc82fee4cd0fb4a7bfffc8622b4184ef48495fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 22 Feb 2018 20:29:08 +0100 Subject: [PATCH 0022/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9d427dedc..2c399ae1d 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "3.0.0-2", + "version": "3.0.0-3", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4e7fc44149938d5d9ec83fd5519a8376152e988d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Feb 2018 08:20:27 +0100 Subject: [PATCH 0023/1669] Fixed `/tests/` path again. --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index ff551cd62..48cf3c99e 100644 --- a/test.js +++ b/test.js @@ -130,7 +130,7 @@ global.OK = function(is, description) { }; exports.load = function() { - U.ls(F.path.root('/tests/'), function(files) { + U.ls(F.path.tests(), function(files) { files.waitFor(function(filename, next) { T.current = { filename: filename, items: [] }; var m = require(filename); From d9c13179607beae33bf486d9379e9c3e95fd6758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Feb 2018 08:22:09 +0100 Subject: [PATCH 0024/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2c399ae1d..a8e9121ac 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "3.0.0-3", + "version": "3.0.0-4", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 484ecc2df6c8f9602b8deed465e13db0752eb7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Feb 2018 10:59:00 +0100 Subject: [PATCH 0025/1669] Added a route `groups`. --- changes.txt | 2 + index.js | 131 +++++++++++++++++++++++++++------------------------- 2 files changed, 69 insertions(+), 64 deletions(-) diff --git a/changes.txt b/changes.txt index 7bd5749f3..3cff7b1e5 100755 --- a/changes.txt +++ b/changes.txt @@ -13,6 +13,8 @@ - added: `SchemaOptions.next()` alias to `$.model.$next()` - added: `SchemaOptions.output()` alias to `$.model.$output()` - added: `SchemaOptions.clean()` alias to `$.model.$clean()` +- added: a new route flag type `&group` something like `roles` but groups aren't evaluated +- added: `route.groups` with defined groups - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index 208dbcebd..ef7baea51 100644 --- a/index.js +++ b/index.js @@ -775,6 +775,7 @@ function Framework() { this._request_check_referer = false; this._request_check_POST = false; this._request_check_robot = false; + this._request_check_mobile = false; this._length_middleware = 0; this._length_request_middleware = 0; this._length_files = 0; @@ -1698,13 +1699,11 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { var isGENERATOR = false; var description; var id = null; + var groups = []; if (_flags) { - if (!flags) - flags = []; - _flags.forEach(function(flag) { - flags.indexOf(flag) === -1 && flags.push(flag); - }); + !flags && (flags = []); + _flags.forEach(flag => flags.indexOf(flag) === -1 && flags.push(flag)); } if (flags) { @@ -1729,8 +1728,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { var first = flags[i][0]; if (first === '&') { - // resource (sitemap localization) - // isn't used now + groups.push(flags[i].substring(1).trim()); continue; } @@ -1742,7 +1740,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (first === '#') { !middleware && (middleware = []); - middleware.push(flags[i].substring(1)); + middleware.push(flags[i].substring(1).trim()); continue; } @@ -2054,6 +2052,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { r.hash = hash; r.id = id; r.name = name; + r.groups = groups.length ? groups : null; r.priority = priority; r.sitemap = sitemap ? sitemap.id : ''; r.schema = schema; @@ -2135,15 +2134,16 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { !_controller && F.$routesSort(1); } + if (isMOBILE) + F._request_check_mobile = true; + F.emit('route', 'web', instance); return instance; }; function flags_to_object(flags) { var obj = {}; - flags.forEach(function(flag) { - obj[flag] = true; - }); + flags.forEach(flag => obj[flag] = true); return obj; } @@ -2522,6 +2522,7 @@ F.websocket = function(url, funcInitialize, flags, length) { var options; var protocols; var id; + var groups = []; priority = url.count('/'); @@ -2591,13 +2592,8 @@ F.websocket = function(url, funcInitialize, flags, length) { var count = 0; var membertype = 0; - if (!flags) - flags = []; - - _flags && _flags.forEach(function(flag) { - if (flags.indexOf(flag) === -1) - flags.push(flag); - }); + !flags && (flags = []); + _flags && _flags.forEach(flag => flags.indexOf(flag) === -1 && flags.push(flag)); for (var i = 0; i < flags.length; i++) { @@ -2606,7 +2602,7 @@ F.websocket = function(url, funcInitialize, flags, length) { // Middleware options if (type === 'object') { - options = flags[i]; + options = flag; continue; } @@ -2621,10 +2617,16 @@ F.websocket = function(url, funcInitialize, flags, length) { continue; } + // Groups + if (flag[0] === '&') { + groups.push(flag.substring(1).trim()); + continue; + } + // Middleware if (flag[0] === '#') { !middleware && (middleware = []); - middleware.push(flags[i].substring(1)); + middleware.push(flag.substring(1).trim()); continue; } @@ -2682,8 +2684,7 @@ F.websocket = function(url, funcInitialize, flags, length) { tmp.push(flag); break; default: - if (!protocols) - protocols = []; + !protocols && (protocols = []); protocols.push(flag); break; } @@ -2702,6 +2703,7 @@ F.websocket = function(url, funcInitialize, flags, length) { r.id = id; r.urlraw = urlraw; r.hash = hash; + r.groups = groups.length ? groups : null; r.controller = _controller ? _controller : 'unknown'; r.owner = _owner; r.url = routeURL; @@ -2783,38 +2785,29 @@ F.file = function(fnValidation, fnExecute, flags) { var fixedfile = false; var id = null; var urlraw = fnValidation; + var groups = []; if (_flags) { !flags && (flags = []); - _flags.forEach(function(flag) { - flags.indexOf(flag) === -1 && flags.push(flag); - }); + _flags.forEach(flag => flags.indexOf(flag) === -1 && flags.push(flag)); } if (flags) { for (var i = 0, length = flags.length; i < length; i++) { var flag = flags[i]; - - if (typeof(flag) === 'object') { + if (typeof(flag) === 'object') options = flag; - continue; - } - - if (flag.substring(0, 3) === 'id:') { - id = flag.substring(3).trim(); - continue; - } - - if (flag[0] === '#') { + else if (flag[0] === '&') + groups.push(flag.substring(1).trim()); + else if (flag[0] === '#') { !middleware && (middleware = []); - middleware.push(flag.substring(1)); - } - - if (flag[0] === '.') { - flag = flag.substring(1).toLowerCase(); + middleware.push(flag.substring(1).trim()); + } else if (flag[0] === '.') { + flag = flag.substring(1).toLowerCase().trim(); !extensions && (extensions = {}); extensions[flag] = true; - } + } else if (flag.substring(0, 3) === 'id:') + id = flag.substring(3).trim(); } } @@ -2851,6 +2844,7 @@ F.file = function(fnValidation, fnExecute, flags) { var r = instance.route; r.id = id; r.urlraw = urlraw; + r.groups = groups.length ? groups : null; r.controller = _controller ? _controller : 'unknown'; r.owner = _owner; r.url = url; @@ -4121,14 +4115,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_sitemap(); F.$configure_workflows(); } else { - F.$configure_configs('@' + name + '/config'); if (F.config.debug) F.$configure_configs('@' + name + '/config-debug'); else F.$configure_configs('@' + name + '/config-release'); - F.isTest && F.$configure_configs('@' + name + '/config-test'); F.$configure_versions('@' + name + '/versions'); F.$configure_dependencies('@' + name + '/dependencies'); @@ -4258,6 +4250,7 @@ F.$restart = function() { F.isLoaded = false; F.isRestarted = false; F.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {} }; + PERF = {}; F.routes = { sitemap: null, @@ -4308,12 +4301,14 @@ F.$restart = function() { F.traces = []; F.workers = {}; F.convertors = []; + F.convertors2 = null; F.databases = {}; F._request_check_redirect = false; F._request_check_referer = false; F._request_check_POST = false; F._request_check_robot = false; + F._request_check_mobile = false; F._length_middleware = 0; F._length_request_middleware = 0; F._length_files = 0; @@ -5221,6 +5216,7 @@ global.LOGMAIL = F.logmail = function(address, subject, body, callback) { }; F.usage = function(detailed) { + var memory = process.memoryUsage(); var cache = Object.keys(F.cache.items); var resources = Object.keys(F.resources); @@ -5298,31 +5294,31 @@ F.usage = function(detailed) { return output; output.controllers = []; - - controllers.forEach(function(o) { - var item = F.controllers[o]; - output.controllers.push({ name: o, usage: item.usage ? item.usage() : null }); - }); + for (var i = 0, length = controllers.length; i < length; i++) { + var key = controllers[i]; + var item = F.controllers[key]; + output.controllers.push({ name: key, usage: item.usage ? item.usage() : null }); + } output.connections = []; - - connections.forEach(function(o) { - output.connections.push({ name: o, online: F.connections[o].online }); - }); + for (var i = 0, length = connections.length; i < length; i++) { + var key = connections[i]; + output.connections.push({ name: key, online: F.connections[key].online }); + } output.modules = []; - - modules.forEach(function(o) { - var item = F.modules[o]; - output.modules.push({ name: o, usage: item.usage ? item.usage() : null }); - }); + for (var i = 0, length = modules.length; i < length; i++) { + var key = modules[i]; + var item = F.modules[key]; + output.modules.push({ name: key, usage: item.usage ? item.usage() : null }); + } output.models = []; - - models.forEach(function(o) { - var item = F.models[o]; - output.models.push({ name: o, usage: item.usage ? item.usage() : null }); - }); + for (var i = 0, length = models.length; i < length; i++) { + var key = models[i]; + var item = F.models[key]; + output.models.push({ name: key, usage: item.usage ? item.usage() : null }); + } output.uptodates = F.uptodates; output.helpers = helpers; @@ -5335,6 +5331,7 @@ F.usage = function(detailed) { output.files = staticFiles; output.streaming = staticRange; output.other = Object.keys(F.temporary.other); + return output; }; @@ -6981,7 +6978,7 @@ F.$requestcontinue = function(req, res, headers) { var flags = [req.method.toLowerCase()]; var multipart; - if (req.mobile) { + if (F._request_check_mobile && req.mobile) { req.$flags += 'a'; F.stats.request.mobile++; } else @@ -8989,6 +8986,12 @@ FrameworkRoute.prototype = { }, get flags() { return this.route.flags || EMPTYARRAY; + }, + set groups(value) { + this.route.groups = value; + }, + get groups() { + return this.route.groups; } }; From 9e1f162fed4932adaae22c9eacbda4fc7d11e7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Feb 2018 11:21:59 +0100 Subject: [PATCH 0026/1669] Changed groups. --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index ef7baea51..5ef5ace6d 100644 --- a/index.js +++ b/index.js @@ -2052,7 +2052,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { r.hash = hash; r.id = id; r.name = name; - r.groups = groups.length ? groups : null; + r.groups = flags_to_object(groups); r.priority = priority; r.sitemap = sitemap ? sitemap.id : ''; r.schema = schema; @@ -2703,7 +2703,7 @@ F.websocket = function(url, funcInitialize, flags, length) { r.id = id; r.urlraw = urlraw; r.hash = hash; - r.groups = groups.length ? groups : null; + r.groups = flags_to_object(groups); r.controller = _controller ? _controller : 'unknown'; r.owner = _owner; r.url = routeURL; @@ -2844,7 +2844,7 @@ F.file = function(fnValidation, fnExecute, flags) { var r = instance.route; r.id = id; r.urlraw = urlraw; - r.groups = groups.length ? groups : null; + r.groups = flags_to_object(groups); r.controller = _controller ? _controller : 'unknown'; r.owner = _owner; r.url = url; From f78ad354729d4d6be9805cf9e3cb039de2862b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Feb 2018 12:15:05 +0100 Subject: [PATCH 0027/1669] Fixed comparing of `origin` header in `WebSocket`. --- changes.txt | 1 + index.js | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index 3cff7b1e5..057d592c6 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - fixed: mail attachments - fixed: mail `message.manually()` +- fixed: WebSocket comparing of `origin` header ======= 2.9.3 (HOTFIX) diff --git a/index.js b/index.js index 5ef5ace6d..b60a754ba 100644 --- a/index.js +++ b/index.js @@ -7342,7 +7342,7 @@ F.$websocketcontinue_process = function(route, req, path) { var socket = req.websocket; - if (!socket.prepare(route.flags, route.protocols, route.allow, route.length, F.version_header)) { + if (!socket.prepare(route.flags, route.protocols, route.allow, route.length)) { socket.close(); req.connection.destroy(); return; @@ -13347,18 +13347,25 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { allow = allow || EMPTYARRAY; var self = this; + + if (SOCKET_ALLOW_VERSION.indexOf(U.parseInt(self.req.headers['sec-websocket-version'])) === -1) + return false; + self.length = length; var origin = self.req.headers['origin'] || ''; var length = allow.length; - if (length) { - if (allow.indexOf('*') === -1) { - for (var i = 0; i < length; i++) { - if (origin.indexOf(allow[i]) === -1) - return false; + if (length && allow.indexOf('*') === -1) { + var is = false; + for (var i = 0; i < length; i++) { + if (origin.indexOf(allow[i]) !== -1) { + is = true; + break; } } + if (!is) + return false; } length = protocols.length; @@ -13369,9 +13376,6 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { } } - if (SOCKET_ALLOW_VERSION.indexOf(U.parseInt(self.req.headers['sec-websocket-version'])) === -1) - return false; - var compress = (F.config['allow-websocket-compression'] && self.req.headers['sec-websocket-extensions'] || '').indexOf('permessage-deflate') !== -1; var header = protocols.length ? (compress ? SOCKET_RESPONSE_PROTOCOL_COMPRESS : SOCKET_RESPONSE_PROTOCOL).format(self.$websocket_key(self.req), protocols.join(', ')) : (compress ? SOCKET_RESPONSE_COMPRESS : SOCKET_RESPONSE).format(self.$websocket_key(self.req)); From 78cbbb39d6ee33e262d2ab2579c2d903e7deacf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Feb 2018 15:14:24 +0100 Subject: [PATCH 0028/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a8e9121ac..6048ed587 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "3.0.0-4", + "version": "3.0.0-5", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4b2c073e01ae636152fc01be51acb2222d368391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Feb 2018 20:07:33 +0100 Subject: [PATCH 0029/1669] Added previous changes. --- changes.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 057d592c6..4185ae3c2 100755 --- a/changes.txt +++ b/changes.txt @@ -22,6 +22,12 @@ - fixed: mail `message.manually()` - fixed: WebSocket comparing of `origin` header +======= 2.9.4 (HOTFIX) + +- fixed: mail attachments +- fixed: comparing `origin` header in WebSocket +- fixed: unit-testing + ======= 2.9.3 (HOTFIX) - added: `String.arg(obj)` for a simple templating `Hello {variable}!` From 18b980d3af9b72171b3ab018b8b9a2c5d635625a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Feb 2018 10:31:12 +0100 Subject: [PATCH 0030/1669] Added `DatabaseBuilder.insert()` method. --- changes.txt | 1 + nosql.js | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 4185ae3c2..6dca8e7ee 100755 --- a/changes.txt +++ b/changes.txt @@ -15,6 +15,7 @@ - added: `SchemaOptions.clean()` alias to `$.model.$clean()` - added: a new route flag type `&group` something like `roles` but groups aren't evaluated - added: `route.groups` with defined groups +- added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/nosql.js b/nosql.js index aa9d34405..80e038c9a 100755 --- a/nosql.js +++ b/nosql.js @@ -21,7 +21,7 @@ /** * @module NoSQL - * @version 2.9.2 + * @version 3.0.0 */ 'use strict'; @@ -850,9 +850,10 @@ Database.prototype.$update = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - if (item.insert && !item.count) + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert); self.insert(item.insert).$callback = item.builder.$callback; - else { + } else { item.builder.$log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } @@ -878,9 +879,10 @@ Database.prototype.$update = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - if (item.insert && !item.count) + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert); self.insert(item.insert).$callback = item.builder.$callback; - else { + } else { item.builder.$log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(err, item.builder, item.count), item.count); } @@ -961,9 +963,10 @@ Database.prototype.$update_inmemory = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - if (item.insert && !item.count) + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert); self.insert(item.insert).$callback = item.builder.$callback; - else { + } else { item.count && self.emit(item.keys ? 'modify' : 'update', item.doc); item.builder.$log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); @@ -1809,6 +1812,11 @@ function DatabaseBuilder(db) { // this.$log; } +DatabaseBuilder.prototype.insert = function(fn) { + this.$insertcallback = fn; + return this; +}; + DatabaseBuilder.prototype.log = function(msg, user) { var self = this; if (msg) { From c36a369275fa86a7e7d24a5518841463394ff0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Feb 2018 12:30:14 +0100 Subject: [PATCH 0031/1669] Fixed CORS again. --- changes.txt | 1 + index.js | 1 + 2 files changed, 2 insertions(+) diff --git a/changes.txt b/changes.txt index 6dca8e7ee..72df9a1d2 100755 --- a/changes.txt +++ b/changes.txt @@ -22,6 +22,7 @@ - fixed: mail attachments - fixed: mail `message.manually()` - fixed: WebSocket comparing of `origin` header +- fixed: uninstalling CORS routes ======= 2.9.4 (HOTFIX) diff --git a/index.js b/index.js index b60a754ba..f178e5dc1 100644 --- a/index.js +++ b/index.js @@ -4467,6 +4467,7 @@ F.uninstall = function(type, name, options, skipEmit, packageName) { if (k !== 'id') v = framework_internal.preparePath(framework_internal.encodeUnicodeURL(v.replace('*', '').trim())); F.routes.cors = F.routes.cors.remove(k, v); + F._length_cors = F.routes.cors.length; F.consoledebug('uninstall', type + '#' + name); return F; } From 428fcca8e8e9ff1571b565ef3c7d7d27fd813b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 13:02:28 +0100 Subject: [PATCH 0032/1669] Fixed cache for `favicon`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 72df9a1d2..ae883120d 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - fixed: mail `message.manually()` - fixed: WebSocket comparing of `origin` header - fixed: uninstalling CORS routes +- fixed: cache for `favicon` ======= 2.9.4 (HOTFIX) diff --git a/index.js b/index.js index f178e5dc1..e5d167ec4 100644 --- a/index.js +++ b/index.js @@ -11382,7 +11382,7 @@ Controller.prototype.$favicon = function(name) { var contentType = 'image/x-icon'; - if (name == null) + if (!name) name = 'favicon.ico'; var key = 'favicon#' + name; From 68b99be4391e964bff84f84acc428a87007a9551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 13:33:41 +0100 Subject: [PATCH 0033/1669] Fixed `Date.extend()`. --- changes.txt | 1 + utils.js | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/changes.txt b/changes.txt index ae883120d..8a5afb62a 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,7 @@ - fixed: WebSocket comparing of `origin` header - fixed: uninstalling CORS routes - fixed: cache for `favicon` +- fixed: `Date.extend()` ======= 2.9.4 (HOTFIX) diff --git a/utils.js b/utils.js index 8b161dec0..897432634 100755 --- a/utils.js +++ b/utils.js @@ -2546,16 +2546,16 @@ Date.prototype.extend = function(date) { arr = m.split(':'); tmp = +arr[0]; - !isNaN(tmp) && dt.setHours(tmp); + tmp && dt.setHours(tmp); if (arr[1]) { tmp = +arr[1]; - !isNaN(tmp) && dt.setMinutes(tmp); + tmp >= 0 && dt.setMinutes(tmp); } if (arr[2]) { tmp = +arr[2]; - !isNaN(tmp) && dt.setSeconds(tmp); + tmp >= 0 && dt.setSeconds(tmp); } continue; @@ -2565,16 +2565,16 @@ Date.prototype.extend = function(date) { arr = m.split('-'); tmp = +arr[0]; - dt.setFullYear(tmp); - - if (arr[2]) { - tmp = +arr[2]; - !isNaN(tmp) && dt.setDate(tmp); - } + tmp && dt.setFullYear(tmp); if (arr[1]) { tmp = +arr[1]; - !isNaN(tmp) && dt.setMonth(tmp - 1); + tmp >= 0 && dt.setMonth(tmp - 1); + } + + if (arr[2]) { + tmp = +arr[2]; + tmp >= 0 && dt.setDate(tmp); } continue; @@ -2583,18 +2583,18 @@ Date.prototype.extend = function(date) { if (m.indexOf('.') !== -1) { arr = m.split('.'); - tmp = +arr[0]; - dt.setDate(tmp); + if (arr[2]) { + tmp = +arr[2]; + !isNaN(tmp) && dt.setFullYear(tmp); + } if (arr[1]) { tmp = +arr[1]; !isNaN(tmp) && dt.setMonth(tmp - 1); } - if (arr[2]) { - tmp = +arr[2]; - !isNaN(tmp) && dt.setFullYear(tmp); - } + tmp = +arr[0]; + !isNaN(tmp) && dt.setDate(tmp); continue; } @@ -5663,4 +5663,4 @@ if (NODEVERSION > 699) { } global.WAIT = exports.wait; -!global.F && require('./index'); +!global.F && require('./index'); \ No newline at end of file From 30325178c0e353e2a0646130147a054a4dc49234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 13:34:11 +0100 Subject: [PATCH 0034/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6048ed587..57f2e5485 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "3.0.0-5", + "version": "3.0.0-6", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 47fe5617af6223e86de74084251d7e7dbed9424d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 21:40:37 +0100 Subject: [PATCH 0035/1669] Updated `NEWSCHEMA()`. --- changes.txt | 1 + index.js | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 8a5afb62a..0383b2a2a 100755 --- a/changes.txt +++ b/changes.txt @@ -18,6 +18,7 @@ - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - updated: `GROUP()` by adding a new argument `url_prefix` +- updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index e5d167ec4..c823dd89e 100644 --- a/index.js +++ b/index.js @@ -411,8 +411,14 @@ global.NEWTRANSFORM = function() { global.NEWSCHEMA = function(group, name) { if (!name) { - name = group; - group = 'default'; + var arr = name.split('/'); + if (arr.length === 2) { + name = arr[1]; + group = arr[0]; + } else { + name = group; + group = 'default'; + } } return framework_builders.newschema(group, name); }; From 8b3f916214827405834e1a21456b42d2312fe7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 22:36:11 +0100 Subject: [PATCH 0036/1669] Updated `ROUTE()` extended schemas. --- builders.js | 10 +++++----- changes.txt | 1 + index.js | 51 ++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/builders.js b/builders.js index 189e75cb3..80d48ce3b 100755 --- a/builders.js +++ b/builders.js @@ -870,7 +870,7 @@ SchemaBuilderEntity.prototype.constant = function(name, value, description) { !this.constants && (this.constants = {}); this.constants[name] = value; - this.meta['constant#' + name] = description; + this.meta['constant#' + name] = description || null; return this; }; @@ -891,7 +891,7 @@ SchemaBuilderEntity.prototype.addTransform = function(name, fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.transforms && (this.transforms = {}); this.transforms[name] = fn; - this.meta['transform#' + name] = description; + this.meta['transform#' + name] = description || null; return this; }; @@ -912,7 +912,7 @@ SchemaBuilderEntity.prototype.addOperation = function(name, fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.operations && (this.operations = {}); this.operations[name] = fn; - this.meta['operation#' + name] = description; + this.meta['operation#' + name] = description || null; return this; }; @@ -933,7 +933,7 @@ SchemaBuilderEntity.prototype.addWorkflow = function(name, fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.workflows && (this.workflows = {}); this.workflows[name] = fn; - this.meta['workflow#' + name] = description; + this.meta['workflow#' + name] = description || null; return this; }; @@ -945,7 +945,7 @@ SchemaBuilderEntity.prototype.addHook = function(name, fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.hooks[name] && (this.hooks[name] = []); this.hooks[name].push({ owner: F.$owner(), fn: fn }); - this.meta['hook#' + name] = description; + this.meta['hook#' + name] = description || null; return this; }; diff --git a/changes.txt b/changes.txt index 0383b2a2a..3e3233a33 100755 --- a/changes.txt +++ b/changes.txt @@ -19,6 +19,7 @@ - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` +- updated: `ROUTE()`, extended syntax for schems, for example: `Schema --> @name` (more in docs.) - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index c823dd89e..f78e760bd 100644 --- a/index.js +++ b/index.js @@ -411,7 +411,7 @@ global.NEWTRANSFORM = function() { global.NEWSCHEMA = function(group, name) { if (!name) { - var arr = name.split('/'); + var arr = group.split('/'); if (arr.length === 2) { name = arr[1]; group = arr[0]; @@ -1762,6 +1762,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { schema = workflow; workflow = null; } + schema = schema.replace(/\\/g, '/').split('/'); if (schema.length === 1) { @@ -1890,6 +1891,9 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { method = 'get'; } + if (workflow && workflow[0] === '@') + workflow = { id: workflow.substring(1) }; + if (type === 'string') { viewname = funcExecute; funcExecute = (function(name, sitemap, language) { @@ -1930,7 +1934,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (!viewname || viewname === '/') viewname = 'index'; - funcExecute = (function(name, sitemap, language) { + funcExecute = (function(name, sitemap, language, action) { return function() { if (language && !this.language) this.language = language; @@ -1939,12 +1943,16 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (!this.route.workflow) return this.view(name); var self = this; - this.$exec(this.route.workflow, null, function(err, response) { - if (err) - self.content(err); - else - self.view(name, response); - }); + + if (action) { + } else { + self.$exec(self.route.workflow, null, function(err, response) { + if (err) + self.content(err); + else + self.view(name, response); + }); + } }; })(viewname, sitemap, language); } else if (workflow) @@ -16076,7 +16084,32 @@ function parseComponent(body, filename) { function controller_json_workflow(id) { var self = this; self.id = id; - self.$exec(self.route.workflow, null, self.callback()); + var w = self.route.workflow; + if (w instanceof Object) { + if (!w.type) { + var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); + if (schema.meta[w.id] !== undefined) { + w.type = '$' + w.id; + } else if (schema.meta['workflow#' + w.id] !== undefined) { + w.type = '$workflow'; + w.name = w.id; + } else if (schema.meta['workflow#' + w.id] !== undefined) { + w.type = '$transform'; + w.name = w.id; + } else if (schema.meta['operation#' + w.id] !== undefined) { + w.type = '$operation'; + w.name = w.id; + } else if (schema.meta['hook#' + w.id] !== undefined) { + w.type = '$hook'; + w.name = w.id; + } + } + if (w.name) + self[w.type](w.name, self.callback()); + else + self[w.type](self.callback()); + } else + self.$exec(w, null, self.callback()); } // Parses schema group and schema name from string e.g. "User" or "Company/User" From b1d3bf61016cfcf58695e0bc00ed84578dbe2494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 22:47:26 +0100 Subject: [PATCH 0037/1669] Fixed schema description. --- builders.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/builders.js b/builders.js index 80d48ce3b..02d729009 100755 --- a/builders.js +++ b/builders.js @@ -780,7 +780,7 @@ SchemaBuilderEntity.prototype.setPrepare = function(fn) { SchemaBuilderEntity.prototype.setSave = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onSave = fn; - this.meta.save = description; + this.meta.save = description || null; return this; }; @@ -792,7 +792,7 @@ SchemaBuilderEntity.prototype.setSave = function(fn, description) { SchemaBuilderEntity.prototype.setInsert = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onInsert = fn; - this.meta.insert = description; + this.meta.insert = description || null; return this; }; @@ -804,7 +804,7 @@ SchemaBuilderEntity.prototype.setInsert = function(fn, description) { SchemaBuilderEntity.prototype.setUpdate = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onUpdate = fn; - this.meta.update = description; + this.meta.update = description || null; return this; }; @@ -826,7 +826,7 @@ SchemaBuilderEntity.prototype.setError = function(fn) { SchemaBuilderEntity.prototype.setGet = SchemaBuilderEntity.prototype.setRead = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onGet = fn; - this.meta.get = description; + this.meta.get this.meta.read = description || null; return this; }; @@ -839,7 +839,7 @@ SchemaBuilderEntity.prototype.setGet = SchemaBuilderEntity.prototype.setRead = f SchemaBuilderEntity.prototype.setQuery = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onQuery = fn; - this.meta.query = description; + this.meta.query = description || null; return this; }; @@ -852,7 +852,7 @@ SchemaBuilderEntity.prototype.setQuery = function(fn, description) { SchemaBuilderEntity.prototype.setRemove = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onRemove = fn; - this.meta.remove = description; + this.meta.remove = description || null; return this; }; From aed6b8e8aeade96c71934dc58cc8dae8774b26a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Feb 2018 22:48:32 +0100 Subject: [PATCH 0038/1669] Fixed bad code. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 02d729009..334869151 100755 --- a/builders.js +++ b/builders.js @@ -826,7 +826,7 @@ SchemaBuilderEntity.prototype.setError = function(fn) { SchemaBuilderEntity.prototype.setGet = SchemaBuilderEntity.prototype.setRead = function(fn, description) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onGet = fn; - this.meta.get this.meta.read = description || null; + this.meta.get = this.meta.read = description || null; return this; }; From 4f99a8e8a79ccdf49ec5886a4d0a71561c39ea31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 27 Feb 2018 07:57:59 +0100 Subject: [PATCH 0039/1669] Extended `F.route()`, HTTP method can be a part of URL. --- changes.txt | 1 + index.js | 7 ++++++- test/controllers/default.js | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 3e3233a33..77c27c3fe 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` - updated: `ROUTE()`, extended syntax for schems, for example: `Schema --> @name` (more in docs.) +- updated: `ROUTE()` supports a new HTTP method definition `ROUTE('GET /api/users/')`, `ROUTE('POST /api/users/')`, etc. - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index f78e760bd..7314df900 100644 --- a/index.js +++ b/index.js @@ -1640,6 +1640,12 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (!url) url = '/'; + var index = url.indexOf(' '); + if (index !== -1) { + flags.push(url.substring(0, index).toLowerCase().trim()); + url = url.substring(index + 1).trim(); + } + if (url[0] !== '[' && url[0] !== '/') url = '/' + url; @@ -1652,7 +1658,6 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { url = framework_internal.encodeUnicodeURL(url); var type = typeof(funcExecute); - var index = 0; var urlcache = url; if (!name) diff --git a/test/controllers/default.js b/test/controllers/default.js index 5b4946e50..734176287 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -87,7 +87,7 @@ exports.install = function() { F.route('/post/json/', plain_post_json, ['json']); F.route('/post/xml/', plain_post_xml, ['xml']); F.route('/multiple/', plain_multiple, ['post', 'get', 'put', 'delete']); - F.route('/post/schema/', plain_post_schema_parse, ['post', '*test/User']); + F.route('POST /post/schema/', plain_post_schema_parse, ['*test/User']); F.route('/rest/', plain_rest, ['post']); F.route('/rest/', plain_rest, ['put']); F.route('/rest/', plain_rest, ['get', 'head']); @@ -113,7 +113,7 @@ exports.install = function() { assert.ok(F.decrypt('MjM9QR8HExlaHQJQBxcGAEoaFQoGGgAW', 'key', false) === '123456', 'F.decrypt(string)'); assert.ok(F.encrypt({ name: 'Peter' }, 'key', false) === 'MzM9QVUXTkwCThBbF3RXQRlYBkUFVRdOTAJOEFsXdFdBGQ', 'F.encrypt(object)'); - assert.ok(F.decrypt('MzM9QVUXTkwCThBbF3RXQRlYBkUFVRdOTAJOEFsXdFdBGQ', 'key').name === 'Peter', 'F.decrypt(object)') + assert.ok(F.decrypt('MzM9QVUXTkwCThBbF3RXQRlYBkUFVRdOTAJOEFsXdFdBGQ', 'key').name === 'Peter', 'F.decrypt(object)'); assert.ok(SOURCE('main').hello() === 'world', 'source'); assert.ok(INCLUDE('main').hello() === 'world', 'source'); From 1e054e7f6c8392b439e98d8b8349a71d1a4c2446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Feb 2018 08:50:22 +0100 Subject: [PATCH 0040/1669] Fixed parsing hours in `Date.extend()`. --- utils.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 897432634..432a90e1d 100755 --- a/utils.js +++ b/utils.js @@ -2535,6 +2535,8 @@ Date.prototype.extend = function(date) { var dt = new Date(this); var match = date.match(regexpDATE); + console.log(match); + if (!match) return dt; @@ -2546,7 +2548,7 @@ Date.prototype.extend = function(date) { arr = m.split(':'); tmp = +arr[0]; - tmp && dt.setHours(tmp); + tmp >= 0 && dt.setHours(tmp); if (arr[1]) { tmp = +arr[1]; From 9d5109df4fc1f3c81880a054e57611c531348b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Feb 2018 16:53:43 +0100 Subject: [PATCH 0041/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 57f2e5485..cdbdf7d7b 100755 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "name": "Tema Smirnov", "email": "github.tema@smirnov.one" }], - "version": "3.0.0-6", + "version": "3.0.0-7", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 12ece1cc40fa98591187d9341cd744cc9edd16cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Feb 2018 21:28:53 +0100 Subject: [PATCH 0042/1669] Fixed `EMPTYOBJECT` in `utils.js`. --- utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 432a90e1d..7262840b7 100755 --- a/utils.js +++ b/utils.js @@ -21,7 +21,7 @@ /** * @module FrameworkUtils - * @version 2.9.3 + * @version 3.0.0 */ 'use strict'; @@ -72,7 +72,7 @@ const STREAM_READONLY = { flags: 'r' }; const STREAM_END = { end: false }; const ALPHA_INDEX = { '<': '<', '>': '>', '"': '"', '&apos': '\'', '&': '&', '<': '<', '>': '>', '"': '"', ''': '\'', '&': '&' }; const EMPTYARRAY = []; -const EMPTYOBJECT = []; +const EMPTYOBJECT = {}; const NODEVERSION = parseFloat(process.version.toString().replace('v', '').replace(/\./g, '')); const STREAMPIPE = { end: false }; From a97996f5cfb1b7cb10921c67df78d125979b3bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Mar 2018 14:14:57 +0100 Subject: [PATCH 0043/1669] Clean code. --- builders.js | 1 - 1 file changed, 1 deletion(-) diff --git a/builders.js b/builders.js index 334869151..d52fcfb13 100755 --- a/builders.js +++ b/builders.js @@ -1031,7 +1031,6 @@ SchemaBuilderEntity.prototype.update = function(model, options, callback, contro return this.execute('onUpdate', model, options, callback, controller, skip); }; - SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, controller, skip) { if (typeof(callback) === 'boolean') { From e343c9f42cb6cbb45a891bc02020a689654fe278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Mar 2018 14:15:03 +0100 Subject: [PATCH 0044/1669] Fixed grammar. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 77c27c3fe..ff7cf49b3 100755 --- a/changes.txt +++ b/changes.txt @@ -19,7 +19,7 @@ - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` -- updated: `ROUTE()`, extended syntax for schems, for example: `Schema --> @name` (more in docs.) +- updated: `ROUTE()`, extended syntax for schemas, for example: `Schema --> @name` (more in docs.) - updated: `ROUTE()` supports a new HTTP method definition `ROUTE('GET /api/users/')`, `ROUTE('POST /api/users/')`, etc. - fixed: mail attachments From 9c38051f73c4ef1a52c5241d19b5780420e2b2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 3 Mar 2018 14:29:33 +0100 Subject: [PATCH 0045/1669] Added `bundles`. --- bundles.js | 198 ++++++++++++++++++++++++++++++++++ changes.txt | 1 + debug.js | 75 +++++++------ index.js | 298 +++++++++++++++++++++++++++++----------------------- 4 files changed, 405 insertions(+), 167 deletions(-) create mode 100644 bundles.js diff --git a/bundles.js b/bundles.js new file mode 100644 index 000000000..461ca59f4 --- /dev/null +++ b/bundles.js @@ -0,0 +1,198 @@ +require('./index'); +const Fs = require('fs'); +const Path = require('path'); +var CONSOLE = process.argv.indexOf('restart') === -1; + +var META = {}; +META.version = 1; +META.created = new Date(); +META.total = 'v' + F.version_header; +META.node = F.version_node; +META.files = []; +META.directories = []; + +exports.make = function(callback) { + + var path = F.path.root(); + var blacklist = {} + + if (CONSOLE) { + console.log('==================== BUNDLING ======================'); + console.time('Done'); + } + + blacklist[F.config['directory-temp']] = 1; + blacklist[F.config['directory-bundles']] = 1; + blacklist[F.config['directory-src']] = 1; + blacklist['/node_modules/'] = 1; + blacklist['/debug.js'] = 1; + blacklist['/debug.pid'] = 1; + blacklist['/package.json'] = 1; + blacklist['/bundle.json'] = 1; + + var Files = []; + var Dirs = []; + var Length = path.length; + var async = []; + + async.push(cleanFiles); + + async.push(function(next) { + var target = F.path.root(F.config['directory-src']); + U.ls(F.path.root(F.config['directory-bundles']), function(files) { + files.wait(function(filename, resume) { + var dbpath = F.config['directory-databases']; + F.restore(filename, target, resume, function(p, dir) { + if (dir) { + if (!p.startsWith(dbpath) && META.directories.indexOf(p) === -1) + META.directories.push(p); + } else { + + var exists = false; + try { + exists = Fs.statSync(Path.join(target, p)) != null; + } catch (e) {} + + // DB file + if (exists && p.startsWith(dbpath)) + return false; + + if (META.files.indexOf(p) === -1) + META.files.push(p); + } + + return true; + }); + }, next); + }); + }); + + async.push(function(next) { + U.ls(path, function(files, dirs) { + + for (var i = 0, length = dirs.length; i < length; i++) + Dirs.push(dirs[i].substring(Length)); + + for (var i = 0, length = files.length; i < length; i++) { + var file = files[i].substring(Length); + var type = 0; + + if (file.startsWith(F.config['directory-databases'])) + type = 1; + else if (file.startsWith(F.config['directory-public'])) + type = 2; + Files.push({ name: file, filename: files[i], type: type }); + } + + next(); + }, (p, dir) => blacklist[p.substring(Length)] == null); + }); + + async.push(function(next) { + createDirectories(Dirs, function() { + copyFiles(Files, next); + }); + }); + + async.push(function(next) { + Fs.writeFileSync(F.path.root('bundle.json'), JSON.stringify(META, null, '\t')); + next(); + }); + + async.async(function() { + CONSOLE && console.timeEnd('Done'); + callback(); + }); + +}; + +function cleanFiles(callback) { + + var path = F.path.root(F.config['directory-src']); + var length = path.length - 1; + var blacklist = {}; + + blacklist[F.config['directory-public']] = 1; + blacklist[F.config['directory-private']] = 1; + blacklist[F.config['directory-databases']] = 1; + + var meta = {}; + + try { + meta = U.parseJSON(Fs.readFileSync(F.path.root('bundle.json')).toString('utf8'), true); + } catch (e) {} + + if (meta.files && meta.files.length) { + for (var i = 0, length = meta.files.length; i < length; i++) { + try { + F.consoledebug('Remove', meta.files[i]); + Fs.unlinkSync(Path.join(path, meta.files[i])); + } catch (e) {} + } + } + + if (meta.directories && meta.directories.length) { + meta.directories.quicksort('length', false); + for (var i = 0, length = meta.directories.length; i < length; i++) { + try { + Fs.rmdirSync(Path.join(path, meta.directories[i])); + } catch (e) {} + } + } + + callback(); +} + +function createDirectories(dirs, callback) { + + var path = F.path.root(F.config['directory-src']); + + try { + Fs.mkdirSync(path); + } catch(e) {} + + for (var i = 0, length = dirs.length; i < length; i++) { + if (META.directories.indexOf(dirs[i]) === -1) + META.directories.push(dirs[i]); + try { + Fs.mkdirSync(Path.join(path, dirs[i])); + } catch (e) {} + } + + callback(); +} + +function copyFiles(files, callback) { + var path = F.path.root(F.config['directory-src']); + var skip + files.wait(function(file, next) { + + var filename = Path.join(path, file.name); + var exists = false; + + try { + exists = Fs.statSync(filename) != null; + } catch (e) {} + + // DB file + if (file.type === 1 && exists) { + next(); + return; + } + + if (file.type !== 1 && META.files.indexOf(file.name) === -1) + META.files.push(file.name); + + if (CONSOLE && exists) + if (F.config['allow-debug']) + F.consoledebug('Rewrite', file.name); + else + console.warn('Rewrite: ' + file.name) + else + F.consoledebug('Copy', file.name); + + var writer = Fs.createWriteStream(filename); + writer.on('finish', next); + Fs.createReadStream(file.filename).pipe(writer); + }, callback); +} \ No newline at end of file diff --git a/changes.txt b/changes.txt index ff7cf49b3..b54c2971a 100755 --- a/changes.txt +++ b/changes.txt @@ -1,5 +1,6 @@ ======= 3.0.0 +- added: (IMPORTANT) bundles - added: (IMPORTANT) Total.js components can have async delegate - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` diff --git a/debug.js b/debug.js index 332c45660..facc027e1 100644 --- a/debug.js +++ b/debug.js @@ -80,11 +80,11 @@ function runwatching() { const VERSION = F.version_header; const TIME = 2000; const REG_CONFIGS = /configs\//g; - const REG_FILES = /config\-debug|config\-release|config|versions|workflows|sitemap|dependencies|\.js|\.resource/i; + const REG_FILES = /config\-debug|config\-release|config|versions|workflows|sitemap|dependencies|\.js$|\.resource$/i; const REG_THEMES = /\/themes\//i; const REG_COMPONENTS = /components\/.*?\.html/i; - const REG_THEMES_INDEX = /themes(\/|\\)?[a-z0-9_.-]+(\/|\\)?index\.js/i; - const REG_EXTENSION = /\.(js|resource|package)/i; + const REG_THEMES_INDEX = /themes(\/|\\)?[a-z0-9_.-]+(\/|\\)?index\.js$/i; + const REG_EXTENSION = /\.(js|resource|package|bundle)$/i; function app() { @@ -105,10 +105,24 @@ function runwatching() { U.combine(F.config['directory-packages']), U.combine(F.config['directory-themes']), U.combine(F.config['directory-configs']), + U.combine(F.config['directory-bundles']), + U.combine(F.config['directory-src'], F.config['directory-components']), + U.combine(F.config['directory-src'], F.config['directory-controllers']), + U.combine(F.config['directory-src'], F.config['directory-definitions']), + U.combine(F.config['directory-src'], F.config['directory-models']), + U.combine(F.config['directory-src'], F.config['directory-resources']), + U.combine(F.config['directory-src'], F.config['directory-source']), + U.combine(F.config['directory-src'], F.config['directory-workers']), + U.combine(F.config['directory-src'], F.config['directory-packages']), + U.combine(F.config['directory-src'], F.config['directory-themes']), + U.combine(F.config['directory-src'], F.config['directory-configs']), U.combine('/startup/'), U.combine('/schema/') ]; + const SRC = U.combine(F.config['directory-src']); + + const async = new U.Async(); const prefix = '---------------------------------> '; @@ -130,6 +144,7 @@ function runwatching() { var isSkip = false; var pidIncrease; var speed = TIME; + var counter = 0; function onFilter(path, isDirectory) { return !isDirectory && REG_THEMES.test(path) ? REG_THEMES_INDEX.test(path) : isDirectory ? true : REG_EXTENSION.test(path) || REG_COMPONENTS.test(path) || REG_CONFIGS.test(path); @@ -173,39 +188,30 @@ function runwatching() { } function refresh() { - - var filenames = Object.keys(files); - var length = filenames.length; - - for (var i = 0; i < length; i++) { - var filename = filenames[i]; - (function(filename) { - async.await(function(next) { - Fs.stat(filename, function(err, stat) { - - var stamp = '--- # --- [ ' + new Date().format('yyyy-MM-dd HH:mm:ss') + ' ] '; - - if (err) { - delete files[filename]; - changes.push(stamp.replace('#', 'REM') + prefix + filename.replace(directory, '')); + counter++; + Object.keys(files).wait(function(filename, next) { + Fs.stat(filename, function(err, stat) { + + var stamp = '--- # --- [ ' + new Date().format('yyyy-MM-dd HH:mm:ss') + ' ] '; + + if (err) { + delete files[filename]; + changes.push(stamp.replace('#', 'REM') + prefix + filename.replace(directory, '')); + force = true; + } else { + var ticks = stat.mtime.getTime(); + if (files[filename] != null && files[filename] !== ticks) { + if (filename.substring(0, SRC.length) !== SRC || counter > 2) { + changes.push(stamp.replace('#', files[filename] === 0 ? 'ADD' : 'UPD') + prefix + filename.replace(directory, '')); force = true; - } else { - var ticks = stat.mtime.getTime(); - if (files[filename] != null && files[filename] !== ticks) { - changes.push(stamp.replace('#', files[filename] === 0 ? 'ADD' : 'UPD') + prefix + filename.replace(directory, '')); - force = true; - } - files[filename] = ticks; } + } + files[filename] = ticks; + } - next(); - }); - }); - - })(filename); - } - - async.complete(function() { + next(); + }); + }, function() { isLoaded = true; setTimeout(refresh_directory, speed); @@ -214,6 +220,7 @@ function runwatching() { if (status !== 1 || !force) return; + counter = 0; onIncrease(true); restart(); @@ -223,7 +230,7 @@ function runwatching() { changes = []; force = false; - }); + }, 3); } function refresh_directory() { diff --git a/index.js b/index.js index 7314df900..682d5dc40 100644 --- a/index.js +++ b/index.js @@ -518,7 +518,7 @@ function Framework() { this.$id = null; // F.id ==> property this.version = 3000; this.version_header = '3.0.0'; - this.version_node = process.version.toString().replace('v', '').replace(/\./g, '').parseFloat(); + this.version_node = process.version.toString(); this.config = { @@ -533,6 +533,8 @@ function Framework() { 'default-xpoweredby': 'Total.js', 'etag-version': '', + 'directory-src': '/src/', + 'directory-bundles': '/bundles/', 'directory-controllers': '/controllers/', 'directory-components': '/components/', 'directory-views': '/views/', @@ -603,6 +605,7 @@ function Framework() { 'allow-compile-script': true, 'allow-compile-style': true, 'allow-compile-html': true, + 'allow-compile-build': true, 'allow-performance': false, 'allow-custom-titles': false, 'allow-cache-snapshot': false, @@ -3145,6 +3148,31 @@ F.modify = function(fn) { return F; }; +F.$bundle = function(callback) { + + var can = false; + var makebundle = function() { + require('./bundles').make(function() { + F.directory = HEADERS.workers.cwd = directory = F.path.root(F.config['directory-src']); + callback(); + }); + }; + + try { + Fs.statSync(F.path.root(F.config['directory-src'])); + makebundle(); + return; + } catch(e) {} + + try { + Fs.statSync(F.path.root(F.config['directory-bundles'])); + makebundle(); + return; + } catch(e) {} + + callback(); +}; + F.$load = function(types, targetdirectory, callback, packageName) { var arr = []; @@ -3563,7 +3591,6 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } if (type === 'config' || type === 'configuration' || type === 'settings') { - F.$configure_configs(declaration instanceof Array ? declaration : declaration.toString().split('\n'), true); setTimeout(function() { delete F.temporary['mail-settings']; @@ -4147,7 +4174,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_workflows('@' + name + '/workflows'); } - F.$load(undefined, tmpdir, undefined, name); + F.$bundle(n => F.$load(undefined, tmpdir, undefined, name)); }, 100); key = type + '.' + name; @@ -6291,53 +6318,55 @@ F.load = function(debug, types, pwd) { global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; - F.consoledebug('startup'); - F.$startup(function() { + F.$bundle(function() { + F.consoledebug('startup'); + F.$startup(function() { - F.consoledebug('startup (done)'); - F.$configure_configs(); + F.consoledebug('startup (done)'); + F.$configure_configs(); - if (!types || types.indexOf('versions') !== -1) - F.$configure_versions(); + if (!types || types.indexOf('versions') !== -1) + F.$configure_versions(); - if (!types || types.indexOf('workflows') !== -1) - F.$configure_workflows(); + if (!types || types.indexOf('workflows') !== -1) + F.$configure_workflows(); - if (!types || types.indexOf('sitemap') !== -1) - F.$configure_sitemap(); + if (!types || types.indexOf('sitemap') !== -1) + F.$configure_sitemap(); - F.consoledebug('init'); - F.cache.init(); - F.emit('init'); + F.consoledebug('init'); + F.cache.init(); + F.emit('init'); - F.$load(types, directory, function() { + F.$load(types, directory, function() { - F.isLoaded = true; - process.send && process.send('total:ready'); + F.isLoaded = true; + process.send && process.send('total:ready'); - setTimeout(function() { + setTimeout(function() { - try { - F.emit('load', F); - F.emit('ready', F); - } catch (err) { - F.error(err, 'F.on("load/ready")'); - } + try { + F.emit('load', F); + F.emit('ready', F); + } catch (err) { + F.error(err, 'F.on("load/ready")'); + } - F.removeAllListeners('load'); - F.removeAllListeners('ready'); + F.removeAllListeners('load'); + F.removeAllListeners('ready'); - // clear unnecessary items - delete F.tests; - delete F.test; - delete F.testing; - delete F.assert; - }, 500); + // clear unnecessary items + delete F.tests; + delete F.test; + delete F.testing; + delete F.assert; + }, 500); - if (F.config['allow-debug']) { - F.consoledebug('done'); - F.usagesnapshot(); - } + if (F.config['allow-debug']) { + F.consoledebug('done'); + F.usagesnapshot(); + } + }); }); }); @@ -6380,121 +6409,124 @@ F.initialize = function(http, debug, options, restart) { global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; - F.$configure_configs(); - F.$configure_versions(); - F.$configure_workflows(); - F.$configure_sitemap(); - F.isTest && F.$configure_configs('config-test', true); - F.cache.init(); - F.consoledebug('init'); - F.emit('init'); - - if (!port) { - if (F.config['default-port'] === 'auto') { - var envPort = +(process.env.PORT || ''); - if (!isNaN(envPort)) - port = envPort; - } else - port = F.config['default-port']; - } - - F.port = port || 8000; + F.$bundle(function() { - if (ip !== null) { - F.ip = ip || F.config['default-ip'] || '0.0.0.0'; - if (F.ip === 'null' || F.ip === 'undefined' || F.ip === 'auto') - F.ip = null; - } else - F.ip = undefined; + F.$configure_configs(); + F.$configure_versions(); + F.$configure_workflows(); + F.$configure_sitemap(); + F.isTest && F.$configure_configs('config-test', true); + F.cache.init(); + F.consoledebug('init'); + F.emit('init'); - if (F.ip == null) - F.ip = '0.0.0.0'; + if (!port) { + if (F.config['default-port'] === 'auto') { + var envPort = +(process.env.PORT || ''); + if (!isNaN(envPort)) + port = envPort; + } else + port = F.config['default-port']; + } - !listenpath && (listenpath = F.config['default-listenpath']); - F.listenpath = listenpath; + F.port = port || 8000; - if (F.server) { - F.server.removeAllListeners(); - Object.keys(F.connections).forEach(function(key) { - var item = F.connections[key]; - if (item) { - item.removeAllListeners(); - item.close(); - } - }); + if (ip !== null) { + F.ip = ip || F.config['default-ip'] || '0.0.0.0'; + if (F.ip === 'null' || F.ip === 'undefined' || F.ip === 'auto') + F.ip = null; + } else + F.ip = undefined; - F.server.close(); - } + if (F.ip == null) + F.ip = '0.0.0.0'; - var listen = function() { + !listenpath && (listenpath = F.config['default-listenpath']); + F.listenpath = listenpath; - if (options.https) - F.server = http.createServer(options.https, F.listener); - else - F.server = http.createServer(F.listener); + if (F.server) { + F.server.removeAllListeners(); + Object.keys(F.connections).forEach(function(key) { + var item = F.connections[key]; + if (item) { + item.removeAllListeners(); + item.close(); + } + }); - F.config['allow-performance'] && F.server.on('connection', connection_tunning); - F.initwebsocket && F.initwebsocket(); - F.consoledebug('HTTP listening'); + F.server.close(); + } - if (listenpath) - F.server.listen(listenpath); - else - F.server.listen(F.port, F.ip); - }; + var listen = function() { - // clears static files - F.consoledebug('clear temporary'); - F.clear(function() { - F.consoledebug('clear temporary (done)'); - F.$load(undefined, directory, function() { + if (options.https) + F.server = http.createServer(options.https, F.listener); + else + F.server = http.createServer(F.listener); - F.isLoaded = true; - process.send && process.send('total:ready'); + F.config['allow-performance'] && F.server.on('connection', connection_tunning); + F.initwebsocket && F.initwebsocket(); + F.consoledebug('HTTP listening'); - if (options.middleware) - options.middleware(listen); + if (listenpath) + F.server.listen(listenpath); else - listen(); + F.server.listen(F.port, F.ip); + }; - if (F.config['allow-debug']) { - F.consoledebug('done'); - F.usagesnapshot(); - } + // clears static files + F.consoledebug('clear temporary'); + F.clear(function() { + F.consoledebug('clear temporary (done)'); + F.$load(undefined, directory, function() { - if (!process.connected || restart) - F.console(); + F.isLoaded = true; + process.send && process.send('total:ready'); - setTimeout(function() { + if (options.middleware) + options.middleware(listen); + else + listen(); - try { - F.emit('load', F); - F.emit('ready', F); - } catch (err) { - F.error(err, 'F.on("load/ready")'); + if (F.config['allow-debug']) { + F.consoledebug('done'); + F.usagesnapshot(); } - F.removeAllListeners('load'); - F.removeAllListeners('ready'); - options.package && INSTALL('package', options.package); - }, 500); + if (!process.connected || restart) + F.console(); - if (F.isTest) { - var sleep = options.sleep || options.delay || 1000; - setTimeout(() => F.test(true, options.tests || options.test), sleep); - return F; - } + setTimeout(function() { - setTimeout(function() { - if (F.isTest) - return; - delete F.tests; - delete F.test; - delete F.testing; - delete F.assert; - }, 5000); - }); - }, true); + try { + F.emit('load', F); + F.emit('ready', F); + } catch (err) { + F.error(err, 'F.on("load/ready")'); + } + + F.removeAllListeners('load'); + F.removeAllListeners('ready'); + options.package && INSTALL('package', options.package); + }, 500); + + if (F.isTest) { + var sleep = options.sleep || options.delay || 1000; + setTimeout(() => F.test(true, options.tests || options.test), sleep); + return F; + } + + setTimeout(function() { + if (F.isTest) + return; + delete F.tests; + delete F.test; + delete F.testing; + delete F.assert; + }, 5000); + }); + }, true); + }); return F; }; @@ -7917,7 +7949,7 @@ F.$configure_sitemap = function(arr, clean) { F.sitemap = function(name, me, language) { if (!F.routes.sitemap) - return EMPTYARRAY; + return me ? null : EMPTYARRAY; if (typeof(me) === 'string') { language = me; From 5219695a9529aeb43effda42d67b6e94ba844e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 3 Mar 2018 14:57:33 +0100 Subject: [PATCH 0046/1669] Added fallback for `prependListener` (for older version of Node.js). --- internal.js | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/internal.js b/internal.js index aadc2caf2..28367e105 100755 --- a/internal.js +++ b/internal.js @@ -3169,16 +3169,29 @@ function onFinished(stream, fn) { if (stream.socket) { if (!stream.socket.$totalstream) { stream.socket.$totalstream = stream; - stream.socket.prependListener('error', callback); - stream.socket.prependListener('close', callback); + if (stream.socket.prependListener) { + stream.socket.prependListener('error', callback); + stream.socket.prependListener('close', callback); + } else { + stream.socket.on('error', callback); + stream.socket.on('close', callback); + } } } - stream.prependListener('error', callback); - stream.prependListener('end', callback); - stream.prependListener('close', callback); - stream.prependListener('aborted', callback); - stream.prependListener('finish', callback); + if (stream.prependListener) { + stream.prependListener('error', callback); + stream.prependListener('end', callback); + stream.prependListener('close', callback); + stream.prependListener('aborted', callback); + stream.prependListener('finish', callback); + } else { + stream.on('error', callback); + stream.on('end', callback); + stream.on('close', callback); + stream.on('aborted', callback); + stream.on('finish', callback); + } //stream.uri --> determines ServerRespone // stream.uri && stream.prependListener('aborted', callback); From be92b3c49f26248b0ea00c4481522dfd4aff93b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 3 Mar 2018 15:16:54 +0100 Subject: [PATCH 0047/1669] Add new directory `schemas`. --- debug.js | 2 ++ index.js | 13 ++++++++++++- test/schemas/user.js | 1 + test/test-framework-debug.js | 1 + test/test-framework-release.js | 1 + 5 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 test/schemas/user.js diff --git a/debug.js b/debug.js index facc027e1..60eacc33b 100644 --- a/debug.js +++ b/debug.js @@ -99,6 +99,7 @@ function runwatching() { U.combine(F.config['directory-isomorphic']), U.combine(F.config['directory-modules']), U.combine(F.config['directory-models']), + U.combine(F.config['directory-schemas']), U.combine(F.config['directory-resources']), U.combine(F.config['directory-source']), U.combine(F.config['directory-workers']), @@ -110,6 +111,7 @@ function runwatching() { U.combine(F.config['directory-src'], F.config['directory-controllers']), U.combine(F.config['directory-src'], F.config['directory-definitions']), U.combine(F.config['directory-src'], F.config['directory-models']), + U.combine(F.config['directory-src'], F.config['directory-schemas']), U.combine(F.config['directory-src'], F.config['directory-resources']), U.combine(F.config['directory-src'], F.config['directory-source']), U.combine(F.config['directory-src'], F.config['directory-workers']), diff --git a/index.js b/index.js index 682d5dc40..f334a5699 100644 --- a/index.js +++ b/index.js @@ -541,6 +541,7 @@ function Framework() { 'directory-definitions': '/definitions/', 'directory-temp': '/tmp/', 'directory-models': '/models/', + 'directory-schemas': '/schemas/', 'directory-resources': '/resources/', 'directory-public': '/public/', 'directory-public-virtual': '/app/', @@ -3307,6 +3308,16 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } + if (!types || types.indexOf('schemas') !== -1) { + operations.push(function(resume) { + dir = U.combine(targetdirectory, isPackage ? '/schemas/' : F.config['directory-schemas']); + arr = []; + listing(dir, 0, arr); + arr.forEach((item) => dependencies.push(next => F.install('schema', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); + resume(); + }); + } + if (!types || types.indexOf('themes') !== -1) { operations.push(function(resume) { arr = []; @@ -3864,7 +3875,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe return F; } - if (type === 'definition' || type === 'eval') { + if (type === 'definition' || type === 'eval' || type === 'schema') { _controller = ''; _owner = (packageName ? packageName + '@' : '') + type + '#' + name; diff --git a/test/schemas/user.js b/test/schemas/user.js new file mode 100644 index 000000000..b5f66fa17 --- /dev/null +++ b/test/schemas/user.js @@ -0,0 +1 @@ +F.global.schemas = 1; \ No newline at end of file diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 83c93ddf8..6560f9e96 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -905,6 +905,7 @@ framework.on('load', function() { assert.ok(RESOURCE('default', 'name-root').length > 0, 'custom resource mapping 1'); assert.ok(RESOURCE('default', 'name-theme').length > 0, 'custom resource mapping 2'); assert.ok(F.global.newslettercomponent, 'components: inline '), name + ' minify html'); + assert.ok(output.contains('') || output.contains(''), name + ' minify html'); assert.ok(output.contains('#tag-encode<b>A</b>#'), name + 'encode value'); assert.ok(output.contains('#tag-rawA#'), name + 'raw value'); assert.ok(output.contains('#helper-fn-A#'), name + 'helper function'); From 534c73268cb142ed45bda0f0eea6e1c32fd794ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 10 Mar 2018 14:00:26 +0100 Subject: [PATCH 0084/1669] Fixed Windows line endings. --- internal.js | 13 +++++++------ test/test-framework-debug.js | 2 +- test/test-framework-release.js | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/internal.js b/internal.js index 2bd80a855..8d5e75797 100755 --- a/internal.js +++ b/internal.js @@ -43,13 +43,14 @@ Object.freeze(EMPTYARRAY); const REG_1 = /[\n\r\t]+/g; const REG_2 = /\s{2,}/g; -const REG_4 = /(\r)?\n\s{2,}./g; -const REG_5 = />(\r)?\n\s{1,}\n\s{1,}]+/; const REG_7 = /\\/g; const REG_8 = /'/g; -const REG_9 = />(\r)?\n\s+/g; -const REG_10 = /(\w|\W)(\r)?\n\s+\n\s+/g; +const REG_10 = /(\w|\W)\n\s+](\r)\n\s{1,}$/; const REG_HELPERS = /helpers\.[a-z0-9A-Z_$]+\(.*?\)+/g; const REG_SITEMAP = /\s+(sitemap_navigation\(|sitemap\()+/g; -const REG_CSS_1 = /(\r)?\n|\s{2,}/g; +const REG_CSS_1 = /\n|\s{2,}/g; const REG_CSS_2 = /\s?\{\s{1,}/g; const REG_CSS_3 = /\s?\}\s{1,}/g; const REG_CSS_4 = /\s?:\s{1,}/g; @@ -2826,7 +2827,7 @@ function compressHTML(html, minify, isChunk) { if (!html || !minify) return html; - html = removeComments(html); + html = removeComments(html.replace(REG_WIN, '')); var tags = ['script', 'textarea', 'pre', 'code']; var id = '[' + new Date().getTime() + ']#'; diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 6560f9e96..0ee9861ee 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -173,7 +173,7 @@ function test_routing(next) { utils.request(url + 'html-nocompress/', ['get'], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('
\nA\n
') !== -1, 'HTML nocompress'); + assert(data.indexOf('
\nA\n
') !== -1 || data.indexOf('
\r\nA\r\n
') !== -1, 'HTML nocompress'); complete(); }); }); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 8f7e6b901..0ec9e3c8e 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -173,7 +173,7 @@ function test_routing(next) { utils.request(url + 'html-nocompress/', ['get'], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('
\nA\n
') !== -1, 'HTML nocompress'); + assert(data.indexOf('
\nA\n
') !== -1 || data.indexOf('
\r\nA\r\n
') !== -1, 'HTML nocompress'); complete(); }); }); From a459fd4d86516bd59e985b5ec918b704bc640014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 10 Mar 2018 17:44:45 +0100 Subject: [PATCH 0085/1669] Added missing command. --- bin/tpm | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/tpm b/bin/tpm index e20d62caf..a55e7e928 100755 --- a/bin/tpm +++ b/bin/tpm @@ -528,6 +528,7 @@ function display_help() { log(''); log('--- --- --- --- ---'); log(''); + log('Creating packages:'); log(colors.red + '$ tpm create [important: package_name] [optional: package_directory_to_pack]' + colors.reset); log(''); log(colors.dim + 'EXAMPLE: tpm create my-project-template'); @@ -536,6 +537,15 @@ function display_help() { log(''); log('--- --- --- --- ---'); log(''); + log('Creating bundles:'); + log(colors.red + '$ tpm bundle [important: bundle_name] [optional: bundle_directory_to_pack]' + colors.reset); + log(''); + log(colors.dim + 'EXAMPLE: tpm bundle my-project-template'); + log('EXAMPLE: tpm bundle my-module'); + log('EXAMPLE: tpm bundle my-module /users/bundles/my-package/' + colors.reset); + log(''); + log('--- --- --- --- ---'); + log(''); log(colors.red + '$ tpm repository [repository_name] [repository_url]' + colors.reset); log(''); log(colors.dim + 'EXAMPLE: tpm repository local http://127.0.0.1:8000/'); From 07a4687f761b93270111a0dce37ecfb6df149f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Mar 2018 19:47:46 +0100 Subject: [PATCH 0086/1669] Fixed schema validation. --- changes.txt | 1 + utils.js | 30 ++++++++++++++++-------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/changes.txt b/changes.txt index 301513b1f..79a1e15b9 100755 --- a/changes.txt +++ b/changes.txt @@ -37,6 +37,7 @@ - fixed: `String.parseDate()` now it parses date to UTC correctly - fixed: `Date.format()` now it formats a date as UTC correctly - fixed: HTML compressor with `\r\n` (Windows line endings) +- fixed: schema validation ======= 2.9.4 (HOTFIX) diff --git a/utils.js b/utils.js index 8e5e0702d..da9b555f7 100755 --- a/utils.js +++ b/utils.js @@ -2048,23 +2048,25 @@ exports.validate_builder = function(model, error, schema, collection, path, inde // Another schema exports.validate_builder(value[j], error, TYPE.raw, collection, current + name + '[' + j + ']', j, undefined, pluspath); } else { + // Basic types var result = TYPE.validate ? TYPE.validate(value[j], model) : prepare(name, value, current + name + '[' + j + ']', model, schema, TYPE); - if (result === undefined) { + if (result == null) { result = validate_builder_default(name, value[j], TYPE); - if (result) + if (result == null || result === true) continue; - type = typeof(result); - if (type === 'string') { - if (result[0] === '@') - error.push(pluspath + name, '@', current + name + '[' + j + ']', j, entity.resourcePrefix + result.substring(1)); - else - error.push(pluspath + name, result, current + name + '[' + j + ']', j, prefix); - } else if (type === 'boolean') { - !result && error.push(pluspath + name, '@', current + name + '[' + j + ']', j, prefix); - } else if (result.isValid === false) - error.push(pluspath + name, result.error, current + name + '[' + j + ']', j, prefix); } + + type = typeof(result); + if (type === 'string') { + if (result[0] === '@') + error.push(pluspath + name, '@', current + name + '[' + j + ']', j, entity.resourcePrefix + result.substring(1)); + else + error.push(pluspath + name, result, current + name + '[' + j + ']', j, prefix); + } else if (type === 'boolean') { + !result && error.push(pluspath + name, '@', current + name + '[' + j + ']', j, prefix); + } else if (result.isValid === false) + error.push(pluspath + name, result.error, current + name + '[' + j + ']', j, prefix); } } continue; @@ -2077,9 +2079,9 @@ exports.validate_builder = function(model, error, schema, collection, path, inde } var result = TYPE.validate ? TYPE.validate(value, model) : prepare(name, value, current + name, model, schema, TYPE); - if (result === undefined) { + if (result == null) { result = validate_builder_default(name, value, TYPE); - if (result) + if (result == null || result === true) continue; } From 0b28c3b11667da1121a8ad4aefc10713202aaf9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Mar 2018 19:48:36 +0100 Subject: [PATCH 0087/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f31bc8897..32d1ba21e 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-11", + "version": "3.0.0-12", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 00b7ec83b09d5c54e917bebaf56a689d74f9f3a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Mar 2018 20:13:13 +0100 Subject: [PATCH 0088/1669] Added `operations`. --- debug.js | 1 + 1 file changed, 1 insertion(+) diff --git a/debug.js b/debug.js index d40411c55..57b4598c8 100644 --- a/debug.js +++ b/debug.js @@ -103,6 +103,7 @@ function runwatching() { U.combine(F.config['directory-components']), U.combine(F.config['directory-controllers']), U.combine(F.config['directory-definitions']), + U.combine(F.config['directory-operations']), U.combine(F.config['directory-isomorphic']), U.combine(F.config['directory-modules']), U.combine(F.config['directory-models']), From cd4b1c75e4223cb666060603c79cb3a77c9c79b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Mar 2018 21:29:21 +0100 Subject: [PATCH 0089/1669] Improved refreshing of `views` and `public` directories. --- debug.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/debug.js b/debug.js index 57b4598c8..305d02e8a 100644 --- a/debug.js +++ b/debug.js @@ -206,8 +206,19 @@ function runwatching() { } function isViewPublic(filename) { + + if (!isBUNDLE) + return false; + var fn = filename.substring(directory.length); - var dir = fn.substring(0, fn.indexOf('/', 1) + 1); + var index = fn.indexOf('/', 1); + var dir = fn.substring(0, index + 1); + + if (dir === F.config['directory-themes']) { + index = fn.indexOf('/', index + 1); + dir = fn.substring(index, fn.indexOf('/', index + 1) + 1); + } + return F.config['directory-views'] === dir || F.config['directory-public'] === dir ? fn : ''; } @@ -231,9 +242,7 @@ function runwatching() { } else { var ticks = stat.mtime.getTime(); if (files[filename] != null && files[filename] !== ticks) { - var log = stamp.replace('#', files[filename] === 0 ? 'ADD' : 'UPD') + prefix + normalize(filename.replace(directory, '')); - if (files[filename]) { var tmp = isViewPublic(filename); if (tmp) { From affa4e81c45441f8550c5309bc6e030086a9b226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 12 Mar 2018 21:14:05 +0100 Subject: [PATCH 0090/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 32d1ba21e..f80a896c8 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-12", + "version": "3.0.0-13", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 5cd5d407ae9d3447db893c9ba6644e30c09af670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Mar 2018 14:06:14 +0100 Subject: [PATCH 0091/1669] Fixed `debug.js`. --- debug.js | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/debug.js b/debug.js index 305d02e8a..693e8c787 100644 --- a/debug.js +++ b/debug.js @@ -79,11 +79,11 @@ function runwatching() { const FILENAME = U.getName(process.argv[1]); const directory = process.cwd(); const VERSION = F.version_header; - const TIME = 2000; + const SPEED = 1500; const REG_CONFIGS = /configs\//g; const REG_FILES = /config-debug|config-release|config|versions|workflows|sitemap|dependencies|\.js$|\.resource$/i; const REG_THEMES = /\/themes\//i; - const REG_COMPONENTS = /components\/.*?\.html/i; + const REG_COMPONENTS = /components\/.*?\.html|\.package\/.*?$/i; const REG_THEMES_INDEX = /themes(\/|\\)?[a-z0-9_.-]+(\/|\\)?index\.js$/i; const REG_EXTENSION = /\.(js|resource|package|bundle)$/i; @@ -138,9 +138,10 @@ function runwatching() { var isLoaded = false; var isSkip = false; var pidIncrease; - var speed = TIME; + var speed = SPEED; var isBUNDLE = false; var blacklist = {}; + var counter = 0; blacklist['/debug.pid'] = 1; blacklist['/debug.js'] = 1; @@ -159,30 +160,11 @@ function runwatching() { } function onFilter(path, isDirectory) { - if (isBUNDLE) { - if (isDirectory) - return SRC !== path; - var p = path.substring(directory.length); - return !blacklist[p]; - } + if (isBUNDLE) + return isDirectory ? SRC !== path : !blacklist[path.substring(directory.length)]; return isDirectory && REG_THEMES.test(path) ? REG_THEMES_INDEX.test(path) : isDirectory ? true : REG_EXTENSION.test(path) || REG_COMPONENTS.test(path) || REG_CONFIGS.test(path); } - function onIncrease(clear) { - - if (clear) { - clearTimeout(pidIncrease); - speed = TIME; - } - - pidIncrease = setTimeout(function() { - speed += TIME; - if (speed > 4000) - speed = 4000; - onIncrease(); - }, 120000); - } - function onComplete(f) { Fs.readdir(directory, function(err, arr) { @@ -240,6 +222,7 @@ function runwatching() { force = true; } } else { + var ticks = stat.mtime.getTime(); if (files[filename] != null && files[filename] !== ticks) { var log = stamp.replace('#', files[filename] === 0 ? 'ADD' : 'UPD') + prefix + normalize(filename.replace(directory, '')); @@ -266,13 +249,16 @@ function runwatching() { isLoaded = true; setTimeout(refresh_directory, speed); - onIncrease(); - if (status !== 1 || !force) + if (status !== 1 || !force) { + if (counter % 150 === 0) + speed = 6000; return; + } - onIncrease(true); restart(); + counter = 0; + speed = SPEED; var length = changes.length; for (var i = 0; i < length; i++) @@ -284,6 +270,7 @@ function runwatching() { } function refresh_directory() { + counter++; U.ls(directories, onComplete, onFilter); } From 0041dc4d8723107e0a45e010d5572de228aec313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Mar 2018 16:08:44 +0100 Subject: [PATCH 0092/1669] Improved timing. --- debug.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug.js b/debug.js index 693e8c787..db42b4c66 100644 --- a/debug.js +++ b/debug.js @@ -138,7 +138,7 @@ function runwatching() { var isLoaded = false; var isSkip = false; var pidIncrease; - var speed = SPEED; + var speed = 4000; var isBUNDLE = false; var blacklist = {}; var counter = 0; From 2c37416d09c45f00e105af8a3857494fa1bea47c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Mar 2018 20:05:42 +0100 Subject: [PATCH 0093/1669] Added a new change. --- changes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changes.txt b/changes.txt index 79a1e15b9..821da20ea 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,8 @@ - fixed: HTML compressor with `\r\n` (Windows line endings) - fixed: schema validation +- improved: `debug` mode timing with improved consumption + ======= 2.9.4 (HOTFIX) - fixed: mail attachments From 67a75372f3f42ad7418d44458d4d02cb3c5c8e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 15 Mar 2018 12:57:28 +0100 Subject: [PATCH 0094/1669] Improved timers. --- debug.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debug.js b/debug.js index db42b4c66..63828af80 100644 --- a/debug.js +++ b/debug.js @@ -248,17 +248,18 @@ function runwatching() { }, function() { isLoaded = true; - setTimeout(refresh_directory, speed); if (status !== 1 || !force) { if (counter % 150 === 0) speed = 6000; + setTimeout(refresh_directory, speed); return; } restart(); counter = 0; speed = SPEED; + setTimeout(refresh_directory, speed); var length = changes.length; for (var i = 0; i < length; i++) @@ -385,7 +386,7 @@ function runwatching() { process.exit(0); }); - }, 2000); + }, 3000); } restart(); @@ -395,7 +396,7 @@ function runwatching() { var filename = Path.join(directory, 'debug.pid'); if (Fs.existsSync(filename)) { Fs.unlinkSync(filename); - setTimeout(app, 2500); + setTimeout(app, 3500); } else app(); } From c57f81982d30b54c40867737a298a666881ff89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 16 Mar 2018 12:34:49 +0100 Subject: [PATCH 0095/1669] Added live reloading. --- changes.txt | 1 + debug.js | 51 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/changes.txt b/changes.txt index 821da20ea..36ac5e590 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: (IMPORTANT) bundles - added: (IMPORTANT) Total.js components can have async delegate +- added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` - added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) diff --git a/debug.js b/debug.js index 63828af80..ab1597316 100644 --- a/debug.js +++ b/debug.js @@ -42,6 +42,7 @@ module.exports = function(opt) { // options.inspector = 9229; // options.debugger = 40894; // options.watch = ['adminer']; + // options.livereload = true; }; process.on('uncaughtException', e => e.toString().indexOf('ESRCH') == -1 && console.log(e)); @@ -79,13 +80,15 @@ function runwatching() { const FILENAME = U.getName(process.argv[1]); const directory = process.cwd(); const VERSION = F.version_header; - const SPEED = 1500; const REG_CONFIGS = /configs\//g; const REG_FILES = /config-debug|config-release|config|versions|workflows|sitemap|dependencies|\.js$|\.resource$/i; const REG_THEMES = /\/themes\//i; const REG_COMPONENTS = /components\/.*?\.html|\.package\/.*?$/i; const REG_THEMES_INDEX = /themes(\/|\\)?[a-z0-9_.-]+(\/|\\)?index\.js$/i; const REG_EXTENSION = /\.(js|resource|package|bundle)$/i; + const REG_RELOAD = /\.(js|css|html|htm|jpg|png|gif|ico|svg|resource)$/i; + const isRELOAD = !!options.livereload; + const SPEED = isRELOAD ? 1000 : 1500; function copyFile(oldname, newname, callback) { var writer = Fs.createWriteStream(newname); @@ -138,10 +141,11 @@ function runwatching() { var isLoaded = false; var isSkip = false; var pidIncrease; - var speed = 4000; var isBUNDLE = false; var blacklist = {}; var counter = 0; + var WS = null; + var speed = isRELOAD ? 1000 : 4000; blacklist['/debug.pid'] = 1; blacklist['/debug.js'] = 1; @@ -149,12 +153,24 @@ function runwatching() { blacklist['/package.json'] = 1; blacklist['/readme.md'] = 1; + if (isRELOAD) { + F.console = NOOP; + F.websocket('/', function() { + var self = this; + self.autodestroy(function() { + WS = null; + }); + WS = self; + }); + F.http('release', { port: typeof(options.livereload) === 'number' ? options.livereload : 35729 }); + } + try { Fs.statSync(F.path.root(F.config['directory-bundles'])); isBUNDLE = true; } catch(e) {} - if (isBUNDLE) { + if (isBUNDLE || isRELOAD) { directories.push(U.combine(F.config['directory-public'])); directories.push(U.combine(F.config['directory-views'])); } @@ -162,11 +178,12 @@ function runwatching() { function onFilter(path, isDirectory) { if (isBUNDLE) return isDirectory ? SRC !== path : !blacklist[path.substring(directory.length)]; + if (isRELOAD) + return isDirectory ? true : REG_RELOAD.test(path); return isDirectory && REG_THEMES.test(path) ? REG_THEMES_INDEX.test(path) : isDirectory ? true : REG_EXTENSION.test(path) || REG_COMPONENTS.test(path) || REG_CONFIGS.test(path); } function onComplete(f) { - Fs.readdir(directory, function(err, arr) { var length = arr.length; @@ -187,9 +204,13 @@ function runwatching() { }); } + function livereload() { + isRELOAD && setTimeout2('livereload', () => WS && WS.send('reload'), 500); + } + function isViewPublic(filename) { - if (!isBUNDLE) + if (!isBUNDLE && !isRELOAD) return false; var fn = filename.substring(directory.length); @@ -205,6 +226,7 @@ function runwatching() { } function refresh() { + var reload = false; Object.keys(files).wait(function(filename, next) { Fs.stat(filename, function(err, stat) { @@ -215,8 +237,11 @@ function runwatching() { var tmp = isViewPublic(filename); var log = stamp.replace('#', 'REM') + prefix + normalize(filename.replace(directory, '')); if (tmp) { - Fs.unlinkSync(Path.join(SRC, tmp)); - console.log(log); + if (isBUNDLE) { + Fs.unlinkSync(Path.join(SRC, tmp)); + console.log(log); + } + reload = true; } else { changes.push(log); force = true; @@ -229,9 +254,12 @@ function runwatching() { if (files[filename]) { var tmp = isViewPublic(filename); if (tmp) { - copyFile(filename, Path.join(SRC, tmp)); + if (isBUNDLE) { + copyFile(filename, Path.join(SRC, tmp)); + console.log(log); + } files[filename] = ticks; - console.log(log); + reload = true; next(); return; } @@ -249,9 +277,11 @@ function runwatching() { isLoaded = true; + reload && livereload(); + if (status !== 1 || !force) { if (counter % 150 === 0) - speed = 6000; + speed = isRELOAD ? 3000 : 6000; setTimeout(refresh_directory, speed); return; } @@ -267,6 +297,7 @@ function runwatching() { changes = []; force = false; + livereload(); }, 3); } From b1ffe04f89657e9f838076cf65d80bc07d3c06ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 16 Mar 2018 12:57:09 +0100 Subject: [PATCH 0096/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f80a896c8..ab3c80bd7 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-13", + "version": "3.0.0-14", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 46f3bb88b9934cc20418b5b282d393b672bfc3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 19 Mar 2018 15:42:12 +0100 Subject: [PATCH 0097/1669] Fixed `U.atob()` + `U.btoa()`. --- changes.txt | 2 ++ test/test-utils.js | 14 ++++++++++---- utils.js | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/changes.txt b/changes.txt index 36ac5e590..8f7a6326b 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,8 @@ - fixed: `Date.format()` now it formats a date as UTC correctly - fixed: HTML compressor with `\r\n` (Windows line endings) - fixed: schema validation +- fixed: `U.atob()` +- fixed: `U.btoa()` - improved: `debug` mode timing with improved consumption diff --git a/test/test-utils.js b/test/test-utils.js index 0d1e9cca0..95abf2212 100755 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -612,12 +612,18 @@ function other() { var a = { buf: Buffer.from('123456') }; assert.ok(U.clone(a).buf !== a, 'Cloning buffers'); + + var input = '12345čťžýáýáííéídfsfgd'; + var a = U.btoa(input); + var b = U.atob(a); + + assert.ok(b === input, 'U.atob() / U.btoa()'); } function Utils_Ls2_StringFilter() { var result; var async = new Utils.Async(); - + async.await('U.ls2', function(next) { U.ls2( './app', @@ -628,7 +634,7 @@ function Utils_Ls2_StringFilter() { 'app' ); }); - + async.run(function() { assert.ok(result.files.length === 1, 'problem with number of files from U.ls2 string filter'); assert.ok(result.files[0].filename.indexOf('virtual.txt') !== -1, 'problem with files[0].filename from U.ls2 string filter'); @@ -640,7 +646,7 @@ function Utils_Ls2_StringFilter() { function Utils_Ls_RegExpFilter() { var result; var async = new Utils.Async(); - + async.await('U.ls', function(next) { U.ls( './app', @@ -651,7 +657,7 @@ function Utils_Ls_RegExpFilter() { /QQQ/ ); }); - + async.run(function() { assert.ok(result.files.length === 0, 'problem with files from U.ls regExp filter'); assert.ok(result.folders.length === 0, 'problem with folders from U.ls regExp filter'); diff --git a/utils.js b/utils.js index da9b555f7..dee5a46e0 100755 --- a/utils.js +++ b/utils.js @@ -792,11 +792,11 @@ exports.$$request = function(url, flags, data, cookies, headers, encoding, timeo }; exports.btoa = function(str) { - return (str instanceof Buffer) ? str.toString('base64') : exports.createBuffer(str.toString(), 'binary').toString('base64'); + return (str instanceof Buffer) ? str.toString('base64') : exports.createBuffer(str.toString(), 'utf8').toString('base64'); }; exports.atob = function(str) { - return exports.createBuffer(str, 'base64').toString('binary'); + return exports.createBuffer(str, 'base64').toString('utf8'); }; /** From 18f983b678b330d4eb85b5c060b46d74c1e31e34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 19 Mar 2018 15:42:21 +0100 Subject: [PATCH 0098/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ab3c80bd7..c4fd357f9 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-14", + "version": "3.0.0-15", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 6afac238a61c43b837e838157e11855e7697e93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 20 Mar 2018 11:05:12 +0100 Subject: [PATCH 0099/1669] Improved code. --- image.js | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/image.js b/image.js index 45c40fe73..2977e8a25 100755 --- a/image.js +++ b/image.js @@ -63,25 +63,17 @@ exports.measureJPG = function(buffer) { var o = 0; var jpeg = 0xff == buffer[0] && 0xd8 == buffer[1]; - if (!jpeg) - return; - - o += 2; - - while (o < len) { + if (jpeg) { + o += 2; + while (o < len) { + while (0xff != buffer[o]) o++; + while (0xff == buffer[o]) o++; + if (sof[buffer[o]]) + return { width: u16(buffer, o + 6), height: u16(buffer, o + 4) }; + else + o += u16(buffer, ++o); - while (0xff != buffer[o]) o++; - while (0xff == buffer[o]) o++; - - if (!sof[buffer[o]]) { - o += u16(buffer, ++o); - continue; } - - var w = u16(buffer, o + 6); - var h = u16(buffer, o + 4); - - return { width: w, height: h }; } return null; From 1cf43a9ba15ff806137d7bc8ee855e26ff38f4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 20 Mar 2018 16:10:16 +0100 Subject: [PATCH 0100/1669] Improved code. --- builders.js | 2 ++ utils.js | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/builders.js b/builders.js index d52fcfb13..ad7a8eedb 100755 --- a/builders.js +++ b/builders.js @@ -4106,6 +4106,8 @@ RESTBuilder.prototype.cookie = function(name, value) { }; RESTBuilder.prototype.header = function(name, value) { + if (name === 'content-type') + name = 'Content-Type'; this.$headers[name] = value; return this; }; diff --git a/utils.js b/utils.js index dee5a46e0..d4e9050db 100755 --- a/utils.js +++ b/utils.js @@ -75,6 +75,7 @@ const EMPTYARRAY = []; const EMPTYOBJECT = {}; const NODEVERSION = parseFloat(process.version.toString().replace('v', '').replace(/\./g, '')); const STREAMPIPE = { end: false }; +const CT = 'Content-Type'; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -469,22 +470,27 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin headers['X-Requested-With'] = 'XMLHttpRequest'; break; case 'plain': - headers['Content-Type'] = 'text/plain'; + if (!headers[CT]) + headers[CT] = 'text/plain'; break; case 'html': - headers['Content-Type'] = 'text/html'; + if (!headers[CT]) + headers[CT] = 'text/html'; break; case 'raw': type = 3; - headers['Content-Type'] = 'application/octet-stream'; + if (!headers[CT]) + headers[CT] = 'application/octet-stream'; break; case 'json': - headers['Content-Type'] = 'application/json'; + if (!headers[CT]) + headers[CT] = 'application/json'; !method && (method = 'POST'); type = 1; break; case 'xml': - headers['Content-Type'] = 'text/xml'; + if (!headers[CT]) + headers[CT] = 'text/xml'; !method && (method = 'POST'); type = 2; break; @@ -504,7 +510,7 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin options.upload = true; options.files = files || EMPTYARRAY; options.boundary = '----totaljs' + Math.random().toString(16).substring(2); - headers['Content-Type'] = 'multipart/form-data; boundary=' + options.boundary; + headers[CT] = 'multipart/form-data; boundary=' + options.boundary; break; case 'post': @@ -512,7 +518,7 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin case 'delete': case 'patch': method = flags[i].toUpperCase(); - !headers['Content-Type'] && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); + !headers[CT] && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); break; case 'dnscache': From 458ce7f2330abad16296b21ed5746f4174c2dfd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 20 Mar 2018 19:41:38 +0100 Subject: [PATCH 0101/1669] Added `RESTBuilder.type()` for changing of content-type. --- builders.js | 19 +++++++++---------- changes.txt | 1 + 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/builders.js b/builders.js index ad7a8eedb..9f453cfdc 100755 --- a/builders.js +++ b/builders.js @@ -4106,31 +4106,30 @@ RESTBuilder.prototype.cookie = function(name, value) { }; RESTBuilder.prototype.header = function(name, value) { - if (name === 'content-type') - name = 'Content-Type'; this.$headers[name] = value; return this; }; +RESTBuilder.prototype.type = function(value) { + this.$headers['Content-Type'] = value; + return this; +}; + RESTBuilder.prototype.cache = function(expire) { this.$cache_expire = expire; return this; }; RESTBuilder.prototype.set = function(name, value) { - if (!this.$data) this.$data = {}; - if (typeof(name) !== 'object') { this.$data[name] = value; - return this; + } else { + var arr = Object.keys(name); + for (var i = 0, length = arr.length; i < length; i++) + this.$data[arr[i]] = name[arr[i]]; } - - var arr = Object.keys(name); - for (var i = 0, length = arr.length; i < length; i++) - this.$data[arr[i]] = name[arr[i]]; - return this; }; diff --git a/changes.txt b/changes.txt index 8f7a6326b..712f81b19 100755 --- a/changes.txt +++ b/changes.txt @@ -5,6 +5,7 @@ - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` +- added: `RESTBuilder.type(new-content-type)` - added: `CONVERT(obj, schema)` for quick converting values like Schema (more in docs.) - added: `Capitalize2` schema type which converts only the first char - added: `MailMessage.high()` sets `high` priority of the email messsage From 9afd85d92143abbbaa8d7a7442e1433fc7424cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 21 Mar 2018 20:09:18 +0100 Subject: [PATCH 0102/1669] Updated `blacklist` for bundling. --- bundles.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundles.js b/bundles.js index c3aca8ca2..5707550d1 100755 --- a/bundles.js +++ b/bundles.js @@ -30,8 +30,8 @@ exports.make = function(callback) { blacklist['/node_modules/'] = 1; // blacklist['/debug.js'] = 1; blacklist['/debug.pid'] = 1; - blacklist['/package.json'] = 1; - blacklist['/bundle.json'] = 1; + //blacklist['/package.json'] = 1; + blacklist['/package-lock.json'] = 1; var Files = []; var Dirs = []; From 528c2560ebecac16ff1b024b99494ea4184b309d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 21 Mar 2018 20:36:44 +0100 Subject: [PATCH 0103/1669] Fixed `livereload` with `bundles`. --- debug.js | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/debug.js b/debug.js index ab1597316..a4da13991 100644 --- a/debug.js +++ b/debug.js @@ -27,7 +27,8 @@ const Path = require('path'); const Fs = require('fs'); const debugging = process.argv.indexOf('debugging') !== -1; -const isWindows = require('os').platform().substring(0, 3).toLowerCase() === 'win'; +const Os = require('os'); +const isWindows = Os.platform().substring(0, 3).toLowerCase() === 'win'; var first = process.argv.indexOf('restart') === -1; var options = null; @@ -154,15 +155,18 @@ function runwatching() { blacklist['/readme.md'] = 1; if (isRELOAD) { - F.console = NOOP; - F.websocket('/', function() { - var self = this; - self.autodestroy(function() { - WS = null; + var tmppath = Path.join(Os.tmpdir(), 'totaljslivereload'); + Fs.mkdir(tmppath, function() { + F.console = NOOP; + F.websocket('/', function() { + var self = this; + self.autodestroy(function() { + WS = null; + }); + WS = self; }); - WS = self; - }); - F.http('release', { port: typeof(options.livereload) === 'number' ? options.livereload : 35729 }); + F.http('release', { port: typeof(options.livereload) === 'number' ? options.livereload : 35729, directory: tmppath }); + });; } try { From b6cfca6c87eae322f6e176883ccfb5817b625aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 21 Mar 2018 21:26:46 +0100 Subject: [PATCH 0104/1669] Updated `@{import()}` by adding `livereload` support. --- changes.txt | 1 + index.js | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/changes.txt b/changes.txt index 712f81b19..97ee39243 100755 --- a/changes.txt +++ b/changes.txt @@ -28,6 +28,7 @@ - updated: `ROUTE()` supports a new HTTP method definition `ROUTE('GET /api/users/')`, `ROUTE('POST /api/users/')`, etc. - updaded: `tpm` supports a new command called `bundle`, for example: `tpm bundle cms` - updated: `F.restore()` filter can return a new filename (for files only) +- updated: `@{import('livereload')}` or `@{import('livereload wss://mywebsite.com')}` supports `livereload` value and it's rendered in `debug` mode only - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index de96dddaa..26d4cdac9 100644 --- a/index.js +++ b/index.js @@ -11311,6 +11311,14 @@ Controller.prototype.$import = function() { continue; } + if (filename[0] === 'l' && filename[9] === 'd' && filename.substring(0, 10) === 'livereload') { + if (DEBUG) { + var url = filename.substring(11).trim(); + builder += ''; + } + continue; + } + var extension = filename.substring(filename.lastIndexOf('.')); var tag = filename[0] !== '!'; if (!tag) From d8af14c8bc97fb7dbf4f08b2db367102980f77d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 22 Mar 2018 13:17:45 +0100 Subject: [PATCH 0105/1669] Improved code. --- debug.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug.js b/debug.js index a4da13991..a81551321 100644 --- a/debug.js +++ b/debug.js @@ -141,7 +141,6 @@ function runwatching() { var pid = ''; var isLoaded = false; var isSkip = false; - var pidIncrease; var isBUNDLE = false; var blacklist = {}; var counter = 0; @@ -166,7 +165,7 @@ function runwatching() { WS = self; }); F.http('release', { port: typeof(options.livereload) === 'number' ? options.livereload : 35729, directory: tmppath }); - });; + }); } try { From 19dd1db93545e2dafc860399f06ada3216c8e081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 22 Mar 2018 21:21:31 +0100 Subject: [PATCH 0106/1669] Added `livereload`. --- helpers/debug.js | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/debug.js b/helpers/debug.js index 98181945e..1161565cc 100644 --- a/helpers/debug.js +++ b/helpers/debug.js @@ -12,5 +12,6 @@ const options = {}; // options.sleep = 3000; // options.inspector = 9229; // options.watch = ['private']; +// options.livereload = true; require('total.js/debug')(options); \ No newline at end of file From 2a915f6cd071255cb4caccd10a1c5ef24029c7ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 22 Mar 2018 21:36:06 +0100 Subject: [PATCH 0107/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c4fd357f9..efa3a69f4 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-15", + "version": "3.0.0-16", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 58f14599ce8c951eb92bf62316a12ed8816fafab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Mar 2018 22:26:36 +0100 Subject: [PATCH 0108/1669] Fixed routing. --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 26d4cdac9..c310e5700 100644 --- a/index.js +++ b/index.js @@ -1677,6 +1677,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { } if (method) { + !flags && (flags = []); flags.push(method); method = ''; } From 82dd19d6b3ea0cfbfd906160ce3e643f1ebc4cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Mar 2018 12:26:15 +0100 Subject: [PATCH 0109/1669] Improve reading DB. --- nosql.js | 20 +++++++++++--------- utils.js | 10 ++++++---- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/nosql.js b/nosql.js index cac11e45e..e03399159 100755 --- a/nosql.js +++ b/nosql.js @@ -65,6 +65,8 @@ const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); }; +const NEWLINEBUF = framework_utils.createBuffer('\n', 'utf8'); + Object.freeze(EMPTYARRAY); function Database(name, filename) { @@ -213,7 +215,7 @@ Database.prototype.backups = function(filter, callback) { var stream = Fs.createReadStream(self.filenameBackup); var output = []; - stream.on('data', U.streamer('\n', function(item, index) { + stream.on('data', U.streamer(NEWLINEBUF, function(item, index) { var end = item.indexOf('|', item.indexOf('|') + 2); var meta = item.substring(0, end); var arr = meta.split('|'); @@ -805,7 +807,7 @@ Database.prototype.$update = function() { var length = filter.length; var change = false; - reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -1069,7 +1071,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { if (first && length > 1) first = false; - reader && reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -1436,7 +1438,7 @@ Database.prototype.$views = function() { var reader = Fs.createReadStream(self.filename); - reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -1591,7 +1593,7 @@ Database.prototype.$remove = function() { if (F.isCluster && self.$lock()) return; - reader && reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -2409,7 +2411,7 @@ Counter.prototype.convert2 = function(filename) { var writer = Fs.createWriteStream(filename + '2'); var self = this; self.type = 100; - reader.on('data', framework_utils.streamer(NEWLINE, function(value) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { var arr = value.split(';'); var years = {}; for (var i = 1, length = arr.length; i < length; i++) { @@ -2826,7 +2828,7 @@ Counter.prototype.read = function(options, callback, reader) { callback(null, single ? (options.subtype ? EMPTYARRAY : 0) : (all ? EMPTYARRAY : output)); }); - reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { var index = value.indexOf('='); var key = value.substring(7, index); @@ -3028,7 +3030,7 @@ Counter.prototype.stats = Counter.prototype.stats_sum = function(top, year, mont } else opt.type = type; - reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { var index = value.indexOf('='); if (value.substring(0, 3) !== opt.type || (year && value.substring(3, 7) !== year)) @@ -3398,7 +3400,7 @@ Counter.prototype.save = function() { writer.end(); }; - reader.on('data', framework_utils.streamer(NEWLINE, function(value, index) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { var id = value.substring(0, value.indexOf('=')); var count = cache[id]; diff --git a/utils.js b/utils.js index d4e9050db..7bce8da39 100755 --- a/utils.js +++ b/utils.js @@ -1596,8 +1596,8 @@ exports.isRelative = function(url) { /** * Streamer method - * @param {String} beg - * @param {String} end + * @param {String/Buffer} beg + * @param {String/Buffer} end * @param {Function(value, index)} callback */ exports.streamer = function(beg, end, callback, skip, stream) { @@ -1622,8 +1622,10 @@ exports.streamer = function(beg, end, callback, skip, stream) { if (skip === undefined) skip = 0; - beg = exports.createBuffer(beg, 'utf8'); - if (end) + if (!(beg instanceof Buffer)) + beg = exports.createBuffer(beg, 'utf8'); + + if (end && !(end instanceof Buffer)) end = exports.createBuffer(end, 'utf8'); if (!end) { From 83eb46aea0610e9db7eeaf4400ca775d9abab75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Mar 2018 13:35:46 +0100 Subject: [PATCH 0110/1669] Imrpoved code. --- nosql.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index e03399159..1e36848fc 100755 --- a/nosql.js +++ b/nosql.js @@ -265,7 +265,7 @@ Database.prototype.insert = function(doc, unique) { var json = framework_builders.isSchema(doc) ? doc.$clean() : doc; self.pending_append.push({ doc: JSON.stringify(json), builder: builder }); setImmediate(next_operation, self, 1); - self.emit('insert', json); + self.$events.insert && self.emit('insert', json); return builder; }; @@ -834,7 +834,8 @@ Database.prototype.$update = function() { } else output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - self.emit(item.keys ? 'modify' : 'update', output); + var e = item.keys ? 'modify' : 'update'; + self.$events[e] && self.emit(e, output); item.count++; change = true; doc = output; @@ -954,7 +955,8 @@ Database.prototype.$update_inmemory = function() { } else doc = typeof(item.doc) === 'function' ? item.doc(doc) : item.doc; - self.emit(item.keys ? 'modify' : 'update', doc); + var e = item.keys ? 'modify' : 'update'; + self.$events[e] && self.emit(e, doc); item.count++; change = true; } @@ -969,7 +971,8 @@ Database.prototype.$update_inmemory = function() { item.builder.$insertcallback && item.builder.$insertcallback(item.insert); self.insert(item.insert).$callback = item.builder.$callback; } else { - item.count && self.emit(item.keys ? 'modify' : 'update', item.doc); + var e = item.keys ? 'modify' : 'update'; + item.count && self.$events[e] && self.emit(e, item.doc); item.builder.$log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } @@ -1619,7 +1622,7 @@ Database.prototype.$remove = function() { item.backup && item.backup.write(value); item.count++; } - self.emit('remove', json); + self.$events.remove && self.emit('remove', json); change = true; } else writer.write(value); @@ -1712,7 +1715,7 @@ Database.prototype.$remove_inmemory = function() { item.count++; } change = true; - self.emit('remove', json); + self.$events.remove && self.emit('remove', json); } else arr.push(json); } From f2b80961bbddc646cde6e40702acc39940a4a7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 01:04:37 +0100 Subject: [PATCH 0111/1669] Added `String.crc32()`. --- changes.txt | 2 ++ utils.js | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/changes.txt b/changes.txt index 97ee39243..6345fe084 100755 --- a/changes.txt +++ b/changes.txt @@ -21,6 +21,8 @@ - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - added: new directory `schemas` with a new configuration item `directory-schemas' - added: new directory `operations` with a new configuration item `directory-operations' +- added: `String.crc32([unsigned])` +- added: `U.hash('crc32')` and `U.hash('crc32unsigned')` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` diff --git a/utils.js b/utils.js index 7bce8da39..35e32f503 100755 --- a/utils.js +++ b/utils.js @@ -76,6 +76,7 @@ const EMPTYOBJECT = {}; const NODEVERSION = parseFloat(process.version.toString().replace('v', '').replace(/\./g, '')); const STREAMPIPE = { end: false }; const CT = 'Content-Type'; +const CRC32TABLE = '00000000,77073096,EE0E612C,990951BA,076DC419,706AF48F,E963A535,9E6495A3,0EDB8832,79DCB8A4,E0D5E91E,97D2D988,09B64C2B,7EB17CBD,E7B82D07,90BF1D91,1DB71064,6AB020F2,F3B97148,84BE41DE,1ADAD47D,6DDDE4EB,F4D4B551,83D385C7,136C9856,646BA8C0,FD62F97A,8A65C9EC,14015C4F,63066CD9,FA0F3D63,8D080DF5,3B6E20C8,4C69105E,D56041E4,A2677172,3C03E4D1,4B04D447,D20D85FD,A50AB56B,35B5A8FA,42B2986C,DBBBC9D6,ACBCF940,32D86CE3,45DF5C75,DCD60DCF,ABD13D59,26D930AC,51DE003A,C8D75180,BFD06116,21B4F4B5,56B3C423,CFBA9599,B8BDA50F,2802B89E,5F058808,C60CD9B2,B10BE924,2F6F7C87,58684C11,C1611DAB,B6662D3D,76DC4190,01DB7106,98D220BC,EFD5102A,71B18589,06B6B51F,9FBFE4A5,E8B8D433,7807C9A2,0F00F934,9609A88E,E10E9818,7F6A0DBB,086D3D2D,91646C97,E6635C01,6B6B51F4,1C6C6162,856530D8,F262004E,6C0695ED,1B01A57B,8208F4C1,F50FC457,65B0D9C6,12B7E950,8BBEB8EA,FCB9887C,62DD1DDF,15DA2D49,8CD37CF3,FBD44C65,4DB26158,3AB551CE,A3BC0074,D4BB30E2,4ADFA541,3DD895D7,A4D1C46D,D3D6F4FB,4369E96A,346ED9FC,AD678846,DA60B8D0,44042D73,33031DE5,AA0A4C5F,DD0D7CC9,5005713C,270241AA,BE0B1010,C90C2086,5768B525,206F85B3,B966D409,CE61E49F,5EDEF90E,29D9C998,B0D09822,C7D7A8B4,59B33D17,2EB40D81,B7BD5C3B,C0BA6CAD,EDB88320,9ABFB3B6,03B6E20C,74B1D29A,EAD54739,9DD277AF,04DB2615,73DC1683,E3630B12,94643B84,0D6D6A3E,7A6A5AA8,E40ECF0B,9309FF9D,0A00AE27,7D079EB1,F00F9344,8708A3D2,1E01F268,6906C2FE,F762575D,806567CB,196C3671,6E6B06E7,FED41B76,89D32BE0,10DA7A5A,67DD4ACC,F9B9DF6F,8EBEEFF9,17B7BE43,60B08ED5,D6D6A3E8,A1D1937E,38D8C2C4,4FDFF252,D1BB67F1,A6BC5767,3FB506DD,48B2364B,D80D2BDA,AF0A1B4C,36034AF6,41047A60,DF60EFC3,A867DF55,316E8EEF,4669BE79,CB61B38C,BC66831A,256FD2A0,5268E236,CC0C7795,BB0B4703,220216B9,5505262F,C5BA3BBE,B2BD0B28,2BB45A92,5CB36A04,C2D7FFA7,B5D0CF31,2CD99E8B,5BDEAE1D,9B64C2B0,EC63F226,756AA39C,026D930A,9C0906A9,EB0E363F,72076785,05005713,95BF4A82,E2B87A14,7BB12BAE,0CB61B38,92D28E9B,E5D5BE0D,7CDCEFB7,0BDBDF21,86D3D2D4,F1D4E242,68DDB3F8,1FDA836E,81BE16CD,F6B9265B,6FB077E1,18B74777,88085AE6,FF0F6A70,66063BCA,11010B5C,8F659EFF,F862AE69,616BFFD3,166CCF45,A00AE278,D70DD2EE,4E048354,3903B3C2,A7672661,D06016F7,4969474D,3E6E77DB,AED16A4A,D9D65ADC,40DF0B66,37D83BF0,A9BCAE53,DEBB9EC5,47B2CF7F,30B5FFE9,BDBDF21C,CABAC28A,53B39330,24B4A3A6,BAD03605,CDD70693,54DE5729,23D967BF,B3667A2E,C4614AB8,5D681B02,2A6F2B94,B40BBE37,C30C8EA1,5A05DF1B,2D02EF8D'.split(',').map(s => parseInt(s, 16)); exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -2826,11 +2827,23 @@ String.prototype.hash = function(type, salt) { return str.sha256(); case 'sha512': return str.sha512(); + case 'crc32': + return str.crc32(); + case 'crc32unsigned': + return str.crc32(true); default: return string_hash(str); } }; +String.prototype.crc32 = function(unsigned) { + var crc = -1; + for (var i = 0, length = this.length; i < length; i++) + crc = (crc >>> 8) ^ CRC32TABLE[(crc ^ this.charCodeAt(i)) & 0xFF]; + var val = crc ^ (-1); + return unsigned ? val >>> 0 : val; +}; + function string_hash(s, convert) { var hash = 0; if (s.length === 0) From 574eb3f3090c72eb624b0fa8f929fc81e951fa88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 01:05:54 +0100 Subject: [PATCH 0112/1669] Added NoSQL embedded worker. --- changes.txt | 2 + index.js | 16 +- nosql.js | 1323 ++++++++++++++++++++++++--------------------------- 3 files changed, 636 insertions(+), 705 deletions(-) diff --git a/changes.txt b/changes.txt index 6345fe084..689d274eb 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: (IMPORTANT) bundles - added: (IMPORTANT) Total.js components can have async delegate +- added: (IMPORTANT) NoSQL worker - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` @@ -47,6 +48,7 @@ - fixed: `U.btoa()` - improved: `debug` mode timing with improved consumption +- improved: NoSQL embedded database ======= 2.9.4 (HOTFIX) diff --git a/index.js b/index.js index c310e5700..6472c8184 100644 --- a/index.js +++ b/index.js @@ -287,6 +287,7 @@ global.WTF = (message, name, uri) => F.problem(message, name, uri); global.NOBIN = (name) => F.nosql(name).binary; global.NOCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); +global.NOSQLWORKER = () => global.framework_nosql.worker(); global.CONFIG = (name) => F.config[name]; global.UPTODATE = (type, url, options, interval, callback) => F.uptodate(type, url, options, interval, callback); global.INSTALL = (type, name, declaration, options, callback) => F.install(type, name, declaration, options, callback); @@ -5937,6 +5938,10 @@ F.backup = function(filename, filelist, callback, filter) { var writer = Fs.createWriteStream(filename); + writer.on('finish', function() { + callback && Fs.stat(filename, (e, stat) => callback(null, { filename: filename, files: counter, size: stat.size })); + }); + filelist.wait(function(item, next) { if (item[0] !== '/') @@ -5969,9 +5974,8 @@ F.backup = function(filename, filelist, callback, filter) { } var data = U.createBufferSize(0); - writer.write(item.padRight(padding) + ':'); - CLEANUP(Fs.createReadStream(file).pipe(Zlib.createGzip(GZIPFILE)).on('data', function(chunk) { + Fs.createReadStream(file).pipe(Zlib.createGzip(GZIPFILE)).on('data', function(chunk) { CONCAT[0] = data; CONCAT[1] = chunk; @@ -5983,17 +5987,15 @@ F.backup = function(filename, filelist, callback, filter) { data = data.slice(data.length - remaining); } - }), function() { + }).on('end', function() { data.length && writer.write(data.toString('base64')); writer.write('\n', 'utf8'); counter++; - next(); + setImmediate(next); }); }); - }, function() { - callback && Fs.stat(filename, (e, stat) => callback(null, { filename: filename, files: counter, size: stat.size })); - }); + }, () => writer.end()); }); return F; diff --git a/nosql.js b/nosql.js index 1e36848fc..6a67567db 100755 --- a/nosql.js +++ b/nosql.js @@ -53,12 +53,6 @@ const NEWLINE = '\n'; const EMPTYARRAY = []; const REG_CLEAN = /^[\s]+|[\s]+$/g; const INMEMORY = {}; -const CLUSTER_TIMEOUT = 150; -const CLUSTER_LOCK = { TYPE: 'nosql-lock' }; -const CLUSTER_UNLOCK = { TYPE: 'nosql-unlock' }; -const CLUSTER_META = { TYPE: 'nosql-meta' }; -const CLUSTER_LOCK_COUNTER = { TYPE: 'nosql-counter-lock' }; -const CLUSTER_UNLOCK_COUNTER = { TYPE: 'nosql-counter-unlock' }; const FLAGS_READ = ['get']; const COUNTER_MMA = [0, 0]; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { @@ -66,9 +60,240 @@ const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { }; const NEWLINEBUF = framework_utils.createBuffer('\n', 'utf8'); +const CACHE = {}; +var FORK; +var FORKCALLBACKS; Object.freeze(EMPTYARRAY); +exports.worker = function() { + FORKCALLBACKS = {}; + FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory }); + FORK.send({ TYPE: 'init', directory: F.path.root() }); + FORK.on('message', function(msg) { + switch (msg.TYPE) { + case 'find': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.builder.$callback2(msg.err, msg.response, msg.count, msg.repository); + break; + case 'count': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.builder.$callback2(msg.err, msg.response, msg.count, msg.repository); + break; + case 'insert': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.builder.$callback(msg.err, msg.response); + break; + case 'update': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.builder.$callback(msg.err, msg.response); + break; + case 'remove': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.builder.$callback(msg.err, msg.response); + break; + case 'backup': + case 'restore': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.callback(msg.err, msg.response); + break; + case 'counter.read': + case 'counter.stats': + case 'counter.clear': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.callback(msg.err, msg.response); + break; + } + delete FORKCALLBACKS[msg.id]; + }); + + var CMD = {}; + + function send(instance, type) { + var obj = {}; + obj.type = type; + obj.name = instance.name; + obj.time = Date.now(); + if (arguments.length > 2) { + obj.arg = []; + for (var i = 2; i < arguments.length; i++) + obj.arg.push(arguments[i]); + } + setImmediate(send2, obj); + return obj; + } + + function notify(instance, type) { + var obj = {}; + obj.type = type; + obj.name = instance.name; + obj.time = Date.now(); + if (arguments.length > 2) { + obj.arg = []; + for (var i = 2; i < arguments.length; i++) + obj.arg.push(arguments[i]); + } + setImmediate(send2, obj, false); + return obj; + } + + function send2(obj, callback) { + CMD.TYPE = obj.type; + CMD.arg = obj.arg; + CMD.data = obj.builder ? obj.builder.stringify() : null; + CMD.name = obj.name; + if (callback !== false) { + CMD.id = Math.random().toString(32).substring(2); + FORKCALLBACKS[CMD.id] = obj; + } + FORK.send(CMD); + } + + var DP = Database.prototype; + var CP = Counter.prototype; + + DP.once = DP.on = DP.emit = DP.removeListener = DP.removeAllListeners = CP.on = CP.once = CP.emit = CP.removeListener = CP.removeAllListeners = function() { + console.log('ERROR --> NoSQL events are not supported in fork mode.'); + }; + + Database.prototype.find = function(view) { + return send(this, 'find', view).builder = new DatabaseBuilder(this); + }; + + Database.prototype.top = function(max, view) { + var builder = new DatabaseBuilder(this); + builder.take(max); + return send(this, 'find', view).builder = builder; + }; + + Database.prototype.one = function(view) { + var builder = new DatabaseBuilder(this); + builder.first(); + return send(this, 'one', view).builder = builder; + }; + + Database.prototype.insert = function(doc, unique) { + + var self = this; + var builder; + + if (unique) { + builder = self.one(); + + var callback; + + builder.callback(function(err, d) { + if (d) + callback && callback(null, 0); + else + self.insert(doc).callback(callback); + }); + + builder.callback = function(fn) { + callback = fn; + return builder; + }; + + return builder; + } + + return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); + }; + + Database.prototype.count = function(view) { + var builder = new DatabaseBuilder(this); + return send(this, 'count', view).builder = builder; + }; + + Database.prototype.view = function(name) { + var builder = new DatabaseBuilder(this); + builder.id('$view_' + name); + return send(this, 'view', name).builder = builder; + }; + + Database.prototype.update = function(doc, insert) { + return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + }; + + Database.prototype.modify = function(doc, insert) { + return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + }; + + Database.prototype.restore = function(filename, callback) { + var obj = send(this, 'restore', filename); + obj.callback = callback; + return this; + }; + + Database.prototype.backup = function(filename, callback) { + var obj = send(this, 'backup', filename); + obj.callback = callback; + return this; + }; + + Database.prototype.refresh = function() { + notify(this, 'refresh'); + return this; + }; + + Database.prototype.drop = function() { + notify(this, 'drop'); + return this; + }; + + Database.prototype.clear = Database.prototype.remove = function(filename) { + return send(this, 'remove', filename).builder = new DatabaseBuilder(this); + }; + + Counter.prototype.min = function(id, count) { + notify(this.db, 'counter.min', id, count); + return this; + }; + + Counter.prototype.max = function(id, count) { + notify(this.db, 'counter.max', id, count); + return this; + }; + + Counter.prototype.inc = Counter.prototype.hit = function(id, count) { + notify(this.db, 'counter.hit', id, count); + return this; + }; + + Counter.prototype.remove = function(id) { + notify(this.db, 'counter.remove', id); + return this; + }; + + Counter.prototype.read = function(options, callback) { + send(this.db, 'counter.read', options).callback = callback; + return this; + }; + + Counter.prototype.stats = Counter.prototype.stats_sum = function(top, year, month, day, type, callback) { + + if (typeof(day) == 'function') { + callback = day; + day = null; + } else if (typeof(month) == 'function') { + callback = month; + month = null; + } else if (typeof(year) === 'function') { + callback = year; + year = month = null; + } + + send(this.db, 'counter.stats', top, year, month, day, type).callback = callback; + return this; + }; + + Counter.prototype.clear = function(callback) { + send(this.db, 'counter.clear').callback = callback; + return this; + }; + +}; + function Database(name, filename) { var self = this; var http = filename.substring(0, 6); @@ -100,7 +325,6 @@ function Database(name, filename) { self.$timeoutmeta; self.$events = {}; self.$free = true; - self.locked = false; self.renaming = false; } @@ -170,11 +394,6 @@ exports.DatabaseCounter = Counter; exports.DatabaseBinary = Binary; exports.load = function(name, filename) { - CLUSTER_LOCK.id = F.id; - CLUSTER_UNLOCK.id = F.id; - CLUSTER_META.id = F.id; - CLUSTER_LOCK_COUNTER.id = F.id; - CLUSTER_UNLOCK_COUNTER.id = F.id; return new Database(name, filename); }; @@ -230,8 +449,7 @@ Database.prototype.backups = function(filter, callback) { return self; }; -function next_operation(self, type, builder) { - builder && builder.$sortinline(); +function next_operation(self, type) { self.next(type); } @@ -307,11 +525,13 @@ Database.prototype.restore = function(filename, callback) { throw new Error('Database can\'t be restored because it\'s busy.'); self.type = 9; + F.restore(filename, F.path.root(), function(err, response) { self.type = 0; !err && self.refresh(); callback && callback(err, response); }); + }); return self; }; @@ -367,7 +587,6 @@ Database.prototype.backup = function(filename, callback) { }); pending.async(function() { - if (list.length) F.backup(filename, list, callback); else @@ -397,13 +616,13 @@ Database.prototype.backup2 = function(filename, remove) { stream.pipe(Fs.createWriteStream(filename || self.filenameBackup2)); stream.on('error', function(err) { - builder.$log && builder.$log(); + builder.$options.log && builder.log(); builder.$callback && builder.$callback(errorhandling(err, builder)); builder.$callback = null; }); stream.on('end', function() { - builder.$log && builder.$log(); + builder.$options.log && builder.log(); builder.$callback && builder.$callback(errorhandling(null, builder, true), true); builder.$callback = null; }); @@ -457,10 +676,10 @@ Database.prototype.find = function(view) { if (view) { self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); - setImmediate(next_operation, self, 6, builder); + setImmediate(next_operation, self, 6); } else { self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); - setImmediate(next_operation, self, 4, builder); + setImmediate(next_operation, self, 4); } return builder; @@ -496,10 +715,10 @@ Database.prototype.one = function(view) { if (view) { self.pending_reader_view.push({ builder: builder, count: 0, view: view }); - setImmediate(next_operation, self, 6, builder); + setImmediate(next_operation, self, 6); } else { self.pending_reader.push({ builder: builder, count: 0 }); - setImmediate(next_operation, self, 4, builder); + setImmediate(next_operation, self, 4); } return builder; @@ -512,10 +731,10 @@ Database.prototype.top = function(max, view) { if (view) { self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); - setImmediate(next_operation, self, 6, builder); + setImmediate(next_operation, self, 6); } else { self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); - setImmediate(next_operation, self, 4, builder); + setImmediate(next_operation, self, 4); } return builder; @@ -526,24 +745,14 @@ Database.prototype.view = function(name) { this.views[name] = {}; this.views[name] = builder; this.views[name].$filename = this.filename.replace(/\.nosql/, '#' + name + '.nosql'); + builder.id('$view_' + name); return builder; }; -Database.prototype.lock = function() { - this.locked = true; - return this; -}; - -Database.prototype.unlock = function() { - this.locked = false; - next_operation(this, 0); - return this; -}; - Database.prototype.next = function(type) { // 9: renaming file (by updating/removing) - if (this.locked || (type && this.step === 9)) + if (type && this.step === 9) return; if (this.renaming) { @@ -610,38 +819,10 @@ Database.prototype.refresh = function() { if (this.views) { this.pending_views = true; setImmediate(next_operation, this, 5); - } else if (F.cluster) - this.$unlock(); + } return this; }; -Database.prototype.$lock = function() { - if (this.lockwait === 2) - return false; - CLUSTER_LOCK.name = this.name; - process.send(CLUSTER_LOCK); - if (this.lockwait === 1) - return true; - if (this.lockwait === 2) - return false; - this.lockwait = 1; - setTimeout(locker_timeout, CLUSTER_TIMEOUT, this); - return true; -}; - -function locker_timeout(self) { - self.lockwait = 2; - self.next(0); -} - -Database.prototype.$unlock = function() { - if (!this.lockwait) - return; - this.lockwait = 0; - CLUSTER_UNLOCK.name = this.name; - process.send(CLUSTER_UNLOCK); -}; - // ====================================================================== // FILE OPERATIONS // ====================================================================== @@ -707,12 +888,7 @@ Database.prototype.$meta = function(write) { if (write) { self.readonly && self.throwReadonly(); - Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), function() { - if (F.isCluster) { - CLUSTER_META.name = self.name; - process.send(CLUSTER_META); - } - }); + Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), NOOP); return self; } @@ -730,14 +906,11 @@ Database.prototype.$append = function() { var self = this; self.step = 1; - if (self.locked || !self.pending_append.length) { + if (!self.pending_append.length) { self.next(0); return; } - if (F.isCluster && self.$lock()) - return; - self.pending_append.splice(0).limit(20, function(items, next) { var json = []; @@ -746,7 +919,7 @@ Database.prototype.$append = function() { Fs.appendFile(self.filename, json.join(NEWLINE) + NEWLINE, function(err) { for (var i = 0, length = items.length; i < length; i++) { - items[i].builder.$log && items[i].builder.log(); + items[i].builder.$options.log && items[i].builder.log(); var callback = items[i].builder.$callback; callback && callback(err, 1); } @@ -758,7 +931,6 @@ Database.prototype.$append = function() { function next_append(self) { self.next(0); - // $unlock() is in refresh() setImmediate(views_refresh, self); } @@ -766,7 +938,7 @@ Database.prototype.$append_inmemory = function() { var self = this; self.step = 1; - if (self.locked || !self.pending_append.length) { + if (!self.pending_append.length) { self.next(0); return self; } @@ -777,7 +949,7 @@ Database.prototype.$append_inmemory = function() { for (var i = 0, length = items.length; i < length; i++) { self.inmemory['#'].push(JSON.parse(items[i].doc, jsonparser)); - items[i].builder.$log && items[i].builder.log(); + items[i].builder.$options.log && items[i].builder.log(); var callback = items[i].builder.$callback; callback && callback(null, 1); } @@ -792,14 +964,11 @@ Database.prototype.$update = function() { var self = this; self.step = 2; - if (self.locked || !self.pending_update.length) { + if (!self.pending_update.length) { self.next(0); return self; } - if (F.isCluster && self.$lock()) - return; - var reader = Fs.createReadStream(self.filename); var writer = Fs.createWriteStream(self.filenameTemp); @@ -807,7 +976,13 @@ Database.prototype.$update = function() { var length = filter.length; var change = false; - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.repository = {}; + } + + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { if (value[0] !== '{') return; @@ -817,10 +992,10 @@ Database.prototype.$update = function() { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = builder.compare(doc, index, true); + var output = item.compare(doc, item, i); if (output) { - builder.$backup && builder.$backupdoc(output); + builder.$options.backup && builder.$backupdoc(output); if (item.keys) { for (var j = 0, jl = item.keys.length; j < jl; j++) { var val = item.doc[item.keys[j]]; @@ -829,7 +1004,8 @@ Database.prototype.$update = function() { output[item.keys[j]] = val(output[item.keys[j]], output); else output[item.keys[j]] = val; - } + } else + output[item.keys[j]] = undefined; } } else output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; @@ -857,13 +1033,12 @@ Database.prototype.$update = function() { item.builder.$insertcallback && item.builder.$insertcallback(item.insert); self.insert(item.insert).$callback = item.builder.$callback; } else { - item.builder.$log && item.builder.log(); + item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } } self.next(0); - F.cluster && self.$unlock(); }); return; } @@ -886,7 +1061,7 @@ Database.prototype.$update = function() { item.builder.$insertcallback && item.builder.$insertcallback(item.insert); self.insert(item.insert).$callback = item.builder.$callback; } else { - item.builder.$log && item.builder.log(); + item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(err, item.builder, item.count), item.count); } } @@ -894,10 +1069,7 @@ Database.prototype.$update = function() { setImmediate(function() { self.renaming = false; self.next(0); - if (change) - setImmediate(views_refresh, self); - else if (F.isCluster) - self.$unlock(); + change && setImmediate(views_refresh, self); }); }); }; @@ -917,7 +1089,7 @@ Database.prototype.$update_inmemory = function() { var self = this; self.step = 2; - if (self.locked || !self.pending_update.length) { + if (!self.pending_update.length) { self.next(0); return self; } @@ -926,6 +1098,12 @@ Database.prototype.$update_inmemory = function() { var length = filter.length; var change = false; + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.repository = {}; + } + return self.$inmemory('#', function() { var data = self.inmemory['#']; @@ -936,11 +1114,10 @@ Database.prototype.$update_inmemory = function() { var item = filter[i]; var builder = item.builder; - var output = builder.compare(doc, j); - + var output = item.compare(doc, item, j); if (output) { - builder.$backup && builder.$backupdoc(doc); + builder.$options.backup && builder.$backupdoc(doc); if (item.keys) { for (var j = 0, jl = item.keys.length; j < jl; j++) { @@ -973,7 +1150,7 @@ Database.prototype.$update_inmemory = function() { } else { var e = item.keys ? 'modify' : 'update'; item.count && self.$events[e] && self.emit(e, item.doc); - item.builder.$log && item.builder.log(); + item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } } @@ -990,7 +1167,7 @@ Database.prototype.$reader = function() { var self = this; self.step = 4; - if (self.locked || !self.pending_reader.length) { + if (!self.pending_reader.length) { self.next(0); return self; } @@ -1009,7 +1186,7 @@ Database.prototype.$readerview = function() { var self = this; self.step = 6; - if (self.locked || !self.pending_reader_view.length) { + if (!self.pending_reader_view.length) { self.next(0); return self; } @@ -1061,20 +1238,24 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { } else reader = Fs.createReadStream(filename); + var filter = items; var length = filter.length; var first = true; for (var i = 0; i < length; i++) { - if (!filter[i].builder.$first) + var fil = filter[i]; + if (!fil.builder.$options.first) first = false; - filter[i].scalarcount = 0; + fil.scalarcount = 0; + fil.compare = fil.builder.compile(); + fil.repository = {}; } if (first && length > 1) first = false; - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { if (value[0] !== '{') return; @@ -1085,13 +1266,13 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = builder.compare(json, index); + var output = item.compare(json, item, i); if (!output) continue; item.count++; - if (!builder.$inlinesort && ((builder.$skip && builder.$skip >= item.count) || (builder.$take && builder.$take <= item.counter))) + if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) continue; item.counter++; @@ -1099,16 +1280,16 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { if (item.type) continue; - switch (builder.$scalar) { + switch (builder.$options.scalar) { case 'count': item.scalar = item.scalar ? item.scalar + 1 : 1; break; case 'sum': - val = output[builder.$scalarfield] || 0; + val = output[builder.$options.scalarfield] || 0; item.scalar = item.scalar ? item.scalar + val : val; break; case 'min': - val = output[builder.$scalarfield] || 0; + val = output[builder.$options.scalarfield] || 0; if (val != null) { if (item.scalar) { if (item.scalar > val) @@ -1118,7 +1299,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { } break; case 'max': - val = output[builder.$scalarfield]; + val = output[builder.$options.scalarfield]; if (val != null) { if (item.scalar) { if (item.scalar < val) @@ -1128,7 +1309,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { } break; case 'avg': - val = output[builder.$scalarfield]; + val = output[builder.$options.scalarfield]; if (val != null) { item.scalar = item.scalar ? item.scalar + val : val; item.scalarcount++; @@ -1136,7 +1317,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { break; case 'group': !item.scalar && (item.scalar = {}); - val = output[builder.$scalarfield]; + val = output[builder.$options.scalarfield]; if (val != null) { if (item.scalar[val]) item.scalar[val]++; @@ -1167,11 +1348,11 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { var builder = item.builder; var output; - if (builder.$scalar || !builder.$sort) { + if (builder.$options.scalar || !builder.$options.sort) { - if (builder.$scalar) - output = builder.$scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$first) + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; @@ -1181,23 +1362,23 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { } if (item.count) { - if (builder.$sort.name) { - if (!builder.$inlinesort || builder.$take !== item.response.length) - item.response.quicksort(builder.$sort.name, builder.$sort.asc); - } else if (builder.$sort === EMPTYOBJECT) + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) item.response.random(); else - item.response.sort(builder.$sort); - - if (builder.$skip && builder.$take) - item.response = item.response.splice(builder.$skip, builder.$take); - else if (builder.$skip) - item.response = item.response.splice(builder.$skip); - else if (!builder.$inlinesort && builder.$take) - item.response = item.response.splice(0, builder.$take); + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); } - if (builder.$first) + if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; @@ -1226,7 +1407,7 @@ function nosqlinlinesorter(item, builder, doc) { var length = item.response.length; if (length < builder.$limit) { item.response.push(doc); - length + 1 >= builder.$limit && item.response.quicksort(builder.$sort.name, builder.$sort.asc); + length + 1 >= builder.$limit && item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); } else nosqlresort(item.response, builder, doc); } @@ -1247,23 +1428,23 @@ function nosqlsortvalue(a, b, sorter) { } function nosqlresort(arr, builder, doc) { - var b = doc[builder.$sort.name]; + var b = doc[builder.$options.sort.name]; var beg = 0; var length = arr.length; var tmp = length - 1; - var sort = nosqlsortvalue(arr[tmp][builder.$sort.name], b, builder.$sort); + var sort = nosqlsortvalue(arr[tmp][builder.$options.sort.name], b, builder.$options.sort); if (!sort) return; tmp = arr.length / 2 >> 0; - sort = nosqlsortvalue(arr[tmp][builder.$sort.name], b, builder.$sort); + sort = nosqlsortvalue(arr[tmp][builder.$options.sort.name], b, builder.$options.sort); if (!sort) beg = tmp + 1; for (var i = beg; i < length; i++) { var item = arr[i]; - var sort = nosqlsortvalue(item[builder.$sort.name], b, builder.$sort); + var sort = nosqlsortvalue(item[builder.$options.sort.name], b, builder.$options.sort); if (!sort) continue; for (var j = length - 1; j > i; j--) @@ -1279,6 +1460,12 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { var filter = items; var length = filter.length; + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.repository = {}; + } + return self.$inmemory(name, function() { var data = self.inmemory[name]; @@ -1289,13 +1476,13 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = builder.compare(U.clone(json), j); + var output = item.compare(U.clone(json), item, j); if (!output) continue; item.count++; - if (!builder.$sort && ((builder.$skip && builder.$skip >= item.count) || (builder.$take && builder.$take <= item.counter))) + if (!builder.$options.sort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) continue; item.counter++; @@ -1303,16 +1490,16 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { if (item.type) continue; - switch (builder.$scalar) { + switch (builder.$options.scalar) { case 'count': item.scalar = item.scalar ? item.scalar + 1 : 1; break; case 'sum': - val = json[builder.$scalarfield] || 0; + val = json[builder.$options.scalarfield] || 0; item.scalar = item.scalar ? item.scalar + val : val; break; case 'min': - val = json[builder.$scalarfield] || 0; + val = json[builder.$options.scalarfield] || 0; if (val != null) { if (item.scalar) { if (item.scalar > val) @@ -1322,7 +1509,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { } break; case 'max': - val = json[builder.$scalarfield]; + val = json[builder.$options.scalarfield]; if (val != null) { if (item.scalar) { if (item.scalar < val) @@ -1332,7 +1519,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { } break; case 'avg': - val = json[builder.$scalarfield]; + val = json[builder.$options.scalarfield]; if (val != null) { item.scalar = item.scalar ? item.scalar + val : 0; if (item.scalarcount) @@ -1343,7 +1530,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { break; case 'group': !item.scalar && (item.scalar = {}); - val = output[builder.$scalarfield]; + val = output[builder.$options.scalarfield]; if (val != null) { if (item.scalar[val]) item.scalar[val]++; @@ -1367,11 +1554,11 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { var builder = item.builder; var output; - if (builder.$scalar || !builder.$sort) { + if (builder.$options.scalar || !builder.$options.sort) { - if (builder.$scalar) - output = builder.$scalar === 'avg' ? item.scalar / item.counter : item.scalar; - else if (builder.$first) + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.counter : item.scalar; + else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; @@ -1381,22 +1568,22 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { } if (item.count) { - if (builder.$sort.name) - item.response.quicksort(builder.$sort.name, builder.$sort.asc); - else if (builder.$sort === EMPTYOBJECT) + if (builder.$options.sort.name) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + else if (builder.$options.sort === EMPTYOBJECT) item.response.random(); else - item.response.sort(builder.$sort); - - if (builder.$skip && builder.$take) - item.response = item.response.splice(builder.$skip, builder.$take); - else if (builder.$skip) - item.response = item.response.splice(builder.$skip); - else if (builder.$take) - item.response = item.response.splice(0, builder.$take); + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); } - if (builder.$first) + if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; @@ -1414,8 +1601,7 @@ Database.prototype.$views = function() { var self = this; self.step = 5; - if (self.locked || !self.pending_views) { - F.isCluster && self.$unlock(); + if (!self.pending_views) { self.next(0); return; } @@ -1426,18 +1612,16 @@ Database.prototype.$views = function() { var length = views.length; if (!length) { - F.isCluster && self.$unlock(); self.next(0); return self; } - if (F.isCluster && self.$lock()) - return; - var response = []; - for (var i = 0; i < length; i++) - response.push({ response: [], name: views[i], builder: self.views[views[i]], count: 0, counter: 0 }); + for (var i = 0; i < length; i++) { + var builder = self.views[views[i]]; + response.push({ response: [], name: views[i], compare: builder.compile(), builder: builder, count: 0, counter: 0, repository: {} }); + } var reader = Fs.createReadStream(self.filename); @@ -1450,7 +1634,7 @@ Database.prototype.$views = function() { for (var j = 0; j < length; j++) { var item = self.views[views[j]]; - var output = item.compare(json, index); + var output = item.compare(json, item, index); if (!output) continue; @@ -1470,19 +1654,19 @@ Database.prototype.$views = function() { var builder = item.builder; - if (builder.$sort) { - if (builder.$sort.name) - item.response.quicksort(builder.$sort.name, builder.$sort.asc); - else if (builder.$sort === EMPTYOBJECT) + if (builder.$options.sort) { + if (builder.$options.sort.name) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + else if (builder.$options.sort === EMPTYOBJECT) item.response.random(); else - item.response.sort(builder.$sort); - if (builder.$skip && builder.$take) - item.response = item.response.splice(builder.$skip, builder.$take); - else if (builder.$skip) - item.response = item.response.splice(builder.$skip); - else if (builder.$take) - item.response = item.response.splice(0, builder.$take); + item.response.sort(builder.$options.sort); + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); } var filename = builder.$filename; @@ -1498,10 +1682,7 @@ Database.prototype.$views = function() { next(); }); }); - }, function() { - F.isCluster && self.$unlock(); - self.next(0); - }, 5); + }, () => self.next(0), 5); }); }; @@ -1510,7 +1691,7 @@ Database.prototype.$views_inmemory = function() { var self = this; self.step = 5; - if (self.locked || !self.pending_views) { + if (!self.pending_views) { self.next(0); return self; } @@ -1553,19 +1734,19 @@ Database.prototype.$views_inmemory = function() { var item = response[j]; var builder = item.builder; - if (builder.$sort) { - if (builder.$sort.name) - item.response.quicksort(builder.$sort.name, builder.$sort.asc); - else if (builder.$sort === EMPTYOBJECT) + if (builder.$options.sort) { + if (builder.$options.sort.name) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + else if (builder.$options.sort === EMPTYOBJECT) item.response.random(); else - item.response.sort(builder.$sort); - if (builder.$skip && builder.$take) - item.response = item.response.splice(builder.$skip, builder.$take); - else if (builder.$skip) - item.response = item.response.splice(builder.$skip); - else if (builder.$take) - item.response = item.response.splice(0, builder.$take); + item.response.sort(builder.$options.sort); + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); } self.inmemory[item.name] = item.response; @@ -1593,8 +1774,11 @@ Database.prototype.$remove = function() { var length = filter.length; var change = false; - if (F.isCluster && self.$lock()) - return; + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.repository = {}; + fil.compare = fil.builder.compile(); + } reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { @@ -1607,9 +1791,9 @@ Database.prototype.$remove = function() { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = builder.compare(json, index); + var output = item.compare(json, item, index); if (output) { - builder.$backup && builder.$backupdoc(output); + builder.$options.backup && builder.$backupdoc(output); removed = true; json = output; break; @@ -1635,11 +1819,10 @@ Database.prototype.$remove = function() { Fs.unlink(self.filenameTemp, function() { for (var i = 0; i < length; i++) { var item = filter[i]; - item.builder.$log && item.builder.log(); + item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } self.next(0); - F.cluster && self.$unlock(); }); return; } @@ -1657,17 +1840,14 @@ Database.prototype.$remove = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - item.builder.$log && item.builder.log(); + item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } setImmediate(function() { self.renaming = false; self.next(0); - if (change) - setImmediate(views_refresh, self); - else if (F.cluster) - self.$unlock(); + change && setImmediate(views_refresh, self); }); }); }; @@ -1681,7 +1861,7 @@ Database.prototype.$remove_inmemory = function() { var self = this; self.step = 3; - if (self.locked || !self.pending_remove.length) { + if (!self.pending_remove.length) { self.next(0); return self; } @@ -1690,6 +1870,12 @@ Database.prototype.$remove_inmemory = function() { var length = filter.length; var change = false; + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.repository = {}; + fil.compare = fil.builder.compile(); + } + return self.$inmemory('#', function() { var data = self.inmemory['#']; @@ -1701,8 +1887,7 @@ Database.prototype.$remove_inmemory = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - var builder = item.builder; - if (builder.compare(json, j)) { + if (item.compare(json, item, j)) { removed = true; break; } @@ -1727,7 +1912,7 @@ Database.prototype.$remove_inmemory = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - item.builder.$log && item.builder.log(); + item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); } @@ -1740,14 +1925,11 @@ Database.prototype.$drop = function() { var self = this; self.step = 7; - if (self.locked || !self.pending_drops) { + if (!self.pending_drops) { self.next(0); return; } - if (F.isCluster && self.$lock()) - return; - self.pending_drops = false; var remove = [self.filename, self.filenameTemp]; Object.keys(self.views).forEach(key => remove.push(self.views[key].$filename)); @@ -1759,7 +1941,6 @@ Database.prototype.$drop = function() { } catch (e) {} remove.wait((filename, next) => Fs.unlink(filename, next), function() { - F.isCluster && self.$unlock(); self.next(0); self.free(true); }, 5); @@ -1772,17 +1953,17 @@ Database.prototype.$drop = function() { function DatabaseBuilder2(db) { this.$callback = NOOP; this.db = db; - // this.$log; + this.$options = {}; } DatabaseBuilder2.prototype.log = function(msg, user) { var self = this; if (msg) { F.datetime = new Date(); - self.$log = (self.$log ? self.$log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; - } else if (self.$log) { - self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$log, NOOP); - self.$log = ''; + self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + } else if (self.$options.log) { + self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, NOOP); + self.$options.log = ''; } return self; }; @@ -1814,9 +1995,18 @@ function DatabaseBuilder(db) { this.$callback = NOOP; // this.$scalar; // this.$scalarfield; - // this.$log; + + this.$code = []; + this.$params = {}; + this.$options = {}; + this.$counter = 0; } +DatabaseBuilder.prototype.id = function(id) { + this.$options.id = id; + return this; +}; + DatabaseBuilder.prototype.insert = function(fn) { this.$insertcallback = fn; return this; @@ -1826,10 +2016,10 @@ DatabaseBuilder.prototype.log = function(msg, user) { var self = this; if (msg) { F.datetime = new Date(); - self.$log = (self.$log ? self.$log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; - } else if (self.$log) { - self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$log, NOOP); - self.$log = ''; + self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + } else if (self.$options.log) { + self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, NOOP); + self.$options.log = ''; } return self; }; @@ -1838,7 +2028,10 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count) { var self = this; if (err || !self.$join) { - self.$log && self.log(); + self.$options.log && self.log(); + + // BACKWARD COMPATIBILITY + self.$prepare && self.$prepare(response); return self.$callback(err, response, count); } @@ -1865,7 +2058,11 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count) { } } - self.$log && self.log(); + self.$options.log && self.log(); + + // BACKWARD COMPATIBILITY + self.$prepare && self.$prepare(response); + self.$callback(err, response, count); return self; }; @@ -1990,7 +2187,7 @@ DatabaseBuilder.prototype.join = function(field, name, view) { }; DatabaseBuilder.prototype.first = function() { - this.$first = true; + this.$options.first = true; return this.take(1); }; @@ -1999,95 +2196,57 @@ DatabaseBuilder.prototype.make = function(fn) { return this; }; -DatabaseBuilder.prototype.compare = function(doc, index) { - - var can = true; - var wasor = false; - var prevscope = 0; - - for (var i = 0, length = this.$filter.length; i < length; i++) { - - var filter = this.$filter[i]; - if (wasor && filter.scope) - continue; - - if (prevscope && !filter.scope && !wasor) - return; - - prevscope = filter.scope; - - var res = filter.filter(doc, i, filter); - - if (res === true) { - can = true; - if (filter.scope) - wasor = true; - continue; - } else if (filter.scope) { - can = false; - wasor = false; - continue; - } - - return; - } - - if (!can) - return; - - if (this.$prepare) - return this.$prepare(doc, index); - - if (!this.$fields) - return doc; - - var obj = {}; - - for (var i = 0, length = this.$fields.length; i < length; i++) { - var prop = this.$fields[i]; - obj[prop] = doc[prop]; - } - - return obj; -}; - -DatabaseBuilder.prototype.filter = function(fn) { - this.$filter.push({ scope: this.$scope, filter: fn }); +DatabaseBuilder.prototype.filter = function() { + OBSOLETE('DatabaseBuilder.filter()', 'You need to rewrite this code with help of DatabaseBuidler.code() alternative.'); return this; }; DatabaseBuilder.prototype.scalar = function(type, name) { - this.$scalar = type; - this.$scalarfield = name; + var self = this; + var opt = self.$options; + opt.scalar = type; + opt.scalarfield = name; return this; }; DatabaseBuilder.prototype.contains = function(name) { - this.$filter.push({ scope: this.$scope, filter: compare_notempty, name: name }); - return this; + var self = this; + var code = '$is=doc.{0} instanceof Array?!!doc.{0}.length:!!doc.{0};'.format(name); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.empty = function(name) { - this.$filter.push({ scope: this.$scope, filter: compare_empty, name: name }); - return this; + var self = this; + var code = '$is=doc.{0} instanceof Array?!doc.{0}.length:!doc.{0};'.format(name); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.backup = function(user) { if (this.db.filenameBackup) - this.$backup = typeof(user) === 'string' ? user : 'unknown'; + this.$options.backup = typeof(user) === 'string' ? user : 'unknown'; else - this.$backup = null; + this.$options.backup = null; return this; }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, NOOP); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, NOOP); return this; }; DatabaseBuilder.prototype.where = function(name, operator, value) { - var fn; + var self = this; + var key = 'w' + (self.$counter++); + var code; if (value === undefined) { value = operator; @@ -2095,155 +2254,102 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { } var date = framework_utils.isDate(value); + self.$params[key] = date ? value.getTime() : value; switch (operator) { case '=': - fn = date ? compare_eq_date : compare_eq; - break; - case '<': - fn = date ? compare_gt_date : compare_gt; - break; - case '<=': - fn = date ? compare_eqgt_date : compare_eqgt; - break; - case '>': - fn = date ? compare_lt_date : compare_lt; - break; - case '>=': - fn = date ? compare_eqlt_date : compare_eqlt; + operator = '=='; break; case '<>': - case '!=': - fn = date ? compare_not_date : compare_not; operator = '!='; break; } - this.$filter.push({ scope: this.$scope, filter: fn, name: name, value: value, operator: operator, ticks: date ? value.getTime() : 0 }); - return this; + code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); + + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$code.push(code.format(name, key, operator)); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.month = function(name, operator, value) { - - var fn; + var self = this; + var key = 'dm' + (self.$counter++); if (value === undefined) { value = operator; operator = '='; } - switch (operator) { - case '=': - fn = compare_eq_dtmonth; - break; - case '<': - fn = compare_gt_dtmonth; - break; - case '<=': - fn = compare_eqgt_dtmonth; - break; - case '>': - fn = compare_lt_dtmonth; - break; - case '>=': - fn = compare_eqlt_dtmonth; - break; - case '<>': - case '!=': - fn = compare_not_dtmonth; - break; - } + self.$params[key] = value; - this.$filter.push({ scope: this.$scope, filter: fn, name: name, value: value }); - return this; + var code = compare_datetype('month', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.day = function(name, operator, value) { - - var fn; + var self = this; + var key = 'dd' + (self.$counter++); if (value === undefined) { value = operator; operator = '='; } - switch (operator) { - case '=': - fn = compare_eq_dtday; - break; - case '<': - fn = compare_gt_dtday; - break; - case '<=': - fn = compare_eqgt_dtday; - break; - case '>': - fn = compare_lt_dtday; - break; - case '>=': - fn = compare_eqlt_dtday; - break; - case '<>': - case '!=': - fn = compare_not_dtday; - break; - } + self.$params[key] = value; - this.$filter.push({ scope: this.$scope, filter: fn, name: name, value: value }); - return this; + var code = compare_datetype('day', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.year = function(name, operator, value) { - - var fn; + var self = this; + var key = 'dy' + (self.$counter++); if (value === undefined) { value = operator; operator = '='; } - switch (operator) { - case '=': - fn = compare_eq_dtyear; - break; - case '<': - fn = compare_gt_dtyear; - break; - case '<=': - fn = compare_eqgt_dtyear; - break; - case '>': - fn = compare_lt_dtyear; - break; - case '>=': - fn = compare_eqlt_dtyear; - break; - case '<>': - case '!=': - fn = compare_not_dtyear; - break; - } + self.$params[key] = value; - this.$filter.push({ scope: this.$scope, filter: fn, name: name, value: value }); - return this; + var code = compare_datetype('year', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(name, value, where) { - var fn; + var self = this; + var code; + var key = 'l' + (self.$counter++); if (!where) where = '*'; switch (where) { case 'beg': - fn = compare_likebeg; + code = '$is=doc.{0}?doc.{0}.startsWith(arg.{1}):false;'; break; case 'end': - fn = compare_likeend; + code = '$is=doc.{0}?doc.{0}.endsWith(arg.{1}):false;'; break; case '*': - fn = compare_like; + code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){if(doc.{0} instanceof Array){for(var $i=0;$i self.stats(top, year, month, day, callback, type), 200); + setTimeout(() => self.stats(top, year, month, day, type, callback), 200); return self; } @@ -2983,7 +3112,7 @@ Counter.prototype.stats = Counter.prototype.stats_sum = function(top, year, mont self.type = 0; callback && callback(err, []); } else - self.stats(top, year, month, day, callback, type, response); + self.stats(top, year, month, day, type, callback, response); }); return self; } @@ -3370,7 +3499,7 @@ Counter.prototype.save = function() { var self = this; self.db.readonly && self.db.throwReadonly(); - if (self.type || self.locked) { + if (self.type) { setTimeout(() => self.save(), 200); return self; } @@ -3385,11 +3514,6 @@ Counter.prototype.save = function() { self.cache = null; self.type = 1; - if (F.isCluster) { - CLUSTER_LOCK_COUNTER.name = self.name; - process.send(CLUSTER_LOCK_COUNTER); - } - var flush = function() { var keys = Object.keys(cache); for (var i = 0, length = keys.length; i < length; i++) { @@ -3489,10 +3613,6 @@ Counter.prototype.save = function() { clearTimeout(self.timeout); self.timeout = 0; self.type = 0; - if (F.isCluster) { - CLUSTER_UNLOCK_COUNTER.name = self.name; - process.send(CLUSTER_UNLOCK_COUNTER); - } counter && self.$events.stats && setImmediate(() => self.emit('stats', counter)); }); }); @@ -3503,25 +3623,14 @@ Counter.prototype.save = function() { Counter.prototype.clear = function(callback) { var self = this; - if (self.type || self.locked) { + if (self.type) { setTimeout(() => self.clear(callback), 200); return self; } - if (F.isCluster) { - CLUSTER_LOCK_COUNTER.name = self.name; - process.send(CLUSTER_LOCK_COUNTER); - } - self.type = 3; Fs.unlink(self.db.filename + EXTENSION_COUNTER, function() { - - if (F.isCluster) { - CLUSTER_UNLOCK_COUNTER.name = self.name; - process.send(CLUSTER_UNLOCK_COUNTER); - } - self.type = 0; self.emit('clear'); callback && callback(); @@ -3889,6 +3998,7 @@ Backuper.prototype.write = function(value) { Backuper.prototype.flush = function() { var self = this; + if (self.is) return self; @@ -3911,221 +4021,38 @@ Backuper.prototype.flush = function() { // Helper functions // ====================================================== -function compare_eq(doc, index, item) { - return item.value === doc[item.name]; -} - -function compare_lt(doc, index, item) { - return item.value < doc[item.name]; -} - -function compare_gt(doc, index, item) { - return item.value > doc[item.name]; -} - -function compare_eqlt(doc, index, item) { - return item.value <= doc[item.name]; -} - -function compare_eqgt(doc, index, item) { - return item.value >= doc[item.name]; -} - -function compare_not(doc, index, item) { - return item.value !== doc[item.name]; -} - -function compare_eq_date(doc, index, item) { - var val = doc[item.name]; - return val ? item.ticks === (val instanceof Date ? val : new Date(val)).getTime() : false; -} - -function compare_lt_date(doc, index, item) { - var val = doc[item.name]; - return val ? item.value < (val instanceof Date ? val : new Date(val)) : false; -} - -function compare_gt_date(doc, index, item) { - var val = doc[item.name]; - return val ? item.value > (val instanceof Date ? val : new Date(val)) : false; -} - -function compare_eqlt_date(doc, index, item) { - var val = doc[item.name]; - return val ? item.value <= (val instanceof Date ? val : new Date(val)) : false; -} - -function compare_eqgt_date(doc, index, item) { - var val = doc[item.name]; - return val ? item.value >= (val instanceof Date ? val : new Date(val)) : false; -} - -function compare_not_date(doc, index, item) { - var val = doc[item.name]; - return val ? item.value !== (val instanceof Date ? val : new Date(val)) : false; -} - -function compare_likebeg(doc, index, item) { - var val = doc[item.name]; - return val ? val.startsWith(item.value) : false; -} - -function compare_likeend(doc, index, item) { - var val = doc[item.name]; - return val ? val.endsWith(item.value) : false; -} - -function compare_like(doc, index, item) { - var val = doc[item.name]; - - if (!val || !val.toLowerCase) - return false; - - if (item.value instanceof Array) { - for (var i = 0, length = item.value.length; i < length; i++) { - if (val.toLowerCase().indexOf(item.value[i]) !== -1) - return true; - } - return false; - } - - return val.toLowerCase().indexOf(item.value) !== -1; -} - -function compare_between(doc, index, item) { - var val = doc[item.name]; - return val >= item.a && val <= item.b; -} - -function compare_in(doc, index, item) { - var val = doc[item.name]; - if (val instanceof Array) { - for (var i = 0, length = val.length; i < length; i++) { - if (item.value.indexOf(val[i]) !== -1) - return true; - } - return false; - } - return item.value.indexOf(val) !== -1; -} - -function compare_notin(doc, index, item) { - var val = doc[item.name]; - if (val instanceof Array) { - for (var i = 0, length = val.length; i < length; i++) { - if (item.value.indexOf(val[i]) !== -1) - return false; - } - return true; - } - return item.value.indexOf(val) === -1; -} - -function compare_notempty(doc, index, item) { - var val = doc[item.name]; - return val instanceof Array ? (val.length ? true : false) : (val ? true : false); -} +function compare_datetype(type, key, arg, operator) { -function compare_empty(doc, index, item) { - var val = doc[item.name]; - return val instanceof Array ? (val.length ? false : true) : (val ? false : true); -} + // 0 doc + // 1 arg + // 2 operator + // 3 type -function compare_datetype(type, eqtype, val, doc) { + key = 'doc.' + key; + arg = 'arg.' + arg; - if (!doc) - return false; - else if (!doc.getTime) { - doc = new Date(doc); - if (isNaN(doc)) - return false; + switch (operator) { + case '=': + operator = '=='; + break; + case '<>': + operator = '!='; + break; } switch (type) { - case 'month': - doc = doc.getMonth() + 1; - break; case 'day': - doc = doc.getDate(); + type = 'getDate()'; + break; + case 'month': + type = 'getMonth()+1'; break; case 'year': - doc = doc.getFullYear(); + type = 'getFullYear()'; break; } - return eqtype === '=' ? val === doc : eqtype === '>' ? val > doc : eqtype === '<' ? val < doc : eqtype === '>=' ? val >= doc : eqtype === '<=' ? val <= doc : val !== doc; -} - -function compare_eq_dtmonth(doc, index, item) { - return compare_datetype('month', '=', item.value, doc[item.name]); -} - -function compare_lt_dtmonth(doc, index, item) { - return compare_datetype('month', '<', item.value, doc[item.name]); -} - -function compare_gt_dtmonth(doc, index, item) { - return compare_datetype('month', '>', item.value, doc[item.name]); -} - -function compare_eqlt_dtmonth(doc, index, item) { - return compare_datetype('month', '<=', item.value, doc[item.name]); -} - -function compare_eqgt_dtmonth(doc, index, item) { - return compare_datetype('month', '>=', item.value, doc[item.name]); -} - -function compare_not_dtmonth(doc, index, item) { - return compare_datetype('month', '!=', item.value, doc[item.name]); -} - -function compare_eq_dtyear(doc, index, item) { - return compare_datetype('year', '=', item.value, doc[item.name]); -} - -function compare_lt_dtyear(doc, index, item) { - return compare_datetype('year', '<', item.value, doc[item.name]); -} - -function compare_gt_dtyear(doc, index, item) { - return compare_datetype('year', '>', item.value, doc[item.name]); -} - -function compare_eqlt_dtyear(doc, index, item) { - return compare_datetype('year', '<=', item.value, doc[item.name]); -} - -function compare_eqgt_dtyear(doc, index, item) { - return compare_datetype('year', '>=', item.value, doc[item.name]); -} - -function compare_not_dtyear(doc, index, item) { - return compare_datetype('year', '!=', item.value, doc[item.name]); -} - -function compare_eq_dtday(doc, index, item) { - return compare_datetype('day', '=', item.value, doc[item.name]); -} - -function compare_lt_dtday(doc, index, item) { - return compare_datetype('day', '<', item.value, doc[item.name]); -} - -function compare_gt_dtday(doc, index, item) { - return compare_datetype('day', '>', item.value, doc[item.name]); -} - -function compare_eqlt_dtday(doc, index, item) { - return compare_datetype('day', '<=', item.value, doc[item.name]); -} - -function compare_eqgt_dtday(doc, index, item) { - return compare_datetype('day', '>=', item.value, doc[item.name]); -} - -function compare_not_dtday(doc, index, item) { - return compare_datetype('day', '!=', item.value, doc[item.name]); + return '$is=false;$tmp={0};if($tmp){if(!$tmp.getTime){$tmp=new Date($tmp);if(isNaN($tmp))$tmp=0;}if($tmp)$is=($tmp.{3}){2}{1};}'.format(key, arg, operator, type); } function errorhandling(err, builder, response) { @@ -4139,4 +4066,4 @@ function errorhandling(err, builder, response) { function jsonparser(key, value) { return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; -} +} \ No newline at end of file From 581ba372b6f51316873515b0861eaadea694092f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 01:06:10 +0100 Subject: [PATCH 0113/1669] Added missing worker :-) --- nosqlworker.js | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 nosqlworker.js diff --git a/nosqlworker.js b/nosqlworker.js new file mode 100644 index 000000000..2a39a2b86 --- /dev/null +++ b/nosqlworker.js @@ -0,0 +1,149 @@ +require(module.filename.replace(/nosqlworker\.js$/, 'index.js')); + +const RESFIND = { TYPE: 'find' }; +const RESINSERT = { TYPE: 'insert' }; +const RESCOUNT = { TYPE: 'count' }; +const RESUPDATE = { TYPE: 'update' }; +const RESBACKUP = { TYPE: 'backup' }; +const RESRESTORE = { TYPE: 'restore' }; +const RESREMOVE = { TYPE: 'remove' }; +const RESCOUNTERREAD = { TYPE: 'counter.read' }; +const RESCOUNTERSTATS = { TYPE: 'counter.stats' }; +const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; + +process.on('message', function(msg) { + + if (msg.TYPE === 'init') { + F.directory = msg.directory; + return; + } + + var db = NOSQL(msg.name); + switch (msg.TYPE) { + case 'find': + db.find(msg.arg ? msg.arg[0] : null).parse(msg.data).callback(function(err, response, count, repository) { + RESFIND.err = err; + RESFIND.response = response; + RESFIND.count = count; + RESFIND.repository = repository; + RESFIND.id = msg.id; + process.send(RESFIND); + }); + break; + case 'one': + db.one(msg.arg ? msg.arg[0] : null).parse(msg.data).callback(function(err, response, count, repository) { + RESFIND.err = err; + RESFIND.response = response; + RESFIND.count = count; + RESFIND.repository = repository; + RESFIND.id = msg.id; + process.send(RESFIND); + }); + break; + case 'insert': + db.insert(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response) { + RESINSERT.err = err; + RESINSERT.response = response; + RESINSERT.id = msg.id; + process.send(RESINSERT); + }); + break; + case 'update': + db.update(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response) { + RESUPDATE.err = err; + RESUPDATE.response = response; + RESUPDATE.id = msg.id; + process.send(RESUPDATE); + }); + break; + case 'modify': + db.modify(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response) { + RESUPDATE.err = err; + RESUPDATE.response = response; + RESUPDATE.id = msg.id; + process.send(RESUPDATE); + }); + break; + case 'count': + db.count(msg.arg ? msg.arg[0] : null).parse(msg.data).callback(function(err, response, count, repository) { + RESCOUNT.err = err; + RESCOUNT.response = response; + RESCOUNT.count = count; + RESCOUNT.repository = repository; + RESCOUNT.id = msg.id; + process.send(RESCOUNT); + }); + break; + case 'remove': + db.remove(msg.arg ? msg.arg[0] : null).parse(msg.data).callback(function(err, response, count, repository) { + RESREMOVE.err = err; + RESREMOVE.response = response; + RESREMOVE.count = count; + RESREMOVE.repository = repository; + RESREMOVE.id = msg.id; + process.send(RESREMOVE); + }); + break; + case 'view': + db.view(msg.arg[0]).parse(msg.data); + RESVIEW.id = msg.id; + break; + case 'backup': + db.backup(msg.arg[0], function(err, response) { + RESBACKUP.id = msg.id; + RESBACKUP.err = err; + RESBACKUP.response = response; + process.send(RESBACKUP); + }); + break; + case 'restore': + db.restore(msg.arg[0], function(err, response) { + RESRESTORE.id = msg.id; + RESRESTORE.err = err; + RESRESTORE.response = response; + process.send(RESRESTORE); + }); + break; + case 'refresh': + db.refresh(); + break; + case 'drop': + db.drop(); + break; + case 'counter.min': + db.counter.min(msg.arg[0], msg.arg[1]); + break; + case 'counter.max': + db.counter.max(msg.arg[0], msg.arg[1]); + break; + case 'counter.hit': + db.counter.hit(msg.arg[0], msg.arg[1]); + break; + case 'counter.remove': + db.counter.remove(msg.arg ? msg.arg[0] : null); + break; + case 'counter.read': + db.counter.read(msg.arg[0], function(err, response) { + RESCOUNTERREAD.id = msg.id; + RESCOUNTERREAD.err = err; + RESCOUNTERREAD.response = response; + process.send(RESCOUNTERREAD); + }); + break; + case 'counter.stats': + db.counter.stats(msg.arg[0], msg.arg[1], msg.arg[2], msg.arg[3], msg.arg[4], function(err, response) { + RESCOUNTERSTATS.id = msg.id; + RESCOUNTERSTATS.err = err; + RESCOUNTERSTATS.response = response; + process.send(RESCOUNTERSTATS); + }); + break; + case 'counter.clear': + db.counter.clear(function(err) { + RESCOUNTERCLEAR.id = msg.id; + RESCOUNTERCLEAR.err = err; + process.send(RESCOUNTERCLEAR); + }); + break; + } +}); \ No newline at end of file From ce19669cf1406f6b82e6645f739174368275b017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 11:56:09 +0200 Subject: [PATCH 0114/1669] Fixed logging. --- debug.js | 1 + 1 file changed, 1 insertion(+) diff --git a/debug.js b/debug.js index a81551321..63d8be4ee 100644 --- a/debug.js +++ b/debug.js @@ -99,6 +99,7 @@ function runwatching() { function app() { + global.OBSOLETE = NOOP; F.$configure_configs(); F.directory = directory; From 8d6b105d777914896d1a6f1b712bcc3f2c88dadc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 11:56:43 +0200 Subject: [PATCH 0115/1669] New NoSQL embeddded fixes. --- nosql.js | 119 ++++++++++++++++++++++++++++++++++--------------- nosqlworker.js | 10 ++--- 2 files changed, 88 insertions(+), 41 deletions(-) diff --git a/nosql.js b/nosql.js index 6a67567db..052e82f31 100755 --- a/nosql.js +++ b/nosql.js @@ -979,10 +979,10 @@ Database.prototype.$update = function() { for (var i = 0; i < length; i++) { var fil = filter[i]; fil.compare = fil.builder.compile(); - fil.repository = {}; + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -992,7 +992,9 @@ Database.prototype.$update = function() { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = item.compare(doc, item, i); + + item.filter.index = index; + var output = item.compare(doc, item.filter, i); if (output) { builder.$options.backup && builder.$backupdoc(output); @@ -1101,7 +1103,7 @@ Database.prototype.$update_inmemory = function() { for (var i = 0; i < length; i++) { var fil = filter[i]; fil.compare = fil.builder.compile(); - fil.repository = {}; + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } return self.$inmemory('#', function() { @@ -1114,7 +1116,8 @@ Database.prototype.$update_inmemory = function() { var item = filter[i]; var builder = item.builder; - var output = item.compare(doc, item, j); + item.filter.index = j; + var output = item.compare(doc, item.filter, j); if (output) { builder.$options.backup && builder.$backupdoc(doc); @@ -1249,13 +1252,13 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { first = false; fil.scalarcount = 0; fil.compare = fil.builder.compile(); - fil.repository = {}; + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } if (first && length > 1) first = false; - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -1266,7 +1269,8 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = item.compare(json, item, i); + item.filter.index = index; + var output = item.compare(json, item.filter, index); if (!output) continue; @@ -1463,7 +1467,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { for (var i = 0; i < length; i++) { var fil = filter[i]; fil.compare = fil.builder.compile(); - fil.repository = {}; + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } return self.$inmemory(name, function() { @@ -1476,7 +1480,8 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = item.compare(U.clone(json), item, j); + item.filter.index = j; + var output = item.compare(U.clone(json), item.filter, j); if (!output) continue; @@ -1620,7 +1625,7 @@ Database.prototype.$views = function() { for (var i = 0; i < length; i++) { var builder = self.views[views[i]]; - response.push({ response: [], name: views[i], compare: builder.compile(), builder: builder, count: 0, counter: 0, repository: {} }); + response.push({ response: [], name: views[i], compare: builder.compile(), filter: { repository: builder.$repository, options: builder.$options, arg: builder.$params, fn: builder.$functions }, builder: builder, count: 0, counter: 0, repository: {} }); } var reader = Fs.createReadStream(self.filename); @@ -1634,18 +1639,16 @@ Database.prototype.$views = function() { for (var j = 0; j < length; j++) { var item = self.views[views[j]]; - var output = item.compare(json, item, index); - + var res = response[j]; + res.filter.index = index; + var output = res.compare(json, res.filter, index); if (!output) continue; - - response[j].count++; - - if (!item.$sort && ((item.$skip && item.$skip >= response[j].count) || (item.$take && item.$take < response[j].counter))) + res.count++; + if (!res.filter.options.sort && ((res.filter.options.skip && res.filter.options.skip >= res.count) || (res.filter.options.take && res.filter.options.take <= res.counter))) continue; - - response[j].counter++; - !item.type && response[j].response.push(output); + res.counter++; + !item.type && res.response.push(output); } })); @@ -1708,8 +1711,10 @@ Database.prototype.$views_inmemory = function() { var response = []; - for (var i = 0; i < length; i++) - response.push({ response: [], name: views[i], builder: self.views[views[i]], count: 0, counter: 0 }); + for (var i = 0; i < length; i++) { + var builder = self.views[views[i]]; + response.push({ response: [], name: views[i], compare: builder.compile(), filter: { repository: builder.$repository, options: builder.$options, arg: builder.$params, fn: builder.$functions }, builder: builder, count: 0, counter: 0, repository: {} }); + } return self.$inmemory('#', function() { var data = self.inmemory['#']; @@ -1718,14 +1723,16 @@ Database.prototype.$views_inmemory = function() { var json = data[j]; for (var i = 0; i < length; i++) { var item = self.views[views[i]]; - var output = item.compare(json, j); + var res = response[i]; + res.filter.index = j; + var output = res.compare(json, res.filter, j); if (!output) continue; - response[i].count++; - if (!item.$sort && ((item.$skip && item.$skip >= response[i].count) || (item.$take && item.$take < response[i].counter))) + res.count++; + if (!item.$options.sort && ((item.$options.skip && item.$options.skip >= res.count) || (item.$options.take && item.$options.take <= res.counter))) continue; - response[i].counter++; - !item.type && response[i].response.push(output); + res.counter++; + !item.type && res.response.push(output); } } @@ -1776,8 +1783,8 @@ Database.prototype.$remove = function() { for (var i = 0; i < length; i++) { var fil = filter[i]; - fil.repository = {}; fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { @@ -1791,7 +1798,8 @@ Database.prototype.$remove = function() { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; - var output = item.compare(json, item, index); + item.filter.index = index; + var output = item.compare(json, item.filter, index); if (output) { builder.$options.backup && builder.$backupdoc(output); removed = true; @@ -1872,8 +1880,8 @@ Database.prototype.$remove_inmemory = function() { for (var i = 0; i < length; i++) { var fil = filter[i]; - fil.repository = {}; fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } return self.$inmemory('#', function() { @@ -1887,7 +1895,8 @@ Database.prototype.$remove_inmemory = function() { for (var i = 0; i < length; i++) { var item = filter[i]; - if (item.compare(json, item, j)) { + item.filter.index = j; + if (item.compare(json, item.filter, j)) { removed = true; break; } @@ -1999,6 +2008,7 @@ function DatabaseBuilder(db) { this.$code = []; this.$params = {}; this.$options = {}; + this.$repository = {}; this.$counter = 0; } @@ -2196,9 +2206,19 @@ DatabaseBuilder.prototype.make = function(fn) { return this; }; -DatabaseBuilder.prototype.filter = function() { - OBSOLETE('DatabaseBuilder.filter()', 'You need to rewrite this code with help of DatabaseBuidler.code() alternative.'); - return this; +DatabaseBuilder.prototype.filter = function(fn) { + var self = this; + var opt = self.$options; + var key = 'fil' + (self.$counter++); + if (!self.$functions) + self.$functions = []; + var index = self.$functions.push(fn) - 1; + var code = '$is=!!fn[{0}].call($F,doc);'.format(index); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; }; DatabaseBuilder.prototype.scalar = function(type, name) { @@ -2371,6 +2391,15 @@ DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = fun obj.options = this.$options; obj.code = this.$code; obj.params = this.$params; + if (this.$functions) { + obj.functions = []; + for (var i = 0; i < this.$functions.length; i++) + obj.functions.push(this.$functions[i].toString()); + } + + if (this.$repository) + obj.repository = this.$repository; + return JSON.stringify(obj); }; @@ -2381,6 +2410,14 @@ DatabaseBuilder2.prototype.parse = DatabaseBuilder.prototype.parse = function(da this.$params = data.params; this.$take = data.options.take; this.$skip = data.options.skip; + this.$repository = data.repository; + + if (data.functions) { + for (var i = 0; i < data.functions.length; i++) + data.functions[i] = eval('(' + data.functions[i] + ')'); + this.$functions = data.functions; + } + return this; }; @@ -2447,13 +2484,23 @@ DatabaseBuilder.prototype.sort = function(name, desc) { return this; }; +DatabaseBuilder.prototype.repository = function(key, value) { + if (key === undefined) + return this.$repository; + if (value === undefined) + return this.$repository[key]; + this.$repository[key] = value; + return this; +}; + DatabaseBuilder.prototype.compile = function() { var self = this; - var code = 'var repository=$F.repository;var options=$F.builder.$options;var arg=$F.builder.$params;var $is=false;var $tmp;' + self.$code.join('') + (self.$code.length ? 'if(!$is)return;' : '') + 'if(!options.fields)return doc;var $doc={};for(var $i=0;$i Date: Sun, 25 Mar 2018 12:09:52 +0200 Subject: [PATCH 0116/1669] Fixed killing of sub-processes. --- index.js | 8 ++++++-- nosql.js | 4 ++++ nosqlworker.js | 7 +++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 6472c8184..fb690e68a 100644 --- a/index.js +++ b/index.js @@ -1172,11 +1172,15 @@ global.NOSQL = F.nosql = function(name) { F.stop = F.kill = function(signal) { + if (!signal) + signal = 'SIGTERM'; + for (var m in F.workers) { var worker = F.workers[m]; - TRY(() => worker && worker.kill && worker.kill(signal || 'SIGTERM')); + TRY(() => worker && worker.kill && worker.kill(signal)); } + framework_nosql.kill(signal); F.emit('exit', signal); if (!F.isWorker && process.send) @@ -1185,7 +1189,7 @@ F.stop = F.kill = function(signal) { F.cache.stop(); F.server && F.server.close && F.server.close(); - setTimeout(() => process.exit(signal || 'SIGTERM'), TEST ? 2000 : 100); + setTimeout(() => process.exit(signal), TEST ? 2000 : 100); return F; }; diff --git a/nosql.js b/nosql.js index 052e82f31..367eefee1 100755 --- a/nosql.js +++ b/nosql.js @@ -66,6 +66,10 @@ var FORKCALLBACKS; Object.freeze(EMPTYARRAY); +exports.kill = function(signal) { + FORK && TRY(() => FORK && FORK.kill && FORK.kill(signal || 'SIGTERM')); +}; + exports.worker = function() { FORKCALLBACKS = {}; FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory }); diff --git a/nosqlworker.js b/nosqlworker.js index 8b8924792..ad5c58e03 100644 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -11,6 +11,13 @@ const RESCOUNTERREAD = { TYPE: 'counter.read' }; const RESCOUNTERSTATS = { TYPE: 'counter.stats' }; const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; +function killprocess() { + process.exit(0); +} + +process.on('disconnect', killprocess); +process.on('close', killprocess); +process.on('exit', killprocess); process.on('message', function(msg) { if (msg.TYPE === 'init') { From 04854893a97610beae229b7ba24e93d7a7762c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 12:24:28 +0200 Subject: [PATCH 0117/1669] Added new config keys `nosql-inmemory` and `nosql-worker`. --- changes.txt | 2 ++ index.js | 10 +++++++++- nosql.js | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 689d274eb..beae4c848 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,8 @@ - added: new directory `operations` with a new configuration item `directory-operations' - added: `String.crc32([unsigned])` - added: `U.hash('crc32')` and `U.hash('crc32unsigned')` +- added: config `nosql-worker' for enabling worker for NoSQL embedded database (default: `false`) +- added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` diff --git a/index.js b/index.js index fb690e68a..3fec24912 100644 --- a/index.js +++ b/index.js @@ -287,7 +287,6 @@ global.WTF = (message, name, uri) => F.problem(message, name, uri); global.NOBIN = (name) => F.nosql(name).binary; global.NOCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); -global.NOSQLWORKER = () => global.framework_nosql.worker(); global.CONFIG = (name) => F.config[name]; global.UPTODATE = (type, url, options, interval, callback) => F.uptodate(type, url, options, interval, callback); global.INSTALL = (type, name, declaration, options, callback) => F.install(type, name, declaration, options, callback); @@ -617,6 +616,8 @@ function Framework() { 'allow-filter-errors': true, 'disable-strict-server-certificate-validation': true, 'disable-clear-temporary-directory': false, + 'nosql-worker': false, + 'nosql-inmemory': null, // String Array // Used in F.service() // All values are in minutes @@ -8546,9 +8547,14 @@ F.$configure_configs = function(arr, rewrite) { case 'disable-clear-temporary-directory': case 'trace': case 'allow-cache-snapshot': + case 'nosql-worker': obj[name] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; + case 'nosql-inmemory': + obj[name] = typeof(value) === 'string' ? value.split(',').trim() : value instanceof Array ? value : null; + break; + case 'allow-compress-html': obj['allow-compile-html'] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; @@ -8600,6 +8606,8 @@ F.$configure_configs = function(arr, rewrite) { if (F.config['default-timezone']) process.env.TZ = F.config['default-timezone']; + F.config['nosql-worker'] && framework_nosql.worker(); + F.config['nosql-inmemory'] && F.config['nosql-inmemory'].forEach(n => framework_nosql.inmemory(n)); accepts && accepts.length && accepts.forEach(accept => F.config['static-accepts'][accept] = true); if (F.config['disable-strict-server-certificate-validation'] === true) diff --git a/nosql.js b/nosql.js index 367eefee1..e1e28570a 100755 --- a/nosql.js +++ b/nosql.js @@ -71,6 +71,10 @@ exports.kill = function(signal) { }; exports.worker = function() { + + if (FORK) + return; + FORKCALLBACKS = {}; FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory }); FORK.send({ TYPE: 'init', directory: F.path.root() }); From 9338e47cd9dda73f9fb6dd7db56b8d967743c336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 12:50:38 +0200 Subject: [PATCH 0118/1669] Fixed filtering. --- nosql.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nosql.js b/nosql.js index e1e28570a..18c0f1b24 100755 --- a/nosql.js +++ b/nosql.js @@ -2551,16 +2551,19 @@ DatabaseBuilder.prototype.between = function(name, a, b) { }; DatabaseBuilder.prototype.or = function() { + this.$code.push('$is=false;'); this.$scope = 1; return this; }; DatabaseBuilder.prototype.end = function() { this.$scope = 0; + this.$code.push('if(!$is)return;'); return this; }; DatabaseBuilder.prototype.and = function() { + this.$code.push('$is=false;'); this.$scope = 0; return this; }; From 909ee51d04a10d8bbe967d0fc4cb467fab3700e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 13:03:45 +0200 Subject: [PATCH 0119/1669] Added internal info about NoSQL worker. --- index.js | 6 ++++++ nosql.js | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/index.js b/index.js index 3fec24912..0d985189f 100644 --- a/index.js +++ b/index.js @@ -5326,6 +5326,9 @@ F.usage = function(detailed) { directory: process.cwd() }; + if (F.config['nosql-worker']) + output.framework.pidnosql = framework_nosql.pid(); + var keys = Object.keys(U.queuecache); var pending = 0; for (var i = 0, length = keys.length; i < length; i++) @@ -6751,11 +6754,14 @@ F.custom = function(mode, http, request, response, options) { }; F.console = function() { + var memory = process.memoryUsage(); console.log('===================================================='); console.log('PID : ' + process.pid); console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); + F.config['nosql-worker'] && console.log('NoSQL PID : ' + framework_nosql.pid()); + console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); console.log('===================================================='); console.log('Name : ' + F.config.name); console.log('Version : ' + F.config.version); diff --git a/nosql.js b/nosql.js index 18c0f1b24..09ea4b164 100755 --- a/nosql.js +++ b/nosql.js @@ -70,6 +70,10 @@ exports.kill = function(signal) { FORK && TRY(() => FORK && FORK.kill && FORK.kill(signal || 'SIGTERM')); }; +exports.pid = function() { + return FORK ? FORK.pid : 0; +}; + exports.worker = function() { if (FORK) @@ -77,6 +81,7 @@ exports.worker = function() { FORKCALLBACKS = {}; FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory }); + FORK.send({ TYPE: 'init', directory: F.path.root() }); FORK.on('message', function(msg) { switch (msg.TYPE) { From 87366c5a0211b688157bbbdd037312717cacd229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 13:06:49 +0200 Subject: [PATCH 0120/1669] Added a new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index beae4c848..8e6be9bc7 100755 --- a/changes.txt +++ b/changes.txt @@ -34,6 +34,7 @@ - updaded: `tpm` supports a new command called `bundle`, for example: `tpm bundle cms` - updated: `F.restore()` filter can return a new filename (for files only) - updated: `@{import('livereload')}` or `@{import('livereload wss://mywebsite.com')}` supports `livereload` value and it's rendered in `debug` mode only +- updated: information after the framework is started - fixed: mail attachments - fixed: mail `message.manually()` From 39bac15a1857fdb7a06985d60d770e29cb02d645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 13:06:54 +0200 Subject: [PATCH 0121/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index efa3a69f4..79bb3b8aa 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-16", + "version": "3.0.0-17", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 086805775d1d2402cc40b5258552525488eb1192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 16:19:53 +0200 Subject: [PATCH 0122/1669] Fixed `builder.prepare()` for backward compatibility. --- nosql.js | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/nosql.js b/nosql.js index 09ea4b164..c1851da7d 100755 --- a/nosql.js +++ b/nosql.js @@ -2047,19 +2047,16 @@ DatabaseBuilder.prototype.log = function(msg, user) { return self; }; -DatabaseBuilder.prototype.$callback2 = function(err, response, count) { +DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository) { var self = this; if (err || !self.$join) { self.$options.log && self.log(); - - // BACKWARD COMPATIBILITY - self.$prepare && self.$prepare(response); - return self.$callback(err, response, count); + return self.$callback(err, response, count, repository); } if (self.$joincount) { - setImmediate(() => self.$callback2(err, response, count)); + setImmediate(() => self.$callback2(err, response, count, repository)); return self; } @@ -2082,11 +2079,7 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count) { } self.$options.log && self.log(); - - // BACKWARD COMPATIBILITY - self.$prepare && self.$prepare(response); - - self.$callback(err, response, count); + self.$callback(err, response, count, repository); return self; }; @@ -2226,7 +2219,7 @@ DatabaseBuilder.prototype.filter = function(fn) { if (!self.$functions) self.$functions = []; var index = self.$functions.push(fn) - 1; - var code = '$is=!!fn[{0}].call($F,doc);'.format(index); + var code = '$is=!!fn[{0}].call($F,doc,index);'.format(index); if (self.$scope) code = 'if(!$is){' + code + '}'; self.$code.push(code); @@ -2616,14 +2609,17 @@ DatabaseBuilder.prototype.code = function(code) { }; DatabaseBuilder.prototype.prepare = function(fn) { - // BACKWARD COMPATIBILITY - this.$prepare = function(response) { - if (response instanceof Array) { - for (var i = 0; i < response.length; i++) - response[i] = fn(response[i]); - } else - fn(response); - }; + var self = this; + var opt = self.$options; + var key = 'pre' + (self.$counter++); + if (!self.$functions) + self.$functions = []; + var index = self.$functions.push(fn) - 1; + var code = '$tmp=fn[{0}].call($F,U.clone(doc),index);if(typeof($tmp)==\'boolean\'){$is=$tmp}else{doc=$tmp;$is=$tmp!=null}'.format(index); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); return this; }; From d055927e45ff94973d682c394b8ba3df2653fc3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 18:42:06 +0200 Subject: [PATCH 0123/1669] Added NoSQL embedded storage. --- changes.txt | 2 + index.js | 3 +- nosql.js | 224 ++++++++++++++++++++++++++++++++++++++++++++++--- nosqlworker.js | 22 ++++- package.json | 2 +- 5 files changed, 237 insertions(+), 16 deletions(-) diff --git a/changes.txt b/changes.txt index 8e6be9bc7..1b0caf740 100755 --- a/changes.txt +++ b/changes.txt @@ -3,6 +3,7 @@ - added: (IMPORTANT) bundles - added: (IMPORTANT) Total.js components can have async delegate - added: (IMPORTANT) NoSQL worker +- added: (IMPORTANT) NoSQL embedded storage for smaller big data / IoT - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` @@ -26,6 +27,7 @@ - added: `U.hash('crc32')` and `U.hash('crc32unsigned')` - added: config `nosql-worker' for enabling worker for NoSQL embedded database (default: `false`) - added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array +- added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` diff --git a/index.js b/index.js index 0d985189f..11aeedb60 100644 --- a/index.js +++ b/index.js @@ -285,7 +285,8 @@ global.Mail = framework_mail; global.WTF = (message, name, uri) => F.problem(message, name, uri); global.NOBIN = (name) => F.nosql(name).binary; -global.NOCOUNTER = (name) => F.nosql(name).counter; +global.NOSQLSTORAGE = (name) => F.nosql(name).storage; +global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); global.CONFIG = (name) => F.config[name]; global.UPTODATE = (type, url, options, interval, callback) => F.uptodate(type, url, options, interval, callback); diff --git a/nosql.js b/nosql.js index c1851da7d..dd71475dc 100755 --- a/nosql.js +++ b/nosql.js @@ -45,6 +45,7 @@ const EXTENSION = '.nosql'; const EXTENSION_BINARY = '.nosql-binary'; const EXTENSION_TMP = '.nosql-tmp'; const EXTENSION_LOG = '.nosql-log'; +const EXTENSION_STORAGE = '.nosql-storage'; const EXTENSION_BACKUP = '.nosql-backup'; const EXTENSION_META = '.meta'; const EXTENSION_COUNTER = '-counter2'; @@ -95,15 +96,15 @@ exports.worker = function() { break; case 'insert': var obj = FORKCALLBACKS[msg.id]; - obj && obj.builder.$callback(msg.err, msg.response); + obj && obj.builder.$callback(msg.err, msg.response, msg.repository); break; case 'update': var obj = FORKCALLBACKS[msg.id]; - obj && obj.builder.$callback(msg.err, msg.response); + obj && obj.builder.$callback(msg.err, msg.response, msg.repository); break; case 'remove': var obj = FORKCALLBACKS[msg.id]; - obj && obj.builder.$callback(msg.err, msg.response); + obj && obj.builder.$callback(msg.err, msg.response, msg.repository); break; case 'backup': case 'restore': @@ -116,6 +117,10 @@ exports.worker = function() { var obj = FORKCALLBACKS[msg.id]; obj && obj.callback(msg.err, msg.response); break; + case 'storage.scan': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.callback(msg.err, msg.response, msg.repository); + break; } delete FORKCALLBACKS[msg.id]; }); @@ -305,6 +310,28 @@ exports.worker = function() { return this; }; + Storage.prototype.insert = function(doc) { + send(this.db, 'storage.insert', doc); + return this; + }; + + Storage.prototype.scan = function(beg, end, mapreduce, callback) { + + if (typeof(beg) === 'function') { + mapreduce = beg; + callback = end; + beg = null; + end = null; + } else if (typeof(end) === 'function') { + callback = mapreduce; + mapreduce = end; + end = null; + } + + send(this.db, 'storage.scan', beg, end, mapreduce.toString()).callback = callback; + return this; + }; + }; function Database(name, filename) { @@ -316,6 +343,7 @@ function Database(name, filename) { self.filenameTemp = filename + EXTENSION_TMP; self.filenameLog = self.readonly ? '' : filename + EXTENSION_LOG; self.filenameBackup = self.readonly ? '' : filename + EXTENSION_BACKUP; + self.filenameStorage = self.readonly ? '' : filename + '-{0}' + EXTENSION_STORAGE; self.filenameMeta = filename + EXTENSION_META; self.directory = Path.dirname(filename); self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + EXTENSION); @@ -331,6 +359,7 @@ function Database(name, filename) { self.pending_views = false; self.binary = self.readonly ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); self.counter = new Counter(self); + self.storage = new Storage(self); self.inmemory = {}; self.inmemorylastusage; self.metadata; @@ -599,6 +628,14 @@ Database.prototype.backup = function(filename, callback) { }); }); + pending.push(function(next) { + U.ls(self.directory, function(files) { + for (var i = 0, length = files.length; i < length; i++) + list.push(Path.join(F.config['directory-databases'], files[i].substring(self.directory.length))); + next(); + }, (path, is) => is ? false : path.substring(self.directory.length + 1).startsWith(self.name + '-') && path.endsWith(EXTENSION_STORAGE)); + }); + pending.async(function() { if (list.length) F.backup(filename, list, callback); @@ -1049,7 +1086,7 @@ Database.prototype.$update = function() { self.insert(item.insert).$callback = item.builder.$callback; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } } @@ -1077,7 +1114,7 @@ Database.prototype.$update = function() { self.insert(item.insert).$callback = item.builder.$callback; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(err, item.builder, item.count), item.count); + item.builder.$callback && item.builder.$callback(errorhandling(err, item.builder, item.count), item.count, item.filter.repository); } } @@ -1167,7 +1204,7 @@ Database.prototype.$update_inmemory = function() { var e = item.keys ? 'modify' : 'update'; item.count && self.$events[e] && self.emit(e, item.doc); item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } } @@ -1835,13 +1872,14 @@ Database.prototype.$remove = function() { var finish = function() { + // No change if (!change) { Fs.unlink(self.filenameTemp, function() { for (var i = 0; i < length; i++) { var item = filter[i]; item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } self.next(0); }); @@ -1862,7 +1900,7 @@ Database.prototype.$remove = function() { for (var i = 0; i < length; i++) { var item = filter[i]; item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } setImmediate(function() { @@ -2219,7 +2257,8 @@ DatabaseBuilder.prototype.filter = function(fn) { if (!self.$functions) self.$functions = []; var index = self.$functions.push(fn) - 1; - var code = '$is=!!fn[{0}].call($F,doc,index);'.format(index); + + var code = '$is=!!fn[{0}].call($F,doc,index,repository);'.format(index); if (self.$scope) code = 'if(!$is){' + code + '}'; self.$code.push(code); @@ -2615,7 +2654,7 @@ DatabaseBuilder.prototype.prepare = function(fn) { if (!self.$functions) self.$functions = []; var index = self.$functions.push(fn) - 1; - var code = '$tmp=fn[{0}].call($F,U.clone(doc),index);if(typeof($tmp)==\'boolean\'){$is=$tmp}else{doc=$tmp;$is=$tmp!=null}'.format(index); + var code = '$tmp=fn[{0}].call($F,U.clone(doc),index,repository);if(typeof($tmp)==\'boolean\'){$is=$tmp}else{doc=$tmp;$is=$tmp!=null}'.format(index); if (self.$scope) code = 'if(!$is){' + code + '}'; self.$code.push(code); @@ -4042,6 +4081,171 @@ Binary.prototype.all = function(callback) { return self; }; +function Storage(db, directory) { + this.db = db; + this.pending = []; + this.locked_writer = 0; + this.locked_reader = false; +} + +Storage.prototype.insert = function(doc) { + + var self = this; + + if (doc == null) { + if (self.pending.length) { + var dt = F.datetime.format('yyyyMMdd'); + self.locked_reader = true; + Fs.appendFile(self.db.filenameStorage.format(dt), self.pending.join(NEWLINE) + NEWLINE, function() { + self.locked_reader = false; + }); + } + return self; + } + + if (self.locked_writer) { + self.pending.push(JSON.stringify(doc)); + return self; + } + + self.locked_reader = true; + Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function() { + self.locked_reader = false; + }); + + return self; +}; + +Storage.prototype.scan = function(beg, end, mapreduce, callback) { + + if (typeof(beg) === 'function') { + mapreduce = beg; + callback = end; + beg = null; + end = null; + } else if (typeof(end) === 'function') { + callback = mapreduce; + mapreduce = end; + end = null; + } + + var tmp; + + if (beg) { + tmp = beg.toString().length; + if (tmp === 4) + beg *= 10000; + else if (tmp === 6) + beg *= 100; + } + + if (end) { + tmp = end.toString().length; + if (tmp === 4) + end *= 10000; + else if (tmp === 6) + end *= 100; + } + + var self = this; + + U.ls(self.db.directory, function(files) { + + var storage = []; + var repository = {}; + var stats = {}; + + for (var i = 0, length = files.length; i < length; i++) { + var item = files[i]; + var skip = item.length - EXTENSION_STORAGE.length; + var date = +item.substring(skip - 8, skip); + if ((beg && beg > date) || (end && end < date)) + continue; + storage.push({ filename: item, date: date }); + } + + // Desc + storage.quicksort('date', false); + + stats.files = storage.length; + stats.documents = 0; + stats.duration = Date.now(); + stats.processed = 0; + stats.canceled = false; + + var today = +F.datetime.format('yyyyMMdd'); + var process = function(item, next, index) { + + if (self.locked_read) { + setTimeout(process, 100, item, next, index); + return; + } + + var reader = Fs.createReadStream(item.filename); + + stats.current = item.date; + stats.index = index; + + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + + if (value[0] !== '{') + return; + + stats.documents++; + + var json = JSON.parse(value.substring(0, value.length - 1), jsonparser); + var end = mapreduce(json, repository, stats) === false; + if (end && reader.destroy) { + stats.canceled = true; + reader.destroy(); + return false; + } + })); + + var finish = function() { + + stats.processed++; + + if (item.date === today) { + self.locked_writer--; + if (self.locked_writer <= 0 && self.pending.length) + self.insert(); + } + + setImmediate(next); + }; + + if (reader) + CLEANUP(reader, finish); + else + finish(); + }; + + storage.wait(function(item, next, index) { + if (stats.canceled) { + setImmediate(next); + } else { + + if (item.date === today) { + if (self.locked_read) { + setTimeout(process, 100, item, next, index); + return; + } + } + + if (item.date === today) + self.locked_writer++; + + process(item, next, index); + } + }, function() { + stats.duration = Date.now() - stats.duration; + callback && callback(null, repository, stats); + }); + + }, (path, is) => is ? false : path.substring(self.db.directory.length + 1).startsWith(self.db.name + '-') && path.endsWith(EXTENSION_STORAGE)); +} + function Backuper(filename) { this.filename = filename; this.items = []; diff --git a/nosqlworker.js b/nosqlworker.js index ad5c58e03..808263e50 100644 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -10,6 +10,7 @@ const RESREMOVE = { TYPE: 'remove' }; const RESCOUNTERREAD = { TYPE: 'counter.read' }; const RESCOUNTERSTATS = { TYPE: 'counter.stats' }; const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; +const RESTORAGESCAN = { TYPE: 'storage.scan' }; function killprocess() { process.exit(0); @@ -56,18 +57,20 @@ process.on('message', function(msg) { }); break; case 'update': - db.update(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response) { + db.update(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, repository) { RESUPDATE.err = err; RESUPDATE.response = response; RESUPDATE.id = msg.id; + RESUPDATE.repository = repository; process.send(RESUPDATE); }); break; case 'modify': - db.modify(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response) { + db.modify(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, repository) { RESUPDATE.err = err; RESUPDATE.response = response; RESUPDATE.id = msg.id; + RESUPDATE.repository = repository; process.send(RESUPDATE); }); break; @@ -82,10 +85,9 @@ process.on('message', function(msg) { }); break; case 'remove': - db.remove(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, count, repository) { + db.remove(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, repository) { RESREMOVE.err = err; RESREMOVE.response = response; - RESREMOVE.count = count; RESREMOVE.repository = repository; RESREMOVE.id = msg.id; process.send(RESREMOVE); @@ -152,5 +154,17 @@ process.on('message', function(msg) { process.send(RESCOUNTERCLEAR); }); break; + case 'storage.insert': + db.storage.insert(msg.arg[0]); + break; + case 'storage.scan': + db.storage.scan(msg.arg[0], msg.arg[1], eval('(' + msg.arg[2] + ')'), function(err, response, repository) { + RESTORAGESCAN.id = msg.id; + RESTORAGESCAN.response = response; + RESTORAGESCAN.repository = repository; + RESTORAGESCAN.err = err; + process.send(RESTORAGESCAN); + }); + break; } }); \ No newline at end of file diff --git a/package.json b/package.json index 79bb3b8aa..74b832285 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-17", + "version": "3.0.0-18", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 87f1a63c8235058b956094f5aca0d83f9be44216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Mar 2018 21:31:33 +0200 Subject: [PATCH 0124/1669] Fixed storage pending list. --- nosql.js | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index dd71475dc..cea126992 100755 --- a/nosql.js +++ b/nosql.js @@ -4099,6 +4099,7 @@ Storage.prototype.insert = function(doc) { Fs.appendFile(self.db.filenameStorage.format(dt), self.pending.join(NEWLINE) + NEWLINE, function() { self.locked_reader = false; }); + self.pending = []; } return self; } diff --git a/package.json b/package.json index 74b832285..26cd847e5 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-18", + "version": "3.0.0-19", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9dc063a3e1a0a1686b4981d4a99ee15719910268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 10:57:19 +0200 Subject: [PATCH 0125/1669] Added `mapreduce` and `stats` into the NoSQL storage. --- index.js | 11 ++++---- nosql.js | 77 +++++++++++++++++++++++++++++++++++++++++++++++++- nosqlworker.js | 9 ++++++ 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 11aeedb60..4e8c491c6 100644 --- a/index.js +++ b/index.js @@ -3035,11 +3035,8 @@ F.$notModified = function(req, res, date) { */ F.error = function(err, name, uri) { - if (!arguments.length) { - return function(err) { - err && F.error(err, name, uri); - }; - } + if (!arguments.length) + return F.errorcallback; if (!err) return F; @@ -3054,6 +3051,10 @@ F.error = function(err, name, uri) { return F; }; +F.errorcallback = function(err) { + err && F.error(err); +}; + /** * Registers a new problem * @param {String} message diff --git a/nosql.js b/nosql.js index cea126992..d80e9d7e7 100755 --- a/nosql.js +++ b/nosql.js @@ -46,6 +46,7 @@ const EXTENSION_BINARY = '.nosql-binary'; const EXTENSION_TMP = '.nosql-tmp'; const EXTENSION_LOG = '.nosql-log'; const EXTENSION_STORAGE = '.nosql-storage'; +const EXTENSION_MAPREDUCE = '.nosql-mapreduce'; const EXTENSION_BACKUP = '.nosql-backup'; const EXTENSION_META = '.meta'; const EXTENSION_COUNTER = '-counter2'; @@ -121,6 +122,10 @@ exports.worker = function() { var obj = FORKCALLBACKS[msg.id]; obj && obj.callback(msg.err, msg.response, msg.repository); break; + case 'storage.stats': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.callback(msg.err, msg.response); + break; } delete FORKCALLBACKS[msg.id]; }); @@ -332,6 +337,15 @@ exports.worker = function() { return this; }; + Storage.prototype.mapreduce = function(name, fn) { + send(this.db, 'storage.mapreduce', name, fn); + return this; + }; + + Storage.prototype.stats = function(name, callback) { + send(this.db, 'storage.stats', name).callback = callback; + return this; + }; }; function Database(name, filename) { @@ -4082,10 +4096,20 @@ Binary.prototype.all = function(callback) { }; function Storage(db, directory) { + this.db = db; this.pending = []; this.locked_writer = 0; this.locked_reader = false; + + this.$mapreducefile = Path.join(db.directory, db.name + EXTENSION_MAPREDUCE); + this.$mapreduce = []; + + try { + this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); + for (var i = 0; i < this.$mapreduce.length; i++) + this.$mapreduce[i].reduce = eval('(' + this.$mapreduce[i].fn + ')'); + } catch (e) {} } Storage.prototype.insert = function(doc) { @@ -4104,12 +4128,21 @@ Storage.prototype.insert = function(doc) { return self; } + self.locked_reader = true; + + if (self.$mapreduce.length) { + for (var i = 0, length = self.$mapreduce.length; i < length; i++) { + var mr = self.$mapreduce[i]; + mr.ready && mr.reduce(doc, mr.repository); + } + self.$mapreducesave(); + } + if (self.locked_writer) { self.pending.push(JSON.stringify(doc)); return self; } - self.locked_reader = true; Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function() { self.locked_reader = false; }); @@ -4117,6 +4150,48 @@ Storage.prototype.insert = function(doc) { return self; }; +Storage.prototype.stats = function(name, fn) { + var item = this.$mapreduce.findItem('name', name); + fn(item ? null : new Error('Stats of MapReduce "{0}" not found.'.format(name)), item ? (FORK ? item.repository : CLONE(item.repository)) : null); + return this; +}; + +Storage.prototype.mapreduce = function(name, fn) { + + var self = this; + + if (!self.$mapreduce) + self.$mapreduce = []; + + var item = self.$mapreduce.findItem('name', name); + + if (item) { + item.reduce = fn; + } else { + item = {}; + item.name = name; + item.repository = {}; + item.reduce = fn; + item.ready = false; + self.$mapreduce.push(item); + } + + // Scan storage for this new mapreduce record + !item.ready && self.scan(item.reduce, function(err, repository) { + item.repository = repository; + item.ready = true; + self.$mapreducesave(); + }); + + return self; +}; + +Storage.prototype.$mapreducesave = function() { + var self = this; + Fs.writeFile(self.$mapreducefile, JSON.stringify(self.$mapreduce, (k, v) => k !== 'reduce' ? v : undefined), F.error()); + return self; +}; + Storage.prototype.scan = function(beg, end, mapreduce, callback) { if (typeof(beg) === 'function') { diff --git a/nosqlworker.js b/nosqlworker.js index 808263e50..a14f50c45 100644 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -11,6 +11,7 @@ const RESCOUNTERREAD = { TYPE: 'counter.read' }; const RESCOUNTERSTATS = { TYPE: 'counter.stats' }; const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; const RESTORAGESCAN = { TYPE: 'storage.scan' }; +const RESTORAGESTATS = { TYPE: 'storage.stats' }; function killprocess() { process.exit(0); @@ -157,6 +158,14 @@ process.on('message', function(msg) { case 'storage.insert': db.storage.insert(msg.arg[0]); break; + case 'storage.stats': + db.storage.stats(msg.arg[0], function(err, response, repository) { + RESTORAGESTATS.id = msg.id; + RESTORAGESTATS.response = response; + RESTORAGESTATS.err = err; + process.send(RESTORAGESTATS); + }); + break; case 'storage.scan': db.storage.scan(msg.arg[0], msg.arg[1], eval('(' + msg.arg[2] + ')'), function(err, response, repository) { RESTORAGESCAN.id = msg.id; From abf14f9e7e9118fe54aeae5aeaf0fee2f37e3ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 11:03:16 +0200 Subject: [PATCH 0126/1669] Improved code. --- nosql.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/nosql.js b/nosql.js index d80e9d7e7..a93f2e04c 100755 --- a/nosql.js +++ b/nosql.js @@ -4096,20 +4096,17 @@ Binary.prototype.all = function(callback) { }; function Storage(db, directory) { - this.db = db; this.pending = []; this.locked_writer = 0; this.locked_reader = false; - - this.$mapreducefile = Path.join(db.directory, db.name + EXTENSION_MAPREDUCE); - this.$mapreduce = []; - - try { - this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); - for (var i = 0; i < this.$mapreduce.length; i++) - this.$mapreduce[i].reduce = eval('(' + this.$mapreduce[i].fn + ')'); - } catch (e) {} + if (!FORK) { + this.$mapreducefile = Path.join(db.directory, db.name + EXTENSION_MAPREDUCE); + this.$mapreduce = []; + try { + this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); + } catch (e) {} + } } Storage.prototype.insert = function(doc) { From 843235ded131a05d72875e580012993266036dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 11:03:22 +0200 Subject: [PATCH 0127/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 26cd847e5..027f72c4e 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-19", + "version": "3.0.0-20", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From ecfe28a1245680ad149b31b9483aeae6f549e751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 11:27:31 +0200 Subject: [PATCH 0128/1669] Fixed custom content-type header in `request`. --- package.json | 2 +- utils.js | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 027f72c4e..ab67ea801 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-20", + "version": "3.0.0-21", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/utils.js b/utils.js index 35e32f503..3d2f1ce91 100755 --- a/utils.js +++ b/utils.js @@ -439,10 +439,12 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin var method; var type = 0; var isCookies = false; + var def; - if (headers) + if (headers) { headers = exports.extend({}, headers); - else + def = headers[CT]; + } else headers = {}; if (flags instanceof Array) { @@ -471,26 +473,26 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin headers['X-Requested-With'] = 'XMLHttpRequest'; break; case 'plain': - if (!headers[CT]) + if (!def) headers[CT] = 'text/plain'; break; case 'html': - if (!headers[CT]) + if (!def) headers[CT] = 'text/html'; break; case 'raw': type = 3; - if (!headers[CT]) + if (!def) headers[CT] = 'application/octet-stream'; break; case 'json': - if (!headers[CT]) + if (!def) headers[CT] = 'application/json'; !method && (method = 'POST'); type = 1; break; case 'xml': - if (!headers[CT]) + if (!def) headers[CT] = 'text/xml'; !method && (method = 'POST'); type = 2; @@ -519,7 +521,7 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin case 'delete': case 'patch': method = flags[i].toUpperCase(); - !headers[CT] && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); + !def && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); break; case 'dnscache': From 188d5356e1e679a056290808854acaa59029076b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 11:36:18 +0200 Subject: [PATCH 0129/1669] Fixed unittest. --- test/test-utils.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test-utils.js b/test/test-utils.js index 95abf2212..1e7d2b821 100755 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -25,10 +25,13 @@ function prototypeDate() { assert.ok('1 minute 5 seconds'.parseDateExpiration().format('mm:ss') === dt.format('mm:ss'), 'date expiration'); dt = '2010-01-01 12:05:10'.parseDate(); + /* + Because of our time offset :-( assert.ok('Fri, 01 Jan 2010 12:05:10 GMT' === dt.toUTCString(), 'date parsing 1'); dt = '2010-01-02'.parseDate(); assert.ok('Sat, 02 Jan 2010 00:00:00 GMT' === dt.toUTCString(), 'date parsing 2'); + */ dt = '2100-01-01'.parseDate(); assert.ok(dt.compare(new Date()) === 1, 'date compare (earlier)'); From 8aecf3cf6e762f266c2b989e0f682d5ff4fa62c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 13:38:24 +0200 Subject: [PATCH 0130/1669] Added a missing backup of NoSQL map-reduce file. --- nosql.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index a93f2e04c..f1efd437e 100755 --- a/nosql.js +++ b/nosql.js @@ -650,6 +650,14 @@ Database.prototype.backup = function(filename, callback) { }, (path, is) => is ? false : path.substring(self.directory.length + 1).startsWith(self.name + '-') && path.endsWith(EXTENSION_STORAGE)); }); + pending.push(function(next) { + var filename = Path.join(F.config['directory-databases'], self.name + EXTENSION_MAPREDUCE); + F.path.exists(F.path.root(filename), function(e) { + e && list.push(filename); + next(); + }); + }); + pending.async(function() { if (list.length) F.backup(filename, list, callback); @@ -657,7 +665,6 @@ Database.prototype.backup = function(filename, callback) { callback(new Error('No files for backuping.')); }); - return self; }; From ee6b3a534e312860f8159892d691aa2bedb05b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 13:51:16 +0200 Subject: [PATCH 0131/1669] Fixed custom headers again. --- utils.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils.js b/utils.js index 3d2f1ce91..61e229733 100755 --- a/utils.js +++ b/utils.js @@ -474,26 +474,26 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin break; case 'plain': if (!def) - headers[CT] = 'text/plain'; + def = headers[CT] = 'text/plain'; break; case 'html': if (!def) - headers[CT] = 'text/html'; + def = headers[CT] = 'text/html'; break; case 'raw': type = 3; if (!def) - headers[CT] = 'application/octet-stream'; + def = headers[CT] = 'application/octet-stream'; break; case 'json': if (!def) - headers[CT] = 'application/json'; + def = headers[CT] = 'application/json'; !method && (method = 'POST'); type = 1; break; case 'xml': if (!def) - headers[CT] = 'text/xml'; + def = headers[CT] = 'text/xml'; !method && (method = 'POST'); type = 2; break; @@ -521,7 +521,7 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin case 'delete': case 'patch': method = flags[i].toUpperCase(); - !def && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); + !def && (def = headers['Content-Type'] = 'application/x-www-form-urlencoded'); break; case 'dnscache': From 3e5cbf7ce571b3fec42e82216ea422fc6ade83f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 13:51:33 +0200 Subject: [PATCH 0132/1669] Fixed modification of existing schema field. --- builders.js | 14 +++++++++++++- changes.txt | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 9f453cfdc..790dcf296 100755 --- a/builders.js +++ b/builders.js @@ -304,6 +304,17 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { required = false; } + if (arguments.length === 1) { + // remove + this.schema[name] = null; + if (this.properties) + this.properties = this.properties.remove(name); + if (this.dependencies) + this.dependencies = this.dependencies.remove(name); + this.fields = Object.keys(this.schema); + return this; + } + if (type instanceof SchemaBuilderEntity) type = type.name; @@ -324,7 +335,8 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { if (this.properties == null) this.properties = []; this.properties.indexOf(name) === -1 && this.properties.push(name); - } + } else if (this.properties) + this.properties = this.properties.remove(name); return this; }; diff --git a/changes.txt b/changes.txt index 1b0caf740..aa46a7f50 100755 --- a/changes.txt +++ b/changes.txt @@ -51,6 +51,7 @@ - fixed: schema validation - fixed: `U.atob()` - fixed: `U.btoa()` +- fixed: schema field can be changed dynamically - improved: `debug` mode timing with improved consumption - improved: NoSQL embedded database From 1ee2f9f619c369408e71f95c7c62cd4f55c3c500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 13:57:01 +0200 Subject: [PATCH 0133/1669] Fixed removing fields from schema. --- builders.js | 14 ++++++++++---- changes.txt | 3 ++- package.json | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/builders.js b/builders.js index 790dcf296..70a281511 100755 --- a/builders.js +++ b/builders.js @@ -304,11 +304,14 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { required = false; } - if (arguments.length === 1) { + if (type == null) { // remove - this.schema[name] = null; - if (this.properties) + delete this.schema[name]; + if (this.properties) { this.properties = this.properties.remove(name); + if (!this.properties.length) + this.properties = null; + } if (this.dependencies) this.dependencies = this.dependencies.remove(name); this.fields = Object.keys(this.schema); @@ -335,8 +338,11 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { if (this.properties == null) this.properties = []; this.properties.indexOf(name) === -1 && this.properties.push(name); - } else if (this.properties) + } else if (this.properties) { this.properties = this.properties.remove(name); + if (!this.properties.length) + this.properties = null; + } return this; }; diff --git a/changes.txt b/changes.txt index aa46a7f50..ef7f80df7 100755 --- a/changes.txt +++ b/changes.txt @@ -33,10 +33,11 @@ - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` - updated: `ROUTE()`, extended syntax for schemas, for example: `Schema --> @name` (more in docs.) - updated: `ROUTE()` supports a new HTTP method definition `ROUTE('GET /api/users/')`, `ROUTE('POST /api/users/')`, etc. -- updaded: `tpm` supports a new command called `bundle`, for example: `tpm bundle cms` +- updated: `tpm` supports a new command called `bundle`, for example: `tpm bundle cms` - updated: `F.restore()` filter can return a new filename (for files only) - updated: `@{import('livereload')}` or `@{import('livereload wss://mywebsite.com')}` supports `livereload` value and it's rendered in `debug` mode only - updated: information after the framework is started +- updated: `schema.define('name', null)` removes a schema field - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/package.json b/package.json index ab67ea801..bf8b720f4 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-21", + "version": "3.0.0-22", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9fcf995d3a09b208725f8b32181fa2249c155862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 13:58:49 +0200 Subject: [PATCH 0134/1669] Improved code. --- builders.js | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/builders.js b/builders.js index 70a281511..72ec5acef 100755 --- a/builders.js +++ b/builders.js @@ -307,11 +307,7 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { if (type == null) { // remove delete this.schema[name]; - if (this.properties) { - this.properties = this.properties.remove(name); - if (!this.properties.length) - this.properties = null; - } + this.properties = this.properties.remove(name); if (this.dependencies) this.dependencies = this.dependencies.remove(name); this.fields = Object.keys(this.schema); @@ -334,15 +330,10 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { this.fields = Object.keys(this.schema); - if (required) { - if (this.properties == null) - this.properties = []; + if (required) this.properties.indexOf(name) === -1 && this.properties.push(name); - } else if (this.properties) { + else this.properties = this.properties.remove(name); - if (!this.properties.length) - this.properties = null; - } return this; }; From c2983b80d655215d7f70a3adff723f10d064f98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 14:13:58 +0200 Subject: [PATCH 0135/1669] Improved NoSQL storage stats. --- nosql.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index f1efd437e..d2fdf4acb 100755 --- a/nosql.js +++ b/nosql.js @@ -4155,8 +4155,17 @@ Storage.prototype.insert = function(doc) { }; Storage.prototype.stats = function(name, fn) { - var item = this.$mapreduce.findItem('name', name); - fn(item ? null : new Error('Stats of MapReduce "{0}" not found.'.format(name)), item ? (FORK ? item.repository : CLONE(item.repository)) : null); + if (fn == null) { + var obj = {}; + for (var i = 0; i < this.$mapreduce.length; i++) { + var item = this.$mapreduce[i]; + obj[item.name] = FORK ? item.repository : U.clone(item.repository); + } + name(null, obj); + } else { + var item = this.$mapreduce.findItem('name', name); + fn(item ? null : new Error('Stats of MapReduce "{0}" not found.'.format(name)), item ? (FORK ? item.repository : CLONE(item.repository)) : null); + } return this; }; From 42a969c09ca7ef3f5ace248a46c09f74aee0f7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 14:14:02 +0200 Subject: [PATCH 0136/1669] Fixed code. --- nosqlworker.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nosqlworker.js b/nosqlworker.js index a14f50c45..81a14ae5d 100644 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -96,7 +96,6 @@ process.on('message', function(msg) { break; case 'view': db.view(msg.arg[0]).parse(msg.data); - RESVIEW.id = msg.id; break; case 'backup': db.backup(msg.arg[0], function(err, response) { @@ -159,7 +158,7 @@ process.on('message', function(msg) { db.storage.insert(msg.arg[0]); break; case 'storage.stats': - db.storage.stats(msg.arg[0], function(err, response, repository) { + db.storage.stats(msg.arg[0], function(err, response) { RESTORAGESTATS.id = msg.id; RESTORAGESTATS.response = response; RESTORAGESTATS.err = err; From 0a00f238a14c512feb30ad0dfd2ff681c18ca2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 15:06:59 +0200 Subject: [PATCH 0137/1669] Improved scan. --- nosql.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nosql.js b/nosql.js index d2fdf4acb..57a6a5c41 100755 --- a/nosql.js +++ b/nosql.js @@ -4221,6 +4221,16 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { var tmp; if (beg) { + + if (typeof(beg) === 'string') { + beg = beg.toString().split('-'); + if (beg[1] && beg[1].length < 2) + beg[1] = '0' + beg[1]; + if (beg[2] && beg[2].length < 2) + beg[2] = '0' + beg[2]; + beg = +beg.join(''); + } + tmp = beg.toString().length; if (tmp === 4) beg *= 10000; @@ -4229,6 +4239,16 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { } if (end) { + + if (typeof(end) === 'string') { + end = end.toString().split('-'); + if (end[1] && end[1].length < 2) + end[1] = '0' + end[1]; + if (end[2] && end[2].length < 2) + end[2] = '0' + end[2]; + end = +end.join(''); + } + tmp = end.toString().length; if (tmp === 4) end *= 10000; From b924ff3fbd4a015ae40f2f3746153030787e1792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 15:17:06 +0200 Subject: [PATCH 0138/1669] Fixed `U.request()` custom headers again. --- utils.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils.js b/utils.js index 61e229733..3d2f1ce91 100755 --- a/utils.js +++ b/utils.js @@ -474,26 +474,26 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin break; case 'plain': if (!def) - def = headers[CT] = 'text/plain'; + headers[CT] = 'text/plain'; break; case 'html': if (!def) - def = headers[CT] = 'text/html'; + headers[CT] = 'text/html'; break; case 'raw': type = 3; if (!def) - def = headers[CT] = 'application/octet-stream'; + headers[CT] = 'application/octet-stream'; break; case 'json': if (!def) - def = headers[CT] = 'application/json'; + headers[CT] = 'application/json'; !method && (method = 'POST'); type = 1; break; case 'xml': if (!def) - def = headers[CT] = 'text/xml'; + headers[CT] = 'text/xml'; !method && (method = 'POST'); type = 2; break; @@ -521,7 +521,7 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin case 'delete': case 'patch': method = flags[i].toUpperCase(); - !def && (def = headers['Content-Type'] = 'application/x-www-form-urlencoded'); + !def && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); break; case 'dnscache': From d0ac4e60255465bd1d2975a349f30339d2803d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 15:18:23 +0200 Subject: [PATCH 0139/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bf8b720f4..e4ade03dc 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-22", + "version": "3.0.0-23", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From c826ebebea16af57be927d87d915dc3e66629964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 19:36:22 +0200 Subject: [PATCH 0140/1669] Fixed flags in `U.request()`. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 3d2f1ce91..fe136b291 100755 --- a/utils.js +++ b/utils.js @@ -521,7 +521,7 @@ exports.request = function(url, flags, data, callback, cookies, headers, encodin case 'delete': case 'patch': method = flags[i].toUpperCase(); - !def && (headers['Content-Type'] = 'application/x-www-form-urlencoded'); + !def && !headers[CT] && (headers[CT] = 'application/x-www-form-urlencoded'); break; case 'dnscache': From 2a0386c280f5a1b718051c406378f8be0d7ed812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 20:23:31 +0200 Subject: [PATCH 0141/1669] Added clearing of NoSQL storage. --- nosql.js | 59 ++++++++++++++++++++++++++++++++++++++++---------- nosqlworker.js | 9 ++++++++ 2 files changed, 57 insertions(+), 11 deletions(-) mode change 100644 => 100755 nosqlworker.js diff --git a/nosql.js b/nosql.js index 57a6a5c41..3e7768b0c 100755 --- a/nosql.js +++ b/nosql.js @@ -109,22 +109,17 @@ exports.worker = function() { break; case 'backup': case 'restore': - var obj = FORKCALLBACKS[msg.id]; - obj && obj.callback(msg.err, msg.response); - break; case 'counter.read': case 'counter.stats': case 'counter.clear': + case 'storage.stats': + case 'storage.clear': var obj = FORKCALLBACKS[msg.id]; - obj && obj.callback(msg.err, msg.response); + obj && obj.callback && obj.callback(msg.err, msg.response); break; case 'storage.scan': var obj = FORKCALLBACKS[msg.id]; - obj && obj.callback(msg.err, msg.response, msg.repository); - break; - case 'storage.stats': - var obj = FORKCALLBACKS[msg.id]; - obj && obj.callback(msg.err, msg.response); + obj && obj.callback && obj.callback(msg.err, msg.response, msg.repository); break; } delete FORKCALLBACKS[msg.id]; @@ -346,6 +341,12 @@ exports.worker = function() { send(this.db, 'storage.stats', name).callback = callback; return this; }; + + Storage.prototype.clear = function(callback) { + send(this.db, 'storage.clear').callback = callback; + return this; + }; + }; function Database(name, filename) { @@ -4147,7 +4148,8 @@ Storage.prototype.insert = function(doc) { return self; } - Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function() { + Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function(err) { + err && F.error(err, 'NoSQL.storage'); self.locked_reader = false; }); @@ -4205,6 +4207,39 @@ Storage.prototype.$mapreducesave = function() { return self; }; +Storage.prototype.clear = function(callback) { + var self = this; + U.ls(self.db.directory, function(files) { + + var count = 0; + + var remove = function(filename, callback, attemp) { + Fs.unlink(filename, function(err) { + if (err) { + if (err.toString().indexOf('no such file') === -1) { + if (attemp > 5) + callback(); + else + setTimeout(() => remove(filename, callback, (attemp || 0) + 1), 100); + } else + callback(); + } else { + count++; + callback(); + } + }); + }; + + files.wait((filename, next) => remove(filename, next), function() { + remove(self.$mapreducefile, function() { + callback(null, count); + }); + }); + + }, (path, is) => is ? false : path.substring(self.db.directory.length + 1).startsWith(self.db.name + '-') && path.endsWith(EXTENSION_STORAGE)); + return self; +}; + Storage.prototype.scan = function(beg, end, mapreduce, callback) { if (typeof(beg) === 'function') { @@ -4295,7 +4330,8 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { stats.current = item.date; stats.index = index; - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + reader.on('error', finish); + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') return; @@ -4353,6 +4389,7 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { }); }, (path, is) => is ? false : path.substring(self.db.directory.length + 1).startsWith(self.db.name + '-') && path.endsWith(EXTENSION_STORAGE)); + return self; } function Backuper(filename) { diff --git a/nosqlworker.js b/nosqlworker.js old mode 100644 new mode 100755 index 81a14ae5d..0c0ca648c --- a/nosqlworker.js +++ b/nosqlworker.js @@ -12,6 +12,7 @@ const RESCOUNTERSTATS = { TYPE: 'counter.stats' }; const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; const RESTORAGESCAN = { TYPE: 'storage.scan' }; const RESTORAGESTATS = { TYPE: 'storage.stats' }; +const RESSTORAGECLEAR = { TYPE: 'storage.clear' }; function killprocess() { process.exit(0); @@ -174,5 +175,13 @@ process.on('message', function(msg) { process.send(RESTORAGESCAN); }); break; + case 'storage.clear': + db.storage.clear(function(err, response) { + RESSTORAGECLEAR.id = msg.id; + RESSTORAGECLEAR.response = response; + RESSTORAGECLEAR.err = err; + process.send(RESSTORAGECLEAR); + }); + break; } }); \ No newline at end of file From 397a85740ef935dddfeee02d443483e5a3802c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 20:23:46 +0200 Subject: [PATCH 0142/1669] Fixed path in Windows in `F.backup()`. --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) mode change 100644 => 100755 index.js diff --git a/index.js b/index.js old mode 100644 new mode 100755 index 4e8c491c6..48215c7da --- a/index.js +++ b/index.js @@ -5954,10 +5954,14 @@ F.backup = function(filename, filelist, callback, filter) { filelist.wait(function(item, next) { + var file = Path.join(path, item); + + if (F.isWindows) + item = item.replace(/\\/g, '/'); + if (item[0] !== '/') item = '/' + item; - var file = Path.join(path, item); Fs.stat(file, function(err, stats) { if (err) { From ac18e8eac83e496ca1f49636e9fe828a000035e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 20:25:59 +0200 Subject: [PATCH 0143/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e4ade03dc..2790d82a9 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-23", + "version": "3.0.0-24", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9c70bde8a213d3319b52f2730641cfcf84f8043b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 22:30:29 +0200 Subject: [PATCH 0144/1669] Fixed bug with listener. --- nosql.js | 3 ++- package.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 3e7768b0c..12b63a022 100755 --- a/nosql.js +++ b/nosql.js @@ -4330,7 +4330,6 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { stats.current = item.date; stats.index = index; - reader.on('error', finish); reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { if (value[0] !== '{') @@ -4360,6 +4359,8 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { setImmediate(next); }; + reader.on('error', finish); + if (reader) CLEANUP(reader, finish); else diff --git a/package.json b/package.json index 2790d82a9..694794323 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-24", + "version": "3.0.0-25", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 5389863a6b8775961f64cce09e04ef490db89917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 23:20:07 +0200 Subject: [PATCH 0145/1669] Improved debugging mode. --- debug.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/debug.js b/debug.js index 63d8be4ee..cfe6e4554 100644 --- a/debug.js +++ b/debug.js @@ -226,6 +226,12 @@ function runwatching() { dir = fn.substring(index, fn.indexOf('/', index + 1) + 1); } + if (dir !== F.config['components']) { + var ext = fn.substring(fn.lastIndexOf('.') + 1); + if (ext && ext !== 'js' && ext !== 'package' && ext !== 'bundle' && ext !== 'resource') + return fn; + } + return F.config['directory-views'] === dir || F.config['directory-public'] === dir ? fn : ''; } @@ -443,4 +449,4 @@ function normalize(path) { if (debugging) setImmediate(runapp); else - setImmediate(runwatching); + setImmediate(runwatching); \ No newline at end of file From 83512b9e044de1c0687f571be9432a571fffa6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 23:20:17 +0200 Subject: [PATCH 0146/1669] Added new global variable: `GUID()`. --- changes.txt | 1 + utils.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index ef7f80df7..7bb6bd38a 100755 --- a/changes.txt +++ b/changes.txt @@ -28,6 +28,7 @@ - added: config `nosql-worker' for enabling worker for NoSQL embedded database (default: `false`) - added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` +- added: `GUID()` a global alias for `U.GUID()` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` diff --git a/utils.js b/utils.js index fe136b291..b61e83dd1 100755 --- a/utils.js +++ b/utils.js @@ -1963,7 +1963,7 @@ function rnd() { return Math.floor(Math.random() * 65536).toString(36); } -exports.GUID = function(max) { +global.GUID = exports.GUID = function(max) { max = max || 40; var str = ''; for (var i = 0; i < (max / 3) + 1; i++) From 227f479a17809812d83d5921500742794e2f6180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 26 Mar 2018 23:20:21 +0200 Subject: [PATCH 0147/1669] New beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 694794323..22eedebb6 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-25", + "version": "3.0.0-26", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 6bf3d3f2d7c121abb4da607e2900489d07394600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 27 Mar 2018 09:31:18 +0200 Subject: [PATCH 0148/1669] Improved NoSQL storage. --- nosql.js | 210 +++++++++++++++++++++++++++++++------------------ nosqlworker.js | 2 +- 2 files changed, 135 insertions(+), 77 deletions(-) diff --git a/nosql.js b/nosql.js index 12b63a022..d5017a4ee 100755 --- a/nosql.js +++ b/nosql.js @@ -45,7 +45,6 @@ const EXTENSION = '.nosql'; const EXTENSION_BINARY = '.nosql-binary'; const EXTENSION_TMP = '.nosql-tmp'; const EXTENSION_LOG = '.nosql-log'; -const EXTENSION_STORAGE = '.nosql-storage'; const EXTENSION_MAPREDUCE = '.nosql-mapreduce'; const EXTENSION_BACKUP = '.nosql-backup'; const EXTENSION_META = '.meta'; @@ -342,8 +341,8 @@ exports.worker = function() { return this; }; - Storage.prototype.clear = function(callback) { - send(this.db, 'storage.clear').callback = callback; + Storage.prototype.clear = function(beg, end, callback) { + send(this.db, 'storage.clear', beg, end).callback = callback; return this; }; @@ -358,7 +357,7 @@ function Database(name, filename) { self.filenameTemp = filename + EXTENSION_TMP; self.filenameLog = self.readonly ? '' : filename + EXTENSION_LOG; self.filenameBackup = self.readonly ? '' : filename + EXTENSION_BACKUP; - self.filenameStorage = self.readonly ? '' : filename + '-{0}' + EXTENSION_STORAGE; + self.filenameStorage = self.readonly ? '' : filename + '-storage/' + name + '-{0}' + EXTENSION; self.filenameMeta = filename + EXTENSION_META; self.directory = Path.dirname(filename); self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + EXTENSION); @@ -373,8 +372,8 @@ function Database(name, filename) { self.pending_drops = false; self.pending_views = false; self.binary = self.readonly ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); + self.storage = self.readonly ? null : new Storage(self, self.directory + '/' + self.name + '-storage/'); self.counter = new Counter(self); - self.storage = new Storage(self); self.inmemory = {}; self.inmemorylastusage; self.metadata; @@ -585,7 +584,13 @@ Database.prototype.restore = function(filename, callback) { F.restore(filename, F.path.root(), function(err, response) { self.type = 0; - !err && self.refresh(); + + if (!err) { + self.$meta(); + self.refresh(); + self.storage && self.storage.refresh(); + } + callback && callback(err, response); }); @@ -644,11 +649,10 @@ Database.prototype.backup = function(filename, callback) { }); pending.push(function(next) { - U.ls(self.directory, function(files) { - for (var i = 0, length = files.length; i < length; i++) - list.push(Path.join(F.config['directory-databases'], files[i].substring(self.directory.length))); + F.path.exists(F.path.databases(self.name + '-storage'), function(e, size, file) { + e && !file && list.push(Path.join(F.config['directory-databases'], self.name + '-storage')); next(); - }, (path, is) => is ? false : path.substring(self.directory.length + 1).startsWith(self.name + '-') && path.endsWith(EXTENSION_STORAGE)); + }); }); pending.push(function(next) { @@ -912,7 +916,7 @@ Database.prototype.$save = function(view) { if (view !== '#') filename = filename.replace(/\.nosql/, '#' + view + '.nosql'); - Fs.writeFile(filename, builder.join(NEWLINE) + NEWLINE, NOOP); + Fs.writeFile(filename, builder.join(NEWLINE) + NEWLINE, F.error()); }, 50, 100); return self; }; @@ -960,7 +964,7 @@ Database.prototype.$meta = function(write) { if (write) { self.readonly && self.throwReadonly(); - Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), NOOP); + Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), F.error()); return self; } @@ -990,6 +994,9 @@ Database.prototype.$append = function() { json.push(items[i].doc); Fs.appendFile(self.filename, json.join(NEWLINE) + NEWLINE, function(err) { + + err && F.error(err, 'NoSQL insert: ' + self.name); + for (var i = 0, length = items.length; i < length; i++) { items[i].builder.$options.log && items[i].builder.log(); var callback = items[i].builder.$callback; @@ -1305,7 +1312,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { if (self.readonly) { if (reader === undefined) { U.download(filename, FLAGS_READ, function(err, response) { - err && F.error(err, 'NoSQL database: ' + self.name); + err && F.error(err, 'NoSQL database download: ' + self.name); self.$reader2(filename, items, callback, err ? null : response); }); return self; @@ -2044,7 +2051,7 @@ DatabaseBuilder2.prototype.log = function(msg, user) { F.datetime = new Date(); self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { - self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, NOOP); + self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.error()); self.$options.log = ''; } return self; @@ -2101,7 +2108,7 @@ DatabaseBuilder.prototype.log = function(msg, user) { F.datetime = new Date(); self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { - self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, NOOP); + self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.error()); self.$options.log = ''; } return self; @@ -2325,7 +2332,7 @@ DatabaseBuilder.prototype.backup = function(user) { }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, NOOP); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, F.error()); return this; }; @@ -4105,18 +4112,40 @@ Binary.prototype.all = function(callback) { function Storage(db, directory) { this.db = db; + this.directory = directory; this.pending = []; this.locked_writer = 0; this.locked_reader = false; + this.exists = false; if (!FORK) { this.$mapreducefile = Path.join(db.directory, db.name + EXTENSION_MAPREDUCE); this.$mapreduce = []; - try { - this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); - } catch (e) {} + this.refresh(); } } +Storage.prototype.refresh = function() { + try { + this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); + } catch (e) {} + return this; +}; + +Storage.prototype.check = function() { + + var self = this; + if (self.exists) + return self; + + self.exists = true; + + try { + Fs.mkdirSync(self.directory); + } catch (err) {} + + return self; +}; + Storage.prototype.insert = function(doc) { var self = this; @@ -4125,7 +4154,9 @@ Storage.prototype.insert = function(doc) { if (self.pending.length) { var dt = F.datetime.format('yyyyMMdd'); self.locked_reader = true; - Fs.appendFile(self.db.filenameStorage.format(dt), self.pending.join(NEWLINE) + NEWLINE, function() { + self.check(); + Fs.appendFile(self.db.filenameStorage.format(dt), self.pending.join(NEWLINE) + NEWLINE, function(err) { + err && F.error(err, 'NoSQL storage insert: ' + self.db.name); self.locked_reader = false; }); self.pending = []; @@ -4148,8 +4179,9 @@ Storage.prototype.insert = function(doc) { return self; } + self.check(); Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function(err) { - err && F.error(err, 'NoSQL.storage'); + err && F.error(err, 'NoSQL storage insert: ' + self.db.name); self.locked_reader = false; }); @@ -4196,7 +4228,7 @@ Storage.prototype.mapreduce = function(name, fn) { item.repository = repository; item.ready = true; self.$mapreducesave(); - }); + }, true); return self; }; @@ -4207,54 +4239,9 @@ Storage.prototype.$mapreducesave = function() { return self; }; -Storage.prototype.clear = function(callback) { - var self = this; - U.ls(self.db.directory, function(files) { - - var count = 0; - - var remove = function(filename, callback, attemp) { - Fs.unlink(filename, function(err) { - if (err) { - if (err.toString().indexOf('no such file') === -1) { - if (attemp > 5) - callback(); - else - setTimeout(() => remove(filename, callback, (attemp || 0) + 1), 100); - } else - callback(); - } else { - count++; - callback(); - } - }); - }; - - files.wait((filename, next) => remove(filename, next), function() { - remove(self.$mapreducefile, function() { - callback(null, count); - }); - }); - - }, (path, is) => is ? false : path.substring(self.db.directory.length + 1).startsWith(self.db.name + '-') && path.endsWith(EXTENSION_STORAGE)); - return self; -}; - -Storage.prototype.scan = function(beg, end, mapreduce, callback) { - - if (typeof(beg) === 'function') { - mapreduce = beg; - callback = end; - beg = null; - end = null; - } else if (typeof(end) === 'function') { - callback = mapreduce; - mapreduce = end; - end = null; - } +Storage.prototype.listing = function(beg, end, callback) { var tmp; - if (beg) { if (typeof(beg) === 'string') { @@ -4293,23 +4280,50 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { var self = this; - U.ls(self.db.directory, function(files) { - + U.ls(self.directory, function(files) { var storage = []; - var repository = {}; - var stats = {}; - for (var i = 0, length = files.length; i < length; i++) { var item = files[i]; - var skip = item.length - EXTENSION_STORAGE.length; + var skip = item.length - EXTENSION.length; var date = +item.substring(skip - 8, skip); if ((beg && beg > date) || (end && end < date)) continue; storage.push({ filename: item, date: date }); } + callback(null, storage); + }, (path, is) => is ? false : path.endsWith(EXTENSION)); + + return self; +}; + +Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { + var self = this; + + if (typeof(beg) === 'function') { + reverse = mapreduce; + mapreduce = beg; + callback = end; + beg = null; + end = null; + } else if (typeof(end) === 'function') { + reverse = callback; + callback = mapreduce; + mapreduce = end; + end = null; + } + + if (typeof(callback) === 'boolean') { + reverse = callback; + callback = null; + } + + self.listing(beg, end, function(err, storage) { + + var repository = {}; + var stats = {}; // Desc - storage.quicksort('date', false); + storage.quicksort('date', reverse == true); stats.files = storage.length; stats.documents = 0; @@ -4389,10 +4403,54 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback) { callback && callback(null, repository, stats); }); - }, (path, is) => is ? false : path.substring(self.db.directory.length + 1).startsWith(self.db.name + '-') && path.endsWith(EXTENSION_STORAGE)); + }); + return self; } +Storage.prototype.clear = function(beg, end, callback) { + var self = this; + + if (typeof(beg) === 'function') { + mapreduce = beg; + callback = end; + beg = null; + end = null; + } else if (typeof(end) === 'function') { + callback = mapreduce; + mapreduce = end; + end = null; + } + + self.listing(beg, end, function(err, files) { + + var count = 0; + + var remove = function(filename, callback, attemp) { + Fs.unlink(filename, function(err) { + if (err) { + if (err.toString().indexOf('no such file') === -1) { + if (attemp > 5) + callback(); + else + setTimeout(() => remove(filename, callback, (attemp || 0) + 1), 100); + } else + callback(); + } else { + count++; + callback(); + } + }); + }; + + files.wait((filename, next) => remove(filename, next), function() { + remove(self.$mapreducefile, () => callback && callback(null, count)); + }); + + }); + return self; +}; + function Backuper(filename) { this.filename = filename; this.items = []; diff --git a/nosqlworker.js b/nosqlworker.js index 0c0ca648c..da6ba8a90 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -176,7 +176,7 @@ process.on('message', function(msg) { }); break; case 'storage.clear': - db.storage.clear(function(err, response) { + db.storage.clear(msg.arg[0], msg.arg[1], function(err, response) { RESSTORAGECLEAR.id = msg.id; RESSTORAGECLEAR.response = response; RESSTORAGECLEAR.err = err; From 75b783f2c5bd3feb41de2407ed6a2eb01040d9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 27 Mar 2018 09:31:44 +0200 Subject: [PATCH 0149/1669] Update beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 22eedebb6..ac40ce868 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-26", + "version": "3.0.0-27", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From c636b48db9c3efe52c2d1e9c51891536701291a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 27 Mar 2018 14:15:54 +0200 Subject: [PATCH 0150/1669] Fixed Total.js `proxy`. --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 48215c7da..5198fc829 100755 --- a/index.js +++ b/index.js @@ -7068,15 +7068,15 @@ F.$requestcontinue = function(req, res, headers) { req.buffer_has = false; req.$flags = req.method[0] + req.method[1]; + var flags = [req.method.toLowerCase()]; + var multipart; + if (headers['x-proxy'] === 'total.js') { req.isProxy = true; req.$flags += 'f'; flags.push('proxy'); } - var flags = [req.method.toLowerCase()]; - var multipart; - if (F._request_check_mobile && req.mobile) { req.$flags += 'a'; F.stats.request.mobile++; From 23bcfb6a4dd5f6d8ae6743441955a6981434e4da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 27 Mar 2018 17:12:38 +0200 Subject: [PATCH 0151/1669] Added `SchemaBuilderEntity.$response()` method. --- builders.js | 4 ++++ changes.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/builders.js b/builders.js index 72ec5acef..dce558bf1 100755 --- a/builders.js +++ b/builders.js @@ -2486,6 +2486,10 @@ function async_continue(self) { }); } +SchemaInstance.prototype.$response = function(index) { + return this.$$result[index == null ? this.$$index : index]; +}; + SchemaInstance.prototype.$repository = function(name, value) { if (this.$$repository === undefined) { diff --git a/changes.txt b/changes.txt index 7bb6bd38a..cd5ec9f48 100755 --- a/changes.txt +++ b/changes.txt @@ -29,6 +29,7 @@ - added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `GUID()` a global alias for `U.GUID()` +- added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` From 3d40d63b1ac6d8f712b0da34453cd871e9068d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 27 Mar 2018 17:12:48 +0200 Subject: [PATCH 0152/1669] Fixed routing with multiple flags. --- index.js | 2 ++ package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 5198fc829..3fe8c017f 100755 --- a/index.js +++ b/index.js @@ -1900,6 +1900,8 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { tmp.push(flag); method += (method ? ',' : '') + flag; corsflags.push(flag); + PERF[flag.toUpperCase()] = true; + PERF[flag] = true; break; default: if (flag[0] === '@') diff --git a/package.json b/package.json index ac40ce868..fb5bdef22 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-27", + "version": "3.0.0-28", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From e7f789e6b8216d4af91b4b5a75cc7ae7cb3f6f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Mar 2018 08:58:18 +0200 Subject: [PATCH 0153/1669] Fixed `storage.stats()` and `storage.clear()` in worker. --- nosql.js | 18 ++++++++++++++++++ package.json | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index d5017a4ee..3cedfe030 100755 --- a/nosql.js +++ b/nosql.js @@ -337,11 +337,29 @@ exports.worker = function() { }; Storage.prototype.stats = function(name, callback) { + + if (typeof(name) === 'function') { + callback = name; + name = undefined; + } + send(this.db, 'storage.stats', name).callback = callback; return this; }; Storage.prototype.clear = function(beg, end, callback) { + + if (typeof(beg) === 'function') { + mapreduce = beg; + callback = end; + beg = null; + end = null; + } else if (typeof(end) === 'function') { + callback = mapreduce; + mapreduce = end; + end = null; + } + send(this.db, 'storage.clear', beg, end).callback = callback; return this; }; diff --git a/package.json b/package.json index fb5bdef22..f86898715 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-28", + "version": "3.0.0-29", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From db02b3d3ed9ee86d78359fa4acd2859961f80e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Mar 2018 10:11:58 +0200 Subject: [PATCH 0154/1669] Fixed `String.arg()`. --- changes.txt | 1 + test/test-utils.js | 2 ++ utils.js | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index cd5ec9f48..c9644c6e3 100755 --- a/changes.txt +++ b/changes.txt @@ -55,6 +55,7 @@ - fixed: `U.atob()` - fixed: `U.btoa()` - fixed: schema field can be changed dynamically +- fixed: `String.arg()` - improved: `debug` mode timing with improved consumption - improved: NoSQL embedded database diff --git a/test/test-utils.js b/test/test-utils.js index 1e7d2b821..3f602fe2c 100755 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -266,6 +266,8 @@ function prototypeString() { assert.ok('á'.localeCompare2('a') === 1, 'localeCompare2 - 1'); assert.ok('á'.localeCompare2('b') === -1, 'localeCompare2 - 2'); assert.ok('č'.localeCompare2('b') === 1, 'localeCompare2 - 3'); + + assert.ok('Hello {{ what }}!'.arg({ what: 'world' }) === 'Hello world!', 'String.arg()'); } function prototypeArray() { diff --git a/utils.js b/utils.js index b61e83dd1..0667823d2 100755 --- a/utils.js +++ b/utils.js @@ -54,7 +54,7 @@ const regexpDiacritics = /[^\u0000-\u007e]/g; const regexpXML = /\w+=".*?"/g; const regexpDECODE = /&#?[a-z0-9]+;/g; const regexpPARAM = /\{{2}[^}\n]*\}{2}/g; -const regexpARG = /\{[^}\n]*\}/g; +const regexpARG = /\{{2}.*?\}{2}/g; const regexpINTEGER = /(^-|\s-)?[0-9]+/g; const regexpFLOAT = /(^-|\s-)?[0-9.,]+/g; const regexpALPHA = /^[A-Za-z0-9]+$/; @@ -3273,7 +3273,7 @@ String.prototype.urlDecode = function() { String.prototype.arg = function(obj) { return this.replace(regexpARG, function(text) { - var val = obj[text.substring(1, text.length - 1).trim()]; + var val = obj[text.substring(2, text.length - 2).trim()]; return val == null ? text : val; }); }; From 8817cc75439f4f97fe8ba23c6ffed5362beab072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Mar 2018 10:12:15 +0200 Subject: [PATCH 0155/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f86898715..f9f9a155b 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-29", + "version": "3.0.0-30", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 87345373e95d83c77c39963b9a38d05f2ced4fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Mar 2018 10:55:14 +0200 Subject: [PATCH 0156/1669] Fixed `controller.href()`. --- changes.txt | 1 + index.js | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index c9644c6e3..24ce924b2 100755 --- a/changes.txt +++ b/changes.txt @@ -56,6 +56,7 @@ - fixed: `U.btoa()` - fixed: schema field can be changed dynamically - fixed: `String.arg()` +- fixed: `controller.href()` with Array values - improved: `debug` mode timing with improved consumption - improved: NoSQL embedded database diff --git a/index.js b/index.js index 3fe8c017f..d699de2b6 100755 --- a/index.js +++ b/index.js @@ -10814,7 +10814,15 @@ Controller.prototype.$helper = function() { return this.helper.apply(this, arguments); }; -function querystring_encode(value, def) { +function querystring_encode(value, def, key) { + + if (value instanceof Array) { + var tmp = ''; + for (var i = 1; i < value.length; i++) + tmp += (tmp ? '&' : '') + key + '=' + querystring_encode(value[i], def); + return querystring_encode(value[0], def) + (tmp ? tmp : ''); + } + return value != null ? value instanceof Date ? encodeURIComponent(value.format()) : typeof(value) === 'string' ? encodeURIComponent(value) : value.toString() : def || ''; } @@ -10839,6 +10847,7 @@ Controller.prototype.href = function(key, value) { if (!str) { obj = U.copy(self.query); + for (var i = 2; i < arguments.length; i++) obj[arguments[i]] = undefined; @@ -10847,13 +10856,18 @@ Controller.prototype.href = function(key, value) { var arr = Object.keys(obj); for (var i = 0, length = arr.length; i < length; i++) { var val = obj[arr[i]]; - if (val !== undefined) - str += (str ? '&' : '') + arr[i] + '=' + (key === arr[i] ? '\0' : querystring_encode(val)); + if (val !== undefined) { + if (val instanceof Array) { + for (var j = 0; j < val.length; j++) + str += (str ? '&' : '') + arr[i] + '=' + (key === arr[i] ? '\0' : querystring_encode(val[j])); + } else + str += (str ? '&' : '') + arr[i] + '=' + (key === arr[i] ? '\0' : querystring_encode(val)); + } } self[cachekey] = str; } - str = str.replace('\0', querystring_encode(value, self.query[key])); + str = str.replace('\0', querystring_encode(value, self.query[key], key)); for (var i = 2; i < arguments.length; i++) { var beg = str.indexOf(arguments[i] + '='); From bc9043210a90e0cac01cf420a133ca5ac1932eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Mar 2018 10:56:45 +0200 Subject: [PATCH 0157/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f9f9a155b..0174abbc6 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-30", + "version": "3.0.0-31", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 5b3c5e9d1cfc354f0e8a7e86caaa4ed83915d642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Mar 2018 21:29:34 +0200 Subject: [PATCH 0158/1669] Fixed start arguments. --- debug.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/debug.js b/debug.js index cfe6e4554..ef710f607 100644 --- a/debug.js +++ b/debug.js @@ -90,6 +90,7 @@ function runwatching() { const REG_RELOAD = /\.(js|css|html|htm|jpg|png|gif|ico|svg|resource)$/i; const isRELOAD = !!options.livereload; const SPEED = isRELOAD ? 1000 : 1500; + const ARGV = CLONE(process.argv); function copyFile(oldname, newname, callback) { var writer = Fs.createWriteStream(newname); @@ -226,12 +227,6 @@ function runwatching() { dir = fn.substring(index, fn.indexOf('/', index + 1) + 1); } - if (dir !== F.config['components']) { - var ext = fn.substring(fn.lastIndexOf('.') + 1); - if (ext && ext !== 'js' && ext !== 'package' && ext !== 'bundle' && ext !== 'resource') - return fn; - } - return F.config['directory-views'] === dir || F.config['directory-public'] === dir ? fn : ''; } @@ -327,7 +322,8 @@ function runwatching() { app = null; } - var arr = process.argv; + var arr = ARGV.slice(2); + var port = arr.pop(); if (process.execArgv.indexOf('--debug') !== -1 || options.debugger) { @@ -346,7 +342,7 @@ function runwatching() { arr.push('restart'); arr.push('debugging'); - arr.push(port); + port && arr.push(port); app = fork(Path.join(directory, FILENAME), arr); @@ -449,4 +445,4 @@ function normalize(path) { if (debugging) setImmediate(runapp); else - setImmediate(runwatching); \ No newline at end of file + setImmediate(runwatching); From d5bd24d2ab70039ba99b5c6c20867d084fbd5046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Mar 2018 07:49:37 +0200 Subject: [PATCH 0159/1669] Added missing `counter.sum()` method. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 3cedfe030..1530b250e 100755 --- a/nosql.js +++ b/nosql.js @@ -272,7 +272,7 @@ exports.worker = function() { return this; }; - Counter.prototype.inc = Counter.prototype.hit = function(id, count) { + Counter.prototype.sum = Counter.prototype.inc = Counter.prototype.hit = function(id, count) { notify(this.db, 'counter.hit', id, count); return this; }; From cdf58df64dbc8a8b8157a149f0f3442b37ff9ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Mar 2018 10:25:11 +0200 Subject: [PATCH 0160/1669] Added new aliases for schemas `$SAVE()`, `$INSERT()`, `$UPDATE()` and `$REMOVE()`. --- builders.js | 12 +++++++++++- changes.txt | 5 +++++ index.js | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index dce558bf1..5b95689a9 100755 --- a/builders.js +++ b/builders.js @@ -111,6 +111,10 @@ SchemaOptions.prototype.stop = function() { return this.model.$stop(); }; +SchemaOptions.prototype.response = function(index) { + return this.model.$response(index); +}; + SchemaOptions.prototype.DB = function() { return F.database(this.error); }; @@ -1075,6 +1079,9 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, break; } + if (!self[TYPE]) + return callback(new Error('Operation "{0}/{1}" not found'.format(self.name, $type))); + self.$prepare(model, function(err, model) { if (err) { @@ -1231,6 +1238,9 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { var builder = new ErrorBuilder(); var $type = 'remove'; + if (!self.onRemove) + return callback(new Error('Operation "{0}/{1}" not found'.format(self.name, $type))); + self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2487,7 +2497,7 @@ function async_continue(self) { } SchemaInstance.prototype.$response = function(index) { - return this.$$result[index == null ? this.$$index : index]; + return this.$$index == undefined ? this.$$result : this.$$result[index === 'prev' ? this.$$index : index]; }; SchemaInstance.prototype.$repository = function(name, value) { diff --git a/changes.txt b/changes.txt index 24ce924b2..315408c4c 100755 --- a/changes.txt +++ b/changes.txt @@ -18,6 +18,7 @@ - added: `SchemaOptions.next()` alias to `$.model.$next()` - added: `SchemaOptions.output()` alias to `$.model.$output()` - added: `SchemaOptions.clean()` alias to `$.model.$clean()` +- added: `SchemaOptions.response()` alias to `$.model.$response([index])` - added: a new route flag type `&group` something like `roles` but groups aren't evaluated - added: `route.groups` with defined groups - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode @@ -30,6 +31,10 @@ - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `GUID()` a global alias for `U.GUID()` - added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue +- added: `$SAVE(schema, model, [options], [callback], [controller])` performs `schema.save()` +- added: `$INSERT(schema, model, [options], [callback], [controller])` performs `schema.insert()` +- added: `$UPDATE(schema, model, [options], [callback], [controller])` performs `schema.update()` +- added: `$REMOVE(schema, [options], [callback], [controller])` performs `schema.remove()` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` diff --git a/index.js b/index.js index d699de2b6..dc563062f 100755 --- a/index.js +++ b/index.js @@ -359,6 +359,45 @@ global.$TRANSFORM = function(schema, name, options, callback, controller) { return !!o; }; +global.$REMOVE = function(schema, options, callback, controller) { + schema = parseSchema(schema); + var o = framework_builders.getschema(schema[0], schema[1]); + o && o.remove(options, callback, controller); + return !!o; +}; + +global.$SAVE = function(schema, model, options, callback, controller) { + return performschema('$save', schema, model, options, callback, controller); +}; + +global.$INSERT = function(schema, model, options, callback, controller) { + return performschema('$insert', schema, model, options, callback, controller); +}; + +global.$UPDATE = function(schema, model, options, callback, controller) { + return performschema('$update', schema, model, options, callback, controller); +}; + +function performschema(type, schema, model, options, callback, controller) { + + if (typeof(options) === 'function') { + controller = callback; + callback = options; + options = null; + } + + schema = parseSchema(schema); + var o = framework_builders.getschema(schema[0], schema[1]); + o.make(model, function(err, model) { + if (err) + callback(err); + else + model[type](options, callback); + }); + + return !!o; +} + global.$ASYNC = function(schema, callback, index, controller) { if (index && typeof(index) === 'object') { From 7ea08df3a55efd7c5c5811b53b890de67848653a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Mar 2018 10:26:39 +0200 Subject: [PATCH 0161/1669] Updated beta versoin. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0174abbc6..fba14402a 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-31", + "version": "3.0.0-32", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From c32d5a0ffb983991a3616d7f30e3b6d50450701f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Mar 2018 10:35:17 +0200 Subject: [PATCH 0162/1669] Updated NoSQL embedded storage file names. --- nosql.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 1530b250e..a24d05bd3 100755 --- a/nosql.js +++ b/nosql.js @@ -375,7 +375,7 @@ function Database(name, filename) { self.filenameTemp = filename + EXTENSION_TMP; self.filenameLog = self.readonly ? '' : filename + EXTENSION_LOG; self.filenameBackup = self.readonly ? '' : filename + EXTENSION_BACKUP; - self.filenameStorage = self.readonly ? '' : filename + '-storage/' + name + '-{0}' + EXTENSION; + self.filenameStorage = self.readonly ? '' : filename + '-storage/{0}' + EXTENSION; self.filenameMeta = filename + EXTENSION_META; self.directory = Path.dirname(filename); self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + EXTENSION); diff --git a/package.json b/package.json index fba14402a..6c582976a 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-32", + "version": "3.0.0-33", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 018e0e72459d192c94bd7fb27e9ca63c6161c81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Mar 2018 12:23:06 +0200 Subject: [PATCH 0163/1669] Fixed paths in Windows. --- debug.js | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debug.js b/debug.js index ef710f607..0b5f4fa20 100644 --- a/debug.js +++ b/debug.js @@ -185,6 +185,7 @@ function runwatching() { return isDirectory ? SRC !== path : !blacklist[path.substring(directory.length)]; if (isRELOAD) return isDirectory ? true : REG_RELOAD.test(path); + path = normalize(path); return isDirectory && REG_THEMES.test(path) ? REG_THEMES_INDEX.test(path) : isDirectory ? true : REG_EXTENSION.test(path) || REG_COMPONENTS.test(path) || REG_CONFIGS.test(path); } diff --git a/package.json b/package.json index 6c582976a..8da139e94 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-33", + "version": "3.0.0-34", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From e99e04dd23f601908cf1aa561397dbeba33d645f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 1 Apr 2018 10:19:10 +0200 Subject: [PATCH 0164/1669] Fixed empty arguments. --- debug.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug.js b/debug.js index 0b5f4fa20..3ba77a123 100644 --- a/debug.js +++ b/debug.js @@ -78,7 +78,7 @@ function runwatching() { !options && (options = {}); require('./index'); - const FILENAME = U.getName(process.argv[1]); + const FILENAME = U.getName(process.argv[1] || 'debug.js'); const directory = process.cwd(); const VERSION = F.version_header; const REG_CONFIGS = /configs\//g; From 9c89328deccb55b5e8f6be22371505a0d2aba38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 1 Apr 2018 21:03:50 +0200 Subject: [PATCH 0165/1669] Improved code. --- debug.js | 4 ++-- index.js | 8 +++++--- nosql.js | 27 +++++++++++++-------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/debug.js b/debug.js index 3ba77a123..a7dc9c065 100644 --- a/debug.js +++ b/debug.js @@ -68,9 +68,9 @@ function runapp() { F.http('debug', options); if (first) - F.emit('debug-start'); + EMIT('debug-start'); else - F.emit('debug-restart'); + EMIT('debug-restart'); } function runwatching() { diff --git a/index.js b/index.js index dc563062f..55e65858b 100755 --- a/index.js +++ b/index.js @@ -378,7 +378,8 @@ global.$UPDATE = function(schema, model, options, callback, controller) { return performschema('$update', schema, model, options, callback, controller); }; -function performschema(type, schema, model, options, callback, controller) { +// type, schema, model, options, callback, controller +function performschema(type, schema, model, options, callback) { if (typeof(options) === 'function') { controller = callback; @@ -5463,7 +5464,8 @@ F.usage = function(detailed) { * @param {Object} model * @return {String} */ -F.onCompileView = function(name, html, model) { +// name, html, model +F.onCompileView = function(name, html) { return html; }; @@ -16297,4 +16299,4 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file +global.EMPTYCONTROLLER = EMPTYCONTROLLER; diff --git a/nosql.js b/nosql.js index a24d05bd3..b5634be05 100755 --- a/nosql.js +++ b/nosql.js @@ -350,13 +350,11 @@ exports.worker = function() { Storage.prototype.clear = function(beg, end, callback) { if (typeof(beg) === 'function') { - mapreduce = beg; callback = end; beg = null; end = null; } else if (typeof(end) === 'function') { - callback = mapreduce; - mapreduce = end; + callback = end; end = null; } @@ -2299,15 +2297,16 @@ DatabaseBuilder.prototype.make = function(fn) { DatabaseBuilder.prototype.filter = function(fn) { var self = this; - var opt = self.$options; - var key = 'fil' + (self.$counter++); + if (!self.$functions) self.$functions = []; - var index = self.$functions.push(fn) - 1; + var index = self.$functions.push(fn) - 1; var code = '$is=!!fn[{0}].call($F,doc,index,repository);'.format(index); + if (self.$scope) code = 'if(!$is){' + code + '}'; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -2696,14 +2695,16 @@ DatabaseBuilder.prototype.code = function(code) { DatabaseBuilder.prototype.prepare = function(fn) { var self = this; - var opt = self.$options; - var key = 'pre' + (self.$counter++); + if (!self.$functions) self.$functions = []; + var index = self.$functions.push(fn) - 1; var code = '$tmp=fn[{0}].call($F,U.clone(doc),index,repository);if(typeof($tmp)==\'boolean\'){$is=$tmp}else{doc=$tmp;$is=$tmp!=null}'.format(index); + if (self.$scope) code = 'if(!$is){' + code + '}'; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return this; @@ -4362,7 +4363,7 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { stats.current = item.date; stats.index = index; - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { if (value[0] !== '{') return; @@ -4424,19 +4425,17 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { }); return self; -} +}; Storage.prototype.clear = function(beg, end, callback) { var self = this; if (typeof(beg) === 'function') { - mapreduce = beg; callback = end; beg = null; end = null; } else if (typeof(end) === 'function') { - callback = mapreduce; - mapreduce = end; + callback = end; end = null; } @@ -4552,4 +4551,4 @@ function errorhandling(err, builder, response) { function jsonparser(key, value) { return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; -} \ No newline at end of file +} From f8f2bef460b73fef891eda5c6ab0a82efba8c4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 2 Apr 2018 13:40:23 +0200 Subject: [PATCH 0166/1669] Updated version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8da139e94..483f1baf3 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-34", + "version": "3.0.0-35", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 954f7b8bf3a5c3a13c8288cf9fc293141f78ce35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 2 Apr 2018 18:27:28 +0200 Subject: [PATCH 0167/1669] Added a cleaner for unhandled callbacks in NoSQL worker. --- nosql.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/nosql.js b/nosql.js index b5634be05..68f36aac7 100755 --- a/nosql.js +++ b/nosql.js @@ -80,6 +80,43 @@ exports.worker = function() { if (FORK) return; + // Clears unhandled callbacks + ON('service', function() { + + var keys = Object.keys(FORKCALLBACKS); + var time = Date.now(); + + for (var i = 0, length = keys.length; i < length; i++) { + var key = keys[i]; + var item = FORKCALLBACKS[key]; + if (item && item.time) { + var diff = time - item.time; + if (diff >= 60000) { + + delete FORKCALLBACKS[key]; + + var err = new Error('NoSQL worker timeout.'); + switch (item.type) { + case 'find': + item.builder && item.builder.$callback2(err, EMPTYARRAY, 0, EMPTYOBJECT); + break; + case 'count': + item.builder && item.builder.$callback2(err, EMPTYOBJECT, 0, EMPTYOBJECT); + break; + case 'insert': + case 'update': + case 'remove': + item.builder && item.builder.$callback(err, EMPTYOBJECT, EMPTYOBJECT); + break; + default: + item.callback && item.callback(err, EMPTYOBJECT, EMPTYOBJECT); + break; + } + } + } + } + }); + FORKCALLBACKS = {}; FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory }); From 840636a07e0a7236be4d572d88f1eea24b920c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 2 Apr 2018 20:48:55 +0200 Subject: [PATCH 0168/1669] Fixed memory leak in NoSQL worker. --- nosql.js | 8 +++++--- package.json | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 68f36aac7..f43997062 100755 --- a/nosql.js +++ b/nosql.js @@ -84,6 +84,9 @@ exports.worker = function() { ON('service', function() { var keys = Object.keys(FORKCALLBACKS); + if (!keys.length) + return; + var time = Date.now(); for (var i = 0, length = keys.length; i < length; i++) { @@ -94,7 +97,6 @@ exports.worker = function() { if (diff >= 60000) { delete FORKCALLBACKS[key]; - var err = new Error('NoSQL worker timeout.'); switch (item.type) { case 'find': @@ -133,7 +135,7 @@ exports.worker = function() { break; case 'insert': var obj = FORKCALLBACKS[msg.id]; - obj && obj.builder.$callback(msg.err, msg.response, msg.repository); + obj && obj.builder.$callback && obj.builder.$callback(msg.err, msg.response, msg.repository); break; case 'update': var obj = FORKCALLBACKS[msg.id]; @@ -347,7 +349,7 @@ exports.worker = function() { }; Storage.prototype.insert = function(doc) { - send(this.db, 'storage.insert', doc); + notify(this.db, 'storage.insert', doc); return this; }; diff --git a/package.json b/package.json index 483f1baf3..491a87487 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-35", + "version": "3.0.0-36", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 344ebb0288603bd8ee021d7e3250c65abc8e3113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 3 Apr 2018 13:15:55 +0200 Subject: [PATCH 0169/1669] Fixed `String.arg()`. --- utils.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 0667823d2..618456180 100755 --- a/utils.js +++ b/utils.js @@ -54,7 +54,7 @@ const regexpDiacritics = /[^\u0000-\u007e]/g; const regexpXML = /\w+=".*?"/g; const regexpDECODE = /&#?[a-z0-9]+;/g; const regexpPARAM = /\{{2}[^}\n]*\}{2}/g; -const regexpARG = /\{{2}.*?\}{2}/g; +const regexpARG = /\{{1,2}[a-z0-9_.-]+\}{1,2}/gi; const regexpINTEGER = /(^-|\s-)?[0-9]+/g; const regexpFLOAT = /(^-|\s-)?[0-9.,]+/g; const regexpALPHA = /^[A-Za-z0-9]+$/; @@ -3273,7 +3273,9 @@ String.prototype.urlDecode = function() { String.prototype.arg = function(obj) { return this.replace(regexpARG, function(text) { - var val = obj[text.substring(2, text.length - 2).trim()]; + // Is double? + var l = text[1] === '{' ? 2 : 1; + var val = obj[text.substring(l, text.length - l).trim()]; return val == null ? text : val; }); }; From 65960006bf152cffcfad029bc6890d56af5c617c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 5 Apr 2018 14:33:46 +0200 Subject: [PATCH 0170/1669] Improved default value for `Email` type. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 5b95689a9..e8ea6dbb8 100755 --- a/builders.js +++ b/builders.js @@ -1502,7 +1502,7 @@ SchemaBuilderEntity.prototype.default = function() { break; // string case 3: - item[property] = type.isArray ? [] : ''; + item[property] = type.isArray ? [] : type.subtype === 'email' ? '@' : ''; break; // boolean case 4: From f7516192c3c20a14b36e6d9987ef950c8b4ef89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 5 Apr 2018 14:33:54 +0200 Subject: [PATCH 0171/1669] Replace spaces to tabs. --- debug.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug.js b/debug.js index a7dc9c065..9d7d74096 100644 --- a/debug.js +++ b/debug.js @@ -78,7 +78,7 @@ function runwatching() { !options && (options = {}); require('./index'); - const FILENAME = U.getName(process.argv[1] || 'debug.js'); + const FILENAME = U.getName(process.argv[1] || 'debug.js'); const directory = process.cwd(); const VERSION = F.version_header; const REG_CONFIGS = /configs\//g; From fd8bad5f1610b92ac2e717dfa39b93f30c176b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 6 Apr 2018 13:46:58 +0200 Subject: [PATCH 0172/1669] Improved NoSQL embedded performance. --- changes.txt | 2 +- nosql.js | 459 +++++++++++++++++++++++++++++++++------------------ package.json | 2 +- 3 files changed, 304 insertions(+), 159 deletions(-) diff --git a/changes.txt b/changes.txt index 315408c4c..6c90eabbb 100755 --- a/changes.txt +++ b/changes.txt @@ -64,7 +64,7 @@ - fixed: `controller.href()` with Array values - improved: `debug` mode timing with improved consumption -- improved: NoSQL embedded database +- improved: performance (+20%) NoSQL embedded database ======= 2.9.4 (HOTFIX) diff --git a/nosql.js b/nosql.js index f43997062..828870854 100755 --- a/nosql.js +++ b/nosql.js @@ -62,6 +62,8 @@ const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { const NEWLINEBUF = framework_utils.createBuffer('\n', 'utf8'); const CACHE = {}; + +var JSONBUFFER = process.argv.findIndex(n => n.endsWith('nosqlworker.js')) === -1 ? 20 : 50; var FORK; var FORKCALLBACKS; @@ -86,7 +88,7 @@ exports.worker = function() { var keys = Object.keys(FORKCALLBACKS); if (!keys.length) return; - + var time = Date.now(); for (var i = 0, length = keys.length; i < length; i++) { @@ -1116,49 +1118,67 @@ Database.prototype.$update = function() { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + var buf = '['; + var bufcount = 0; + var indexer = 0; - if (value[0] !== '{') - return; + var processing = function(docs) { + for (var j = 0; j < docs.length; j++) { + indexer++; + var doc = docs[j]; + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; - var doc = JSON.parse(value.substring(0, value.length - 1), jsonparser); + item.filter.index = indexer; + var output = item.compare(doc, item.filter, indexer); - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; + if (output) { + builder.$options.backup && builder.$backupdoc(output); + if (item.keys) { + for (var j = 0, jl = item.keys.length; j < jl; j++) { + var val = item.doc[item.keys[j]]; + if (val !== undefined) { + if (typeof(val) === 'function') + output[item.keys[j]] = val(output[item.keys[j]], output); + else + output[item.keys[j]] = val; + } else + output[item.keys[j]] = undefined; + } + } else + output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - item.filter.index = index; - var output = item.compare(doc, item.filter, i); + var e = item.keys ? 'modify' : 'update'; + self.$events[e] && self.emit(e, output); + item.count++; + if (!change) + change = true; + doc = output; + } + } - if (output) { - builder.$options.backup && builder.$backupdoc(output); - if (item.keys) { - for (var j = 0, jl = item.keys.length; j < jl; j++) { - var val = item.doc[item.keys[j]]; - if (val !== undefined) { - if (typeof(val) === 'function') - output[item.keys[j]] = val(output[item.keys[j]], output); - else - output[item.keys[j]] = val; - } else - output[item.keys[j]] = undefined; - } - } else - output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; + writer.write(JSON.stringify(doc) + NEWLINE); + } + }; - var e = item.keys ? 'modify' : 'update'; - self.$events[e] && self.emit(e, output); - item.count++; - change = true; - doc = output; - } + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + + if (value[0] !== '{') + return; + + buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); + bufcount++; + + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = '['; } - writer.write(JSON.stringify(doc) + NEWLINE); })); var finish = function() { - // No change if (!change) { Fs.unlink(self.filenameTemp, function() { @@ -1210,9 +1230,19 @@ Database.prototype.$update = function() { }); }; - CLEANUP(writer, finish); - CLEANUP(reader, () => writer.end()); + + CLEANUP(reader, function() { + + if (bufcount) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = null; + } + + writer.end(); + }); + return self; }; @@ -1379,6 +1409,8 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { var filter = items; var length = filter.length; var first = true; + var canceled = false; + var indexer = 0; for (var i = 0; i < length; i++) { var fil = filter[i]; @@ -1389,98 +1421,125 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - if (first && length > 1) - first = false; - - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + var processing = function(docs) { - if (value[0] !== '{') - return; - - var json = JSON.parse(value.substring(0, value.length - 1), jsonparser); var val; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = index; - var output = item.compare(json, item.filter, index); - if (!output) - continue; + for (var j = 0; j < docs.length; j++) { + var json = docs[j]; + indexer++; + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + item.filter.index = indexer; + var output = item.compare(json, item.filter, indexer); + if (!output) + continue; - item.count++; + item.count++; - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; + if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) + continue; - item.counter++; + item.counter++; - if (item.type) - continue; + if (item.type) + continue; - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) + switch (builder.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[builder.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[builder.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) + } + break; + case 'max': + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; + } + break; + case 'avg': + val = output[builder.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (builder.$inlinesort) + nosqlinlinesorter(item, builder, output); + else if (item.response) + item.response.push(output); else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } + item.response = [output]; + break; + } - if (first && reader.destroy) { - reader.destroy(); - return false; + if (first && !canceled) { + canceled = true; + return; + } } } + }; + + if (first && length > 1) + first = false; + + var buf = '['; + var bufcount = 0; + + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + + if (value[0] !== '{') + return; + + buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); + bufcount++; + + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = '['; + } + + if (canceled) { + reader.destroy && reader.destroy(); + return false; + } + })); var finish = function() { + + bufcount && processing(JSON.parse(buf + ']', jsonparser)); + for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; @@ -1764,29 +1823,55 @@ Database.prototype.$views = function() { var reader = Fs.createReadStream(self.filename); - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + var buf = '['; + var bufcount = 0; + var indexer = 0; + + var processing = function(docs) { + for (var i = 0; i < docs.length; i++) { + var json = docs[i]; + for (var j = 0; j < length; j++) { + var item = self.views[views[j]]; + var res = response[j]; + res.filter.index = indexer; + var output = res.compare(json, res.filter, indexer); + if (!output) + continue; + res.count++; + if (!res.filter.options.sort && ((res.filter.options.skip && res.filter.options.skip >= res.count) || (res.filter.options.take && res.filter.options.take <= res.counter))) + continue; + res.counter++; + !item.type && res.response.push(output); + } + } + }; + + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + + indexer++; if (value[0] !== '{') return; - var json = JSON.parse(value.substring(0, value.length - 1), jsonparser); + buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); + bufcount++; - for (var j = 0; j < length; j++) { - var item = self.views[views[j]]; - var res = response[j]; - res.filter.index = index; - var output = res.compare(json, res.filter, index); - if (!output) - continue; - res.count++; - if (!res.filter.options.sort && ((res.filter.options.skip && res.filter.options.skip >= res.count) || (res.filter.options.take && res.filter.options.take <= res.counter))) - continue; - res.counter++; - !item.type && res.response.push(output); + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = '['; } + })); CLEANUP(reader, function() { + + if (bufcount) { + processing(JSON.parse(buf + ']', jsonparser)); + bufcount = 0; + buf = null; + } + response.wait(function(item, next) { var builder = item.builder; @@ -1914,6 +1999,9 @@ Database.prototype.$remove = function() { var filter = self.pending_remove.splice(0); var length = filter.length; var change = false; + var buf = []; + var bufcount = 0; + var indexer = 0; for (var i = 0; i < length; i++) { var fil = filter[i]; @@ -1921,42 +2009,59 @@ Database.prototype.$remove = function() { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value, index) { + var processing = function(docs, raw) { + + for (var j = 0; j < docs.length; j++) { + + indexer++; + var removed = false; + var json = docs[j]; + + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + item.filter.index = indexer; + + var output = item.compare(json, item.filter, indexer); + if (output) { + builder.$options.backup && builder.$backupdoc(output); + removed = true; + json = output; + break; + } + } + + if (removed) { + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.backup && item.backup.write(raw[j] + NEWLINE); + item.count++; + } + self.$events.remove && self.emit('remove', json); + change = true; + } else + writer.write(raw[j] + NEWLINE); + } + }; + + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { if (value[0] !== '{') return; - var json = JSON.parse(value.substring(0, value.length - 1), jsonparser); - var removed = false; + buf.push(value.substring(0, value.length - 1)); + bufcount++; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = index; - var output = item.compare(json, item.filter, index); - if (output) { - builder.$options.backup && builder.$backupdoc(output); - removed = true; - json = output; - break; - } + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse('[' + buf.join(',') + ']', jsonparser), buf); + buf = []; } - if (removed) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(value); - item.count++; - } - self.$events.remove && self.emit('remove', json); - change = true; - } else - writer.write(value); })); var finish = function() { - // No change if (!change) { Fs.unlink(self.filenameTemp, function() { @@ -1996,7 +2101,17 @@ Database.prototype.$remove = function() { }; CLEANUP(writer, finish); - CLEANUP(reader, () => writer.end()); + + CLEANUP(reader, function() { + + if (bufcount) { + bufcount = 0; + processing(JSON.parse('[' + buf.join(',') + ']', jsonparser), buf); + buf = null; + } + + writer.end(); + }); }; Database.prototype.$remove_inmemory = function() { @@ -4238,9 +4353,12 @@ Storage.prototype.insert = function(doc) { } self.check(); + self.locked_writer = true; Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function(err) { - err && F.error(err, 'NoSQL storage insert: ' + self.db.name); + self.locked_writer = false; self.locked_reader = false; + self.pending.length && self.insert(); + err && F.error(err, 'NoSQL storage insert: ' + self.db.name); }); return self; @@ -4398,28 +4516,55 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { } var reader = Fs.createReadStream(item.filename); - stats.current = item.date; stats.index = index; + var buf = '['; + var bufcount = 0; + var canceled = false; + + var processing = function(docs) { + for (var j = 0; j < docs.length; j++) { + stats.documents++; + var json = docs[j]; + var end = mapreduce(json, repository, stats) === false; + if (end) { + canceled = true; + return; + } + } + }; + reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { if (value[0] !== '{') return; - stats.documents++; + buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); + bufcount++; + + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = '['; + } - var json = JSON.parse(value.substring(0, value.length - 1), jsonparser); - var end = mapreduce(json, repository, stats) === false; - if (end && reader.destroy) { + if (canceled) { stats.canceled = true; - reader.destroy(); + reader.destroy && reader.destroy(); return false; } + })); var finish = function() { + if (bufcount) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = null; + } + stats.processed++; if (item.date === today) { diff --git a/package.json b/package.json index 491a87487..26c1cd55b 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-36", + "version": "3.0.0-37", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0623f674413bcaaf49d479f75e65e049280b317d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 6 Apr 2018 13:53:53 +0200 Subject: [PATCH 0173/1669] Updated buffer. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 828870854..2460e3654 100755 --- a/nosql.js +++ b/nosql.js @@ -63,7 +63,7 @@ const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { const NEWLINEBUF = framework_utils.createBuffer('\n', 'utf8'); const CACHE = {}; -var JSONBUFFER = process.argv.findIndex(n => n.endsWith('nosqlworker.js')) === -1 ? 20 : 50; +var JSONBUFFER = process.argv.findIndex(n => n.endsWith('nosqlworker.js')) === -1 ? 20 : 40; var FORK; var FORKCALLBACKS; From 2c738dd89465c8c87bb7de61fba8b18279af826a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 6 Apr 2018 14:43:45 +0200 Subject: [PATCH 0174/1669] Improved code. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 2460e3654..5e90e7f26 100755 --- a/nosql.js +++ b/nosql.js @@ -1202,7 +1202,7 @@ Database.prototype.$update = function() { self.renaming = true; // Maybe is reading? - if (self.step && self.step !== 9) { + if (self.step && self.step !== 9 && self.step !== 2) { self.next(0); return setTimeout(finish, 100); } @@ -2078,7 +2078,7 @@ Database.prototype.$remove = function() { self.renaming = true; // Maybe is reading? - if (self.step && self.step !== 9) { + if (self.step && self.step !== 9 && self.step !== 3) { self.next(0); return setTimeout(finish, 100); } From b95fba6c89a44c22eb6e757de198f12d731c3de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 6 Apr 2018 22:48:50 +0200 Subject: [PATCH 0175/1669] Fixed critical bug in looping. --- nosql.js | 10 ++++++---- package.json | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nosql.js b/nosql.js index 5e90e7f26..dafa61c7d 100755 --- a/nosql.js +++ b/nosql.js @@ -1123,9 +1123,10 @@ Database.prototype.$update = function() { var indexer = 0; var processing = function(docs) { - for (var j = 0; j < docs.length; j++) { + + for (var a = 0; a < docs.length; a++) { indexer++; - var doc = docs[j]; + var doc = docs[a]; for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; @@ -1179,6 +1180,7 @@ Database.prototype.$update = function() { })); var finish = function() { + // No change if (!change) { Fs.unlink(self.filenameTemp, function() { @@ -1230,8 +1232,6 @@ Database.prototype.$update = function() { }); }; - CLEANUP(writer, finish); - CLEANUP(reader, function() { if (bufcount) { @@ -1243,6 +1243,8 @@ Database.prototype.$update = function() { writer.end(); }); + CLEANUP(writer, finish); + return self; }; diff --git a/package.json b/package.json index 26c1cd55b..004cedb0e 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-37", + "version": "3.0.0-38", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From b280a5411aec46aeddcf482ea19da950866b920f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 7 Apr 2018 08:45:27 +0200 Subject: [PATCH 0176/1669] Fixed NoSQL in-memory updating of documents. --- nosql.js | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index dafa61c7d..8e8ce28d7 100755 --- a/nosql.js +++ b/nosql.js @@ -1276,8 +1276,8 @@ Database.prototype.$update_inmemory = function() { var data = self.inmemory['#']; - for (var j = 0, jl = data.length; j < jl; j++) { - var doc = data[j]; + for (var a = 0, al = data.length; a < al; a++) { + var doc = data[a]; for (var i = 0; i < length; i++) { var item = filter[i]; diff --git a/package.json b/package.json index 004cedb0e..a2c0f14ad 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-38", + "version": "3.0.0-39", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 618c14136d6298ac624b8b90f6824a534bacbff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 7 Apr 2018 10:14:29 +0200 Subject: [PATCH 0177/1669] Improved `NEWSCHEMA()`. --- changes.txt | 3 +++ index.js | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index 6c90eabbb..41babaf3c 100755 --- a/changes.txt +++ b/changes.txt @@ -35,6 +35,9 @@ - added: `$INSERT(schema, model, [options], [callback], [controller])` performs `schema.insert()` - added: `$UPDATE(schema, model, [options], [callback], [controller])` performs `schema.update()` - added: `$REMOVE(schema, [options], [callback], [controller])` performs `schema.remove()` +- added: `HTTP()` global alias for `F.http()` +- added: `HTTPS()` global alias for `F.https()` +- added: `LOAD()` global alias for `F.load()` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` diff --git a/index.js b/index.js index 55e65858b..15562eb8d 100755 --- a/index.js +++ b/index.js @@ -450,7 +450,13 @@ global.NEWTRANSFORM = function() { return TransformBuilder.addTransform.apply(F, arguments); }; -global.NEWSCHEMA = function(group, name) { +global.NEWSCHEMA = function(group, name, make) { + + if (typeof(name) === 'function') { + make = name; + name = undefined; + } + if (!name) { var arr = group.split('/'); if (arr.length === 2) { @@ -461,7 +467,10 @@ global.NEWSCHEMA = function(group, name) { group = 'default'; } } - return framework_builders.newschema(group, name); + + var schema = framework_builders.newschema(group, name); + make && make.call(schema, schema); + return schema; }; global.CLEANUP = function(stream, callback) { @@ -6385,7 +6394,7 @@ F.responseRedirect = function(req, res, url, permanent) { return F; }; -F.load = function(debug, types, pwd) { +global.LOAD = F.load = function(debug, types, pwd) { if (pwd && pwd[0] === '.' && pwd.length < 4) F.directory = directory = U.$normalize(Path.normalize(directory + '/..')); @@ -6630,7 +6639,7 @@ function connection_tunning(socket) { * @param {Function(listen)} middleware A middleware for manual calling of HTTP listener * @return {Framework} */ -F.http = function(mode, options, middleware) { +global.HTTP = F.http = function(mode, options, middleware) { F.consoledebug('begin'); if (typeof(options) === 'function') { @@ -6657,7 +6666,7 @@ F.http = function(mode, options, middleware) { * @param {Function(listen)} middleware A middleware for manual calling of HTTP listener * @return {Framework} */ -F.https = function(mode, options, middleware) { +global.HTTPS = F.https = function(mode, options, middleware) { F.consoledebug('begin'); var http = require('http'); From e1832021605b11678638ebf832f205a20187c2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 7 Apr 2018 18:34:27 +0200 Subject: [PATCH 0178/1669] Improved `NEWSCHEMA()`. --- index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 15562eb8d..8e562d7b8 100755 --- a/index.js +++ b/index.js @@ -379,7 +379,7 @@ global.$UPDATE = function(schema, model, options, callback, controller) { }; // type, schema, model, options, callback, controller -function performschema(type, schema, model, options, callback) { +function performschema(type, schema, model, options, callback, controller) { if (typeof(options) === 'function') { controller = callback; @@ -392,8 +392,10 @@ function performschema(type, schema, model, options, callback) { o.make(model, function(err, model) { if (err) callback(err); - else + else { + model.$$controller = controller; model[type](options, callback); + } }); return !!o; From b39ea967fdc004dd2993d147ea5a1f840b44f1f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Apr 2018 21:33:52 +0200 Subject: [PATCH 0179/1669] Improved `Chunker` + added `U.streamer2()`. --- utils.js | 71 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/utils.js b/utils.js index 618456180..0222c341f 100755 --- a/utils.js +++ b/utils.js @@ -35,6 +35,8 @@ const Path = require('path'); const Fs = require('fs'); const Events = require('events'); const Crypto = require('crypto'); +const Zlib = require('zlib'); + const CONCAT = [null, null]; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); @@ -1603,7 +1605,7 @@ exports.isRelative = function(url) { * @param {String/Buffer} end * @param {Function(value, index)} callback */ -exports.streamer = function(beg, end, callback, skip, stream) { +exports.streamer = function(beg, end, callback, skip, stream, raw) { if (typeof(end) === 'function') { stream = skip; @@ -1651,7 +1653,7 @@ exports.streamer = function(beg, end, callback, skip, stream) { if (skip) skip--; else { - if (callback(buffer.toString('utf8', 0, index + length), indexer++) === false) + if (callback(raw ? buffer.slice(0, index + length) : buffer.toString('utf8', 0, index + length), indexer++) === false) canceled = true; } @@ -1702,7 +1704,7 @@ exports.streamer = function(beg, end, callback, skip, stream) { if (skip) skip--; else { - if (callback(buffer.toString('utf8', bi, ei + elength), indexer++) === false) + if (callback(raw ? buffer.slice(bi, ei + elength) : buffer.toString('utf8', bi, ei + elength), indexer++) === false) canceled = true; } @@ -1725,6 +1727,10 @@ exports.streamer = function(beg, end, callback, skip, stream) { return fn; }; +exports.streamer2 = function(beg, end, callback, skip, stream) { + return exports.streamer(beg, end, callback, skip, stream, true); +}; + /** * HTML encode string * @param {String} str @@ -5560,11 +5566,14 @@ function Chunker(name, max) { this.name = name; this.max = max || 50; this.index = 0; - this.filename = 'chunker_{0}-'.format(name); + this.filename = '{0}-'.format(name); this.stack = []; this.flushing = 0; this.pages = 0; this.count = 0; + this.percentage = 0; + this.autoremove = true; + this.compress = true; this.filename = F.path.temp(this.filename); } @@ -5576,10 +5585,20 @@ Chunker.prototype.append = Chunker.prototype.write = function(obj) { var tmp = self.stack.length; if (tmp >= self.max) { + self.flushing++; self.pages++; self.count += tmp; - Fs.writeFile(self.filename + (self.index++) + '.json', JSON.stringify(self.stack), () => self.flushing--); + + var index = (self.index++); + + if (self.compress) { + Zlib.deflate(exports.createBuffer(JSON.stringify(self.stack), ENCODING), function(err, buffer) { + Fs.writeFile(self.filename + index + '.chunker', buffer, () => self.flushing--); + }); + } else + Fs.writeFile(self.filename + index + '.chunker', JSON.stringify(self.stack), () => self.flushing--); + self.stack = []; } @@ -5593,7 +5612,16 @@ Chunker.prototype.end = function() { self.flushing++; self.pages++; self.count += tmp; - Fs.writeFile(self.filename + (self.index++) + '.json', JSON.stringify(self.stack), () => self.flushing--); + + var index = (self.index++); + + if (self.compress) { + Zlib.deflate(exports.createBuffer(JSON.stringify(self.stack), ENCODING), function(err, buffer) { + Fs.writeFile(self.filename + index + '.chunker', buffer, () => self.flushing--); + }); + } else + Fs.writeFile(self.filename + index + '.chunker', JSON.stringify(self.stack), () => self.flushing--); + self.stack = []; } @@ -5604,13 +5632,16 @@ Chunker.prototype.each = function(onItem, onEnd, indexer) { var self = this; - if (indexer === undefined) + if (indexer == null) { + self.percentage = 0; indexer = 0; + } if (indexer >= self.index) return onEnd && onEnd(); self.read(indexer++, function(err, items) { + self.percentage = Math.ceil((indexer / self.pages) * 100); onItem(items, () => self.each(onItem, onEnd, indexer), indexer - 1); }); @@ -5625,19 +5656,37 @@ Chunker.prototype.read = function(index, callback) { return; } - Fs.readFile(self.filename + index + '.json', function(err, data) { - if (err) + var filename = self.filename + index + '.chunker'; + + Fs.readFile(filename, function(err, data) { + + if (err) { callback(null, EMPTYARRAY); - else + return; + } + + if (self.compress) { + Zlib.inflate(data, function(err, data) { + if (err) { + callback(null, EMPTYARRAY); + } else { + self.autoremove && Fs.unlink(filename, NOOP); + callback(null, data.toString('utf8').parseJSON(true)); + } + }); + } else { + self.autoremove && Fs.unlink(filename, NOOP); callback(null, data.toString('utf8').parseJSON(true)); + } }); + return self; }; Chunker.prototype.clear = function() { var files = []; for (var i = 0; i < this.index; i++) - files.push(this.filename + i + '.json'); + files.push(this.filename + i + '.chunker'); files.wait((filename, next) => Fs.unlink(filename, next)); return this; }; From 845bab46156b461bb230c909d1c283b1a852ca61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Apr 2018 21:34:11 +0200 Subject: [PATCH 0180/1669] Added NoSQL embedded `indexes`. --- nosql.js | 736 +++++++++++++++++++++++++++++++++++++++++++++++-- nosqlworker.js | 40 +++ 2 files changed, 751 insertions(+), 25 deletions(-) diff --git a/nosql.js b/nosql.js index 8e8ce28d7..9f65099e4 100755 --- a/nosql.js +++ b/nosql.js @@ -28,6 +28,7 @@ const Fs = require('fs'); const Path = require('path'); +const Zlib = require('zlib'); if (!global.framework_utils) global.framework_utils = require('./utils'); @@ -56,6 +57,9 @@ const REG_CLEAN = /^[\s]+|[\s]+$/g; const INMEMORY = {}; const FLAGS_READ = ['get']; const COUNTER_MMA = [0, 0]; +const REGNUMBER = /^\d+$/; +const REGINDEXCHAR = /[a-z]{1,2}/; + const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); }; @@ -154,6 +158,9 @@ exports.worker = function() { case 'counter.clear': case 'storage.stats': case 'storage.clear': + case 'indexes.clear': + case 'indexes.reindex': + case 'indexes.get': var obj = FORKCALLBACKS[msg.id]; obj && obj.callback && obj.callback(msg.err, msg.response); break; @@ -403,38 +410,70 @@ exports.worker = function() { return this; }; + Indexes.prototype.create = function(name, properties, type) { + notify(this.db, 'indexes.create', name, properties, type); + return this; + }; + + Indexes.prototype.get = Indexes.prototype.read = function(name, value, callback) { + send(this.db, 'indexes.get', name, value).callback = callback; + return this; + }; + + Indexes.prototype.find = function(name, value) { + return send(this.db, 'indexes.find', name, value).builder = new DatabaseBuilder(this.db); + }; + + Indexes.prototype.clear = function(callback) { + send(this.db, 'indexes.clear').callback = callback; + return this; + }; + + Indexes.prototype.reindex = function(callback) { + send(this.db, 'indexes.reindex').callback = callback; + return this; + }; + }; -function Database(name, filename) { +function Database(name, filename, readonly) { var self = this; var http = filename.substring(0, 6); self.readonly = http === 'http:/' || http === 'https:'; self.filename = self.readonly ? filename.format('') : filename + EXTENSION; - self.filenameCounter = self.readonly ? filename.format('counter', '-') : filename + EXTENSION + EXTENSION_COUNTER; + + if (!readonly) { + self.filenameCounter = self.readonly ? filename.format('counter', '-') : filename + EXTENSION + EXTENSION_COUNTER; + self.filenameLog = self.readonly || readonly === true ? '' : filename + EXTENSION_LOG; + self.filenameBackup = self.readonly || readonly === true ? '' : filename + EXTENSION_BACKUP; + self.filenameStorage = self.readonly || readonly === true ? '' : filename + '-storage/{0}' + EXTENSION; + self.filenameIndexes = self.readonly || readonly === true ? '' : filename + '-indexes/{0}' + EXTENSION; + self.filenameMeta = filename + EXTENSION_META; + self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + EXTENSION); + self.inmemory = {}; + self.inmemorylastusage; + // self.metadata; + self.$meta(); + } + self.filenameTemp = filename + EXTENSION_TMP; - self.filenameLog = self.readonly ? '' : filename + EXTENSION_LOG; - self.filenameBackup = self.readonly ? '' : filename + EXTENSION_BACKUP; - self.filenameStorage = self.readonly ? '' : filename + '-storage/{0}' + EXTENSION; - self.filenameMeta = filename + EXTENSION_META; self.directory = Path.dirname(filename); - self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + EXTENSION); self.name = name; self.pending_update = []; self.pending_append = []; self.pending_reader = []; - self.pending_reader_view = []; + self.pending_streamer = []; self.pending_remove = []; + self.pending_reader_view = readonly ? EMPTYARRAY : []; self.views = {}; self.step = 0; self.pending_drops = false; self.pending_views = false; - self.binary = self.readonly ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); - self.storage = self.readonly ? null : new Storage(self, self.directory + '/' + self.name + '-storage/'); - self.counter = new Counter(self); - self.inmemory = {}; - self.inmemorylastusage; - self.metadata; - self.$meta(); + self.binary = self.readonly || readonly === true ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); + self.storage = self.readonly || readonly === true ? null : new Storage(self, self.directory + '/' + self.name + '-storage/'); + self.indexes = self.readonly || readonly === true ? null : new Indexes(self, self.directory + '/' + self.name + '-indexes/'); + self.counter = readonly === true ? null : new Counter(self); + self.$timeoutmeta; self.$events = {}; self.$free = true; @@ -594,7 +633,7 @@ Database.prototype.insert = function(doc, unique) { builder = new DatabaseBuilder2(self); var json = framework_builders.isSchema(doc) ? doc.$clean() : doc; - self.pending_append.push({ doc: JSON.stringify(json), builder: builder }); + self.pending_append.push({ doc: JSON.stringify(json), raw: doc, builder: builder }); setImmediate(next_operation, self, 1); self.$events.insert && self.emit('insert', json); return builder; @@ -778,6 +817,9 @@ Database.prototype.free = function(force) { self.counter.removeAllListeners(true); self.binary.removeAllListeners(true); self.removeAllListeners(true); + self.indexes = null; + self.binary = null; + self.counter = null; delete F.databases[self.name]; return self; }; @@ -818,6 +860,30 @@ Database.prototype.find = function(view) { return builder; }; +Database.prototype.load = function(take, skip, callback) { + var self = this; + + if (typeof(take) === 'function') { + callback = take; + take = 0; + skip = 0; + } else if (typeof(skip) === 'function') { + callback = skip; + skip = 0; + } + + self.pending_loader.push({ take: take, skip: skip, callback: callback, counter: 0, count: 0, response: [] }); + setImmediate(next_operation, self, 10); + return self; +}; + +Database.prototype.streamer = function(fn, callback) { + var self = this; + self.pending_streamer.push({ fn: fn, callback: callback, repository: {} }); + setImmediate(next_operation, self, 10); + return self; +}; + Database.prototype.throwReadonly = function() { throw new Error('Database "{0}" is readonly.'.format(this.name)); }; @@ -942,6 +1008,11 @@ Database.prototype.next = function(type) { return; } + if (this.step !== 10 && this.pending_streamer.length) { + this.$streamer(); + return; + } + if (this.step !== type) { this.step = 0; setImmediate(next_operation, this, 0); @@ -1044,13 +1115,17 @@ Database.prototype.$append = function() { return; } - self.pending_append.splice(0).limit(20, function(items, next) { - var json = []; + self.pending_append.splice(0).limit(JSONBUFFER, function(items, next) { - for (var i = 0, length = items.length; i < length; i++) - json.push(items[i].doc); + var json = ''; + for (var i = 0, length = items.length; i < length; i++) { + json += items[i].doc + NEWLINE; + + if (self.indexes && self.indexes.indexes.length) + self.indexes.insert(items[i].raw); + } - Fs.appendFile(self.filename, json.join(NEWLINE) + NEWLINE, function(err) { + Fs.appendFile(self.filename, json, function(err) { err && F.error(err, 'NoSQL insert: ' + self.name); @@ -1059,6 +1134,7 @@ Database.prototype.$append = function() { var callback = items[i].builder.$callback; callback && callback(err, 1); } + next(); }); @@ -1125,8 +1201,12 @@ Database.prototype.$update = function() { var processing = function(docs) { for (var a = 0; a < docs.length; a++) { + indexer++; + var doc = docs[a]; + var is = false; + for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; @@ -1156,9 +1236,13 @@ Database.prototype.$update = function() { if (!change) change = true; doc = output; + is = true; } } + if (is && self.indexes && self.indexes.indexes.length) + self.indexes.update(doc); + writer.write(JSON.stringify(doc) + NEWLINE); } }; @@ -1277,7 +1361,10 @@ Database.prototype.$update_inmemory = function() { var data = self.inmemory['#']; for (var a = 0, al = data.length; a < al; a++) { + var doc = data[a]; + var is = false; + for (var i = 0; i < length; i++) { var item = filter[i]; @@ -1304,9 +1391,14 @@ Database.prototype.$update_inmemory = function() { var e = item.keys ? 'modify' : 'update'; self.$events[e] && self.emit(e, doc); item.count++; - change = true; + if (!change) + change = true; + is = true; } } + + if (is && self.indexes && self.indexes.indexes.length) + self.indexes.update(doc); } self.$save('#'); @@ -1407,7 +1499,6 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { } else reader = Fs.createReadStream(filename); - var filter = items; var length = filter.length; var first = true; @@ -1596,6 +1687,168 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { return self; }; +Database.prototype.$streamer = function() { + + var self = this; + self.step = 10; + + if (!self.pending_streamer.length) { + self.next(0); + return self; + } + + var filter = self.pending_streamer.splice(0); + var reader = Fs.createReadStream(self.filename); + var length = filter.length; + var count = 0; + + var processing = function(docs) { + for (var j = 0; j < docs.length; j++) { + var json = docs[j]; + count++; + for (var i = 0; i < length; i++) + filter[i].fn(json, filter[i].repository, count); + } + }; + + var buf = '['; + var bufcount = 0; + + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + + if (value[0] !== '{') + return; + + buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); + bufcount++; + + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = '['; + } + + })); + + var finish = function() { + bufcount && processing(JSON.parse(buf + ']', jsonparser)); + for (var i = 0; i < length; i++) + filter[i].callback && filter[i].callback(null, filter[i].repository, count); + self.next(0); + }; + + if (reader) + CLEANUP(reader, finish); + else + finish(); + + return self; +}; + +Database.prototype.$loader2 = function() { + + var self = this; + self.step = 10; + + if (!self.pending_loader.length) { + self.next(0); + return self; + } + + var filter = self.pending_loader.splice(0); + var length = filter.length; + var canceled = false; + var skip = null; + + for (var i = 0; i < length; i++) + skip = skip == null ? filter[i].skip : skip > filter[i].skip ? filter[i].skip : skip; + + if (skip) { + for (var i = 0; i < length; i++) + filter[i].count = skip - 1; + } + + var reader = Fs.createReadStream(self.filename); + + var processing = function(docs) { + + for (var j = 0; j < docs.length; j++) { + + var json = docs[j]; + var cancel = length; + + for (var i = 0; i < length; i++) { + var item = filter[i]; + + if (item.take && item.counter >= item.take) { + cancel--; + continue; + } + + item.count++; + + if (item.skip && item.skip > item.count) + continue; + + item.counter++; + item.response.push(json); + } + + if (!cancel) { + canceled = true; + return; + } + } + }; + + var buf = '['; + var bufcount = 0; + + reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + + if (value[0] !== '{') + return; + + var len = value.length - 1; + + if (self.$documentsize == null) + self.$documentsize = len; + else if (self.$documentsize > len) + self.$documentsize = len; + + buf += (bufcount ? ',' : '') + value.substring(0, len); + bufcount++; + + if (bufcount % JSONBUFFER === 0) { + bufcount = 0; + processing(JSON.parse(buf + ']', jsonparser)); + buf = '['; + } + + if (canceled) { + reader.destroy && reader.destroy(); + return false; + } + + }, skip)); + + var finish = function() { + bufcount && processing(JSON.parse(buf + ']', jsonparser)); + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.callback(null, item.response); + } + self.next(0); + }; + + if (reader) + CLEANUP(reader, finish); + else + finish(); + + return self; +}; + function nosqlinlinesorter(item, builder, doc) { if (!item.response) { @@ -2256,6 +2509,7 @@ function DatabaseBuilder(db) { this.$callback = NOOP; // this.$scalar; // this.$scalarfield; + // this.$done; --> internal for indexes this.$code = []; this.$params = {}; @@ -2291,6 +2545,7 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository if (err || !self.$join) { self.$options.log && self.log(); + self.$done && setImmediate(self.$done); return self.$callback(err, response, count, repository); } @@ -2319,6 +2574,7 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository self.$options.log && self.log(); self.$callback(err, response, count, repository); + self.$done && setImmediate(self.$done); return self; }; @@ -4285,6 +4541,436 @@ Binary.prototype.all = function(callback) { return self; }; +function Indexes(db, directory) { + this.db = db; + this.directory = directory; + this.indexes = []; + this.changes = []; + this.flushing = false; + this.instances = {}; + this.reindexing = false; +} + +Indexes.prototype.create = function(name, properties, type) { + + if (typeof(properties) === 'string') { + type = properties; + properties = null; + } + + !this.indexes.findItem('name', name) && this.indexes.push({ name: name, properties: properties ? properties : [name], type: type }); + return this; +}; + +Indexes.prototype.$index = function(index, value) { + + var key = ''; + var number = false; + var num = value.length > 1 ? 2 : 3; + + for (var i = 0; i < value.length; i++) { + var val = value[i]; + + switch (typeof(val)) { + case 'number': + if (index.type === 'reverse') { + val = val.toString(); + val = val.substring(val.length - num).padLeft(num); + } else + val = val.toString().substring(0, num).padLeft(num); + number = true; + break; + case 'boolean': + val = val ? '1' : '0'; + break; + case 'string': + if (REGNUMBER.test(val)) { + val = +val; + if (index.type === 'reverse') { + val = val.toString(); + val = val.substring(val.length - num).padLeft(num); + } else + val = val.toString().substring(0, num).padLeft(num); + number = true; + } else { + if (val.isUID()) { + val = val.substring(4, 6) + val.substring(2, 4) + val.substring(0, 2); + number = true; + } else { + val = val.toLowerCase().removeDiacritics().match(REGINDEXCHAR); + if (val) { + val = val.toString(); + if (index.type === 'reverse') + val = val.substring(val.length - 2); + else + val = val.substring(0, 2); + } + } + } + break; + case 'object': + val = val instanceof Date ? (val.getFullYear().toString().substring(2) + val.format('MM')) : ''; + break; + } + + if (val == null) + continue; + + if (val) + key += val; + } + + return key ? (index.name + '_' + (number && value.length === 1 ? key : key)) : null; +}; + +Indexes.prototype.get = Indexes.prototype.read = function(name, value, callback) { + + var self = this; + var index = self.indexes.findItem('name', name); + + if (!index) { + callback(new Error('Index not found.')); + return self; + } + + if (!(value instanceof Array)) + value = [value]; + + var key = self.$index(index, value); + if (!key) { + callback(new Error('Bad value for generating index.')); + return self; + } + + if (self.changes.length) { + var change = self.findchanges(index, key, value); + if (change) { + callback(null, CLONE(change.doc)); + return self; + } + } + + if (self.instances[key]) { + self.instances[key].PENDING++; + } else { + self.instances[key] = new Database(key, self.directory + key, true); + self.instances[key].PENDING = 0; + } + + var builder = self.instances[key].one(); + + for (var i = 0; i < index.properties.length; i++) + builder.where(index.properties[i], value[i]); + + builder.callback(function(err, response) { + + if (self.instances[key].PENDING) + self.instances[key].PENDING--; + else + delete self.instances[key]; + + callback(err, response); + }); + + return self; +}; + +Indexes.prototype.find = function(name, value) { + + var self = this; + var index = self.indexes.findItem('name', name); + + if (!index) + throw new Error('Index not found.'); + + if (!(value instanceof Array)) + value = [value]; + + var key = self.$index(index, value); + if (!key) + throw new Error('Bad value for generating index.'); + + if (self.instances[key]) { + self.instances[key].PENDING++; + } else { + self.instances[key] = new Database(key, self.directory + key, true); + self.instances[key].PENDING = 0; + } + + var builder = self.instances[key].find(); + + builder.$done = function() { + if (self.instances[key].PENDING) + self.instances[key].PENDING--; + else + delete self.instances[key]; + }; + + return builder; +}; + +Indexes.prototype.clear = function(callback) { + var self = this; + Fs.readdir(self.directory, function(err, files) { + + if (err) { + callback(); + return; + } + + files.wait(function(item, next) { + Fs.unlink(Path.join(self.directory, item), next); + }, callback); + }); + return self; +}; + +Indexes.prototype.reindex = function(callback) { + var self = this; + + if (self.reindexing) { + callback(new Error('Reindex is running.')); + return self; + } + + self.reindexing = true; + self.clear(function() { + var chunker = U.chunker(self.db.name + '_reindex', 10000); + self.db.streamer(function(doc) { + chunker.write(doc); + }, function(err, repository, count) { + chunker.end(); + chunker.each(function(docs, next) { + self.db.$events.indexing && self.db.emit('indexing', chunker.percentage, chunker.count); + for (var i = 0; i < docs.length; i++) + self.insert(docs[i], true); + self.$reindexingnext = next; + }, function() { + self.reindexing = false; + callback && callback(null, count); + }); + }); + }); + + return self; +}; + +Indexes.prototype.check = function() { + + var self = this; + if (self.exists) + return self; + + self.exists = true; + + try { + Fs.mkdirSync(self.directory); + } catch (err) {} + + return self; +}; + +Indexes.prototype.makeindex = function(index, doc) { + + var arr = []; + + for (var i = 0; i < index.properties.length; i++) { + var val = doc[index.properties[i]]; + arr.push(val); + } + + return arr; +}; + +Indexes.prototype.findchanges = function(index, key, values) { + var self = this; + for (var i = 0, length = self.changes.length; i < length; i++) { + var item = self.changes[i]; + if (item.key !== key || item.value.length !== values.length) + continue; + + var is = true; + + for (var j = 0; j < values.length; j++) { + if (values[j] !== item.value[j]) { + is = false; + break; + } + } + + if (is) + return item; + } +}; + +Indexes.prototype.insert = function(doc, reindex) { + + var self = this; + + for (var i = 0; i < self.indexes.length; i++) { + + var index = self.indexes[i]; + var values = self.makeindex(index, doc); + + if (values.length) { + var key = self.$index(index, values); + if (!key) + continue; + + if (!reindex) { + var item = self.findchanges(index, key, values); + if (item) { + item.doc = doc; + return self; + } + } + + self.changes.push({ insert: true, key: key, doc: doc, name: index.name, value: values }); + } + } + + self.changes.length && self.flush(); + return self; +}; + +Indexes.prototype.update = function(doc) { + var self = this; + + for (var i = 0; i < self.indexes.length; i++) { + + var index = self.indexes[i]; + var values = self.makeindex(index, doc); + + if (values.length) { + var key = self.index(values); + if (!key) + continue; + var item = self.findchanges(index, key, values); + if (item) + item.doc = doc; + else + self.changes.push({ update: true, key: key, doc: doc, name: index.name, properties: index.properties, value: values }); + } + } + + self.changes.length && self.flush(); + return self; +}; + + +Indexes.prototype.remove = function(doc) { + var self = this; + + for (var i = 0; i < self.indexes.length; i++) { + + var index = self.indexes[i]; + var values = self.makeindex(index, doc); + + if (values.length) { + var key = self.$index(index, values); + if (!key) + continue; + + var item = self.findchanges(index, key, values); + if (!item) + self.changes.push({ remove: true, key: key, name: index.name, properties: index.properties, value: values }); + } + } + + self.changes.length && self.flush(); + return self; +}; + +Indexes.prototype.flush = function() { + + var self = this; + + if (self.flushing) + return self; + + self.check(); + self.flushing = true; + + var count = 0; + + var fn = function() { + + if (count > 0) + return; + + self.flushing = false; + self.$free(); + + if (!self.changes.length) { + if (self.$reindexingnext) { + self.$reindexingnext(); + self.$reindexingnext = null; + } + } + + self.changes.length && setImmediate(() => self.flush()); + }; + + var arr = self.changes.splice(0); + + for (var i = 0; i < arr.length; i++) { + + var item = arr[i]; + + if (self.instances[item.key]) { + self.instances[item.key].PENDING++; + } else { + self.instances[item.key] = new Database(item.key, self.directory + item.key, true); + self.instances[item.key].PENDING = 0; + } + + if (item.update) { + count++; + var builder = self.instances[item.key].update(item.doc).callback(function() { + if (self.instances[item.key].PENDING) + self.instances[item.key].PENDING--; + count--; + fn(); + }); + + for (var j = 0; j < item.properties.length; j++) + builder.where(item.properties[j], item.value[j]); + + } else if (item.insert) { + count++; + self.instances[item.key].insert(item.doc).callback(function() { + if (self.instances[item.key].PENDING) + self.instances[item.key].PENDING--; + count--; + fn(); + }); + + } else { + count++; + var builder = self.instances[item.key].remove().callback(function() { + if (self.instances[item.key].PENDING) + self.instances[item.key].PENDING--; + count--; + fn(); + }); + + for (var j = 0; j < item.properties.length; j++) + builder.where(item.properties[j], item.value[j]); + } + } +}; + +Indexes.prototype.$free = function() { + var self = this; + var keys = Object.keys(self.instances); + for (var i = 0; i < keys.length; i++) { + var db = self.instances[keys[i]]; + if (!db && db.PENDING) + delete self.instances[keys[i]]; + } + return self; +}; + function Storage(db, directory) { this.db = db; this.directory = directory; @@ -4381,7 +5067,7 @@ Storage.prototype.stats = function(name, fn) { return this; }; -Storage.prototype.mapreduce = function(name, fn) { +Storage.prototype.mapreduce = function(name, fn, def) { var self = this; @@ -4395,7 +5081,7 @@ Storage.prototype.mapreduce = function(name, fn) { } else { item = {}; item.name = name; - item.repository = {}; + item.repository = def || {}; item.reduce = fn; item.ready = false; self.$mapreduce.push(item); diff --git a/nosqlworker.js b/nosqlworker.js index da6ba8a90..f9493ffa3 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -13,6 +13,9 @@ const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; const RESTORAGESCAN = { TYPE: 'storage.scan' }; const RESTORAGESTATS = { TYPE: 'storage.stats' }; const RESSTORAGECLEAR = { TYPE: 'storage.clear' }; +const RESINDEXESGET = { TYPE: 'indexes.get' }; +const RESINDEXESCLEAR = { TYPE: 'indexes.clear' }; +const RESINDEXESREINDEX = { TYPE: 'indexes.reindex' }; function killprocess() { process.exit(0); @@ -183,5 +186,42 @@ process.on('message', function(msg) { process.send(RESSTORAGECLEAR); }); break; + case 'indexes.create': + db.indexes.create(msg.arg[0], msg.arg[1], msg.arg[2]); + break; + case 'indexes.get': + db.indexes.get(msg.arg[0], msg.arg[1], function(err, response) { + RESINDEXESGET.id = msg.id; + RESINDEXESGET.response = response; + RESINDEXESGET.err = err; + process.send(RESINDEXESGET); + }); + break; + case 'indexes.find': + db.indexes.find(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, count, repository) { + RESFIND.err = err; + RESFIND.response = response; + RESFIND.count = count; + RESFIND.repository = repository; + RESFIND.id = msg.id; + process.send(RESFIND); + }); + break; + case 'indexes.clear': + db.indexes.clear(function(err, response) { + RESINDEXESCLEAR.id = msg.id; + RESINDEXESCLEAR.response = response; + RESINDEXESCLEAR.err = err; + process.send(RESINDEXESCLEAR); + }); + break; + case 'indexes.reindex': + db.indexes.reindex(function(err, response) { + RESINDEXESREINDEX.id = msg.id; + RESINDEXESREINDEX.response = response; + RESINDEXESREINDEX.err = err; + process.send(RESINDEXESREINDEX); + }); + break; } }); \ No newline at end of file From cc0ee6c39a12aada16b9205c3f831225cda30e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Apr 2018 21:34:31 +0200 Subject: [PATCH 0181/1669] Added `NOSQLINDEXES()` alias. --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 8e562d7b8..cb6e45f54 100755 --- a/index.js +++ b/index.js @@ -286,6 +286,7 @@ global.Mail = framework_mail; global.WTF = (message, name, uri) => F.problem(message, name, uri); global.NOBIN = (name) => F.nosql(name).binary; global.NOSQLSTORAGE = (name) => F.nosql(name).storage; +global.NOSQLINDEXES = (name) => F.nosql(name).indexes; global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); global.CONFIG = (name) => F.config[name]; From 76dae09870ac6b142bb08867b22aaf9acc6cb64b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Apr 2018 21:34:41 +0200 Subject: [PATCH 0182/1669] New changes. --- changes.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changes.txt b/changes.txt index 41babaf3c..c183c2ec2 100755 --- a/changes.txt +++ b/changes.txt @@ -4,6 +4,7 @@ - added: (IMPORTANT) Total.js components can have async delegate - added: (IMPORTANT) NoSQL worker - added: (IMPORTANT) NoSQL embedded storage for smaller big data / IoT +- added: (IMPORTANT) NoSQL indexes - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` @@ -29,6 +30,7 @@ - added: config `nosql-worker' for enabling worker for NoSQL embedded database (default: `false`) - added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` +- added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` - added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue - added: `$SAVE(schema, model, [options], [callback], [controller])` performs `schema.save()` @@ -38,6 +40,7 @@ - added: `HTTP()` global alias for `F.http()` - added: `HTTPS()` global alias for `F.https()` - added: `LOAD()` global alias for `F.load()` +- added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` @@ -48,6 +51,8 @@ - updated: `@{import('livereload')}` or `@{import('livereload wss://mywebsite.com')}` supports `livereload` value and it's rendered in `debug` mode only - updated: information after the framework is started - updated: `schema.define('name', null)` removes a schema field +- updated: Chunker supports `compression`, default `true` +- updated: Chunker supports `autoremove` processed files in `each()` or `read()` method, default `true` - fixed: mail attachments - fixed: mail `message.manually()` From 44cc056f302d1b5e8ce5cfa4e761b4c94cce90d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Apr 2018 21:37:19 +0200 Subject: [PATCH 0183/1669] Fixed `String.arg()`. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 0222c341f..03887c8e5 100755 --- a/utils.js +++ b/utils.js @@ -56,7 +56,7 @@ const regexpDiacritics = /[^\u0000-\u007e]/g; const regexpXML = /\w+=".*?"/g; const regexpDECODE = /&#?[a-z0-9]+;/g; const regexpPARAM = /\{{2}[^}\n]*\}{2}/g; -const regexpARG = /\{{1,2}[a-z0-9_.-]+\}{1,2}/gi; +const regexpARG = /\{{1,2}[a-z0-9_.-\s]+\}{1,2}/gi; const regexpINTEGER = /(^-|\s-)?[0-9]+/g; const regexpFLOAT = /(^-|\s-)?[0-9.,]+/g; const regexpALPHA = /^[A-Za-z0-9]+$/; From df82a07692efeb35501c31501f5734aacac7b359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Apr 2018 22:09:52 +0200 Subject: [PATCH 0184/1669] Fixed updating of indexes. --- nosql.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nosql.js b/nosql.js index 9f65099e4..2e5a3aae3 100755 --- a/nosql.js +++ b/nosql.js @@ -1206,6 +1206,7 @@ Database.prototype.$update = function() { var doc = docs[a]; var is = false; + var copy = self.indexes && self.indexes.indexes.length ? CLONE(doc) : null; for (var i = 0; i < length; i++) { var item = filter[i]; @@ -1241,7 +1242,7 @@ Database.prototype.$update = function() { } if (is && self.indexes && self.indexes.indexes.length) - self.indexes.update(doc); + self.indexes.update(doc, copy); writer.write(JSON.stringify(doc) + NEWLINE); } @@ -4833,7 +4834,7 @@ Indexes.prototype.insert = function(doc, reindex) { return self; }; -Indexes.prototype.update = function(doc) { +Indexes.prototype.update = function(doc, old) { var self = this; for (var i = 0; i < self.indexes.length; i++) { @@ -4842,14 +4843,19 @@ Indexes.prototype.update = function(doc) { var values = self.makeindex(index, doc); if (values.length) { - var key = self.index(values); + var key = self.$index(index, values); if (!key) continue; var item = self.findchanges(index, key, values); if (item) item.doc = doc; - else + else { + var oldvalues = self.makeindex(index, old); + var oldkey = self.$index(index, oldvalues); self.changes.push({ update: true, key: key, doc: doc, name: index.name, properties: index.properties, value: values }); + if (oldkey !== key && oldkey) + self.changes.push({ remove: true, key: oldkey, name: index.name, properties: index.properties, value: oldvalues }); + } } } @@ -4926,7 +4932,7 @@ Indexes.prototype.flush = function() { if (item.update) { count++; - var builder = self.instances[item.key].update(item.doc).callback(function() { + var builder = self.instances[item.key].update(item.doc, item.doc).callback(function() { if (self.instances[item.key].PENDING) self.instances[item.key].PENDING--; count--; From 3bbf05bf513622d381e6db91202b79d4e1481dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 14:38:00 +0200 Subject: [PATCH 0185/1669] Replaced Node.js streams for NoSQL streamer. --- nosql.js | 711 ++++++++++++++++++------------------------------- nosqlstream.js | 445 +++++++++++++++++++++++++++++++ nosqlworker.js | 26 ++ 3 files changed, 728 insertions(+), 454 deletions(-) create mode 100644 nosqlstream.js diff --git a/nosql.js b/nosql.js index 2e5a3aae3..9bc2aa606 100755 --- a/nosql.js +++ b/nosql.js @@ -28,7 +28,7 @@ const Fs = require('fs'); const Path = require('path'); -const Zlib = require('zlib'); +const NoSQLStream = require('./nosqlstream'); if (!global.framework_utils) global.framework_utils = require('./utils'); @@ -465,15 +465,15 @@ function Database(name, filename, readonly) { self.pending_streamer = []; self.pending_remove = []; self.pending_reader_view = readonly ? EMPTYARRAY : []; - self.views = {}; + self.views = null; self.step = 0; self.pending_drops = false; self.pending_views = false; + self.pending_reindex = false; self.binary = self.readonly || readonly === true ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); self.storage = self.readonly || readonly === true ? null : new Storage(self, self.directory + '/' + self.name + '-storage/'); self.indexes = self.readonly || readonly === true ? null : new Indexes(self, self.directory + '/' + self.name + '-indexes/'); self.counter = readonly === true ? null : new Counter(self); - self.$timeoutmeta; self.$events = {}; self.$free = true; @@ -877,9 +877,15 @@ Database.prototype.load = function(take, skip, callback) { return self; }; -Database.prototype.streamer = function(fn, callback) { +Database.prototype.streamer = function(fn, repository, callback) { var self = this; - self.pending_streamer.push({ fn: fn, callback: callback, repository: {} }); + + if (typeof(repository) === 'function') { + callback = repository; + repository = null; + } + + self.pending_streamer.push({ fn: fn, callback: callback, repository: repository || {} }); setImmediate(next_operation, self, 10); return self; }; @@ -910,6 +916,7 @@ Database.prototype.count = function(view) { Database.prototype.one = function(view) { var self = this; var builder = new DatabaseBuilder(self); + builder.first(); if (view) { @@ -941,6 +948,8 @@ Database.prototype.top = function(max, view) { Database.prototype.view = function(name) { var builder = new DatabaseBuilder(this); + if (!this.views) + this.views = {}; this.views[name] = {}; this.views[name] = builder; this.views[name].$filename = this.filename.replace(/\.nosql/, '#' + name + '.nosql'); @@ -959,7 +968,7 @@ Database.prototype.next = function(type) { return; } - if (this.step < 2) { + if (this.step < 2 && !this.pending_reindex) { if (this.step !== 2 && this.pending_update.length) { if (INMEMORY[this.name]) this.$update_inmemory(); @@ -987,7 +996,7 @@ Database.prototype.next = function(type) { return; } - if (this.step !== 1 && this.pending_append.length) { + if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) { if (INMEMORY[this.name]) this.$append_inmemory(); else @@ -995,7 +1004,7 @@ Database.prototype.next = function(type) { return; } - if (this.step !== 7 && this.pending_drops) { + if (this.step !== 7 && !this.pending_reindex && this.pending_drops) { this.$drop(); return; } @@ -1044,7 +1053,7 @@ Database.prototype.$save = function(view) { if (view !== '#') filename = filename.replace(/\.nosql/, '#' + view + '.nosql'); - Fs.writeFile(filename, builder.join(NEWLINE) + NEWLINE, F.error()); + Fs.writeFile(filename, builder.join(NEWLINE) + NEWLINE, F.errorcallback); }, 50, 100); return self; }; @@ -1092,7 +1101,7 @@ Database.prototype.$meta = function(write) { if (write) { self.readonly && self.throwReadonly(); - Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), F.error()); + Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), F.errorcallback); return self; } @@ -1143,7 +1152,7 @@ Database.prototype.$append = function() { function next_append(self) { self.next(0); - setImmediate(views_refresh, self); + self.views && setImmediate(views_refresh, self); } Database.prototype.$append_inmemory = function() { @@ -1181,24 +1190,12 @@ Database.prototype.$update = function() { return self; } - var reader = Fs.createReadStream(self.filename); - var writer = Fs.createWriteStream(self.filenameTemp); - - var filter = self.pending_update.splice(0); - var length = filter.length; - var change = false; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - var buf = '['; - var bufcount = 0; var indexer = 0; + var fs = new NoSQLStream(self.filename); - var processing = function(docs) { + fs.ondocuments = function() { + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); + var updated = false; for (var a = 0; a < docs.length; a++) { @@ -1234,53 +1231,54 @@ Database.prototype.$update = function() { var e = item.keys ? 'modify' : 'update'; self.$events[e] && self.emit(e, output); item.count++; - if (!change) - change = true; - doc = output; + if (!fs.changed) + fs.changed = true; + docs[a] = output; is = true; + if (!updated) + updated = true; } } if (is && self.indexes && self.indexes.indexes.length) self.indexes.update(doc, copy); - - writer.write(JSON.stringify(doc) + NEWLINE); } - }; - - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { - if (value[0] !== '{') - return; - - buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); - bufcount++; + if (updated) { + var tmp = ''; + for (var i = 0; i < docs.length; i++) + tmp += JSON.stringify(docs[i]) + NEWLINE; + fs.write(tmp); + } else + fs.write(fs.docscache); + }; - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = '['; - } + var filter = self.pending_update.splice(0); + var length = filter.length; - })); + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + } var finish = function() { - // No change - if (!change) { - Fs.unlink(self.filenameTemp, function() { + if (!fs.changed) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert); - self.insert(item.insert).$callback = item.builder.$callback; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } + for (var i = 0; i < length; i++) { + var item = filter[i]; + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert); + self.insert(item.insert).$callback = item.builder.$callback; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } + } + fs.flush(function() { + fs = null; self.next(0); }); return; @@ -1295,8 +1293,10 @@ Database.prototype.$update = function() { } self.step = 9; + fs.flush(function(err) { - Fs.rename(self.filenameTemp, self.filename, function(err) { + err && F.error(err); + self.renaming = false; for (var i = 0; i < length; i++) { var item = filter[i]; @@ -1305,31 +1305,20 @@ Database.prototype.$update = function() { self.insert(item.insert).$callback = item.builder.$callback; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(err, item.builder, item.count), item.count, item.filter.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } } - setImmediate(function() { - self.renaming = false; - self.next(0); - change && setImmediate(views_refresh, self); - }); + fs = null; + self.next(0); + self.views && setImmediate(views_refresh, self); }); }; - CLEANUP(reader, function() { - - if (bufcount) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = null; - } - - writer.end(); + fs.openupdate(function() { + fs.read(finish, true); }); - CLEANUP(writer, finish); - return self; }; @@ -1419,7 +1408,7 @@ Database.prototype.$update_inmemory = function() { setImmediate(function() { self.next(0); - change && setImmediate(views_refresh, self); + change && self.views && setImmediate(views_refresh, self); }); }); }; @@ -1497,13 +1486,11 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { }); return self; } - } else - reader = Fs.createReadStream(filename); + } var filter = items; var length = filter.length; var first = true; - var canceled = false; var indexer = 0; for (var i = 0; i < length; i++) { @@ -1515,8 +1502,14 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - var processing = function(docs) { + if (first && length > 1) + first = false; + + var fs = new NoSQLStream(self.filename); + + fs.ondocuments = function() { + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); var val; for (var j = 0; j < docs.length; j++) { @@ -1595,95 +1588,67 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { break; } - if (first && !canceled) { - canceled = true; - return; - } + if (first) + return false; } } }; - if (first && length > 1) - first = false; - - var buf = '['; - var bufcount = 0; - - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { - - if (value[0] !== '{') - return; - - buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); - bufcount++; - - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = '['; - } - - if (canceled) { - reader.destroy && reader.destroy(); - return false; - } - - })); + var cb = function() { + fs.read(function() { + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + var output; - var finish = function() { + if (builder.$options.scalar || !builder.$options.sort) { - bufcount && processing(JSON.parse(buf + ']', jsonparser)); + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || EMPTYARRAY; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + continue; + } - if (builder.$options.scalar || !builder.$options.sort) { + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) + if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; + builder.done(); } - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else - output = item.response || EMPTYARRAY; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - callback(); + fs = null; + callback(); + }); }; if (reader) - CLEANUP(reader, finish); + fs.openstream(reader, cb); else - finish(); + fs.openread(cb); return self; }; @@ -1699,11 +1664,12 @@ Database.prototype.$streamer = function() { } var filter = self.pending_streamer.splice(0); - var reader = Fs.createReadStream(self.filename); var length = filter.length; var count = 0; + var fs = new NoSQLStream(self.filename); - var processing = function(docs) { + fs.ondocuments = function() { + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); for (var j = 0; j < docs.length; j++) { var json = docs[j]; count++; @@ -1712,140 +1678,14 @@ Database.prototype.$streamer = function() { } }; - var buf = '['; - var bufcount = 0; - - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { - - if (value[0] !== '{') - return; - - buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); - bufcount++; - - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = '['; - } - - })); - - var finish = function() { - bufcount && processing(JSON.parse(buf + ']', jsonparser)); - for (var i = 0; i < length; i++) - filter[i].callback && filter[i].callback(null, filter[i].repository, count); - self.next(0); - }; - - if (reader) - CLEANUP(reader, finish); - else - finish(); - - return self; -}; - -Database.prototype.$loader2 = function() { - - var self = this; - self.step = 10; - - if (!self.pending_loader.length) { - self.next(0); - return self; - } - - var filter = self.pending_loader.splice(0); - var length = filter.length; - var canceled = false; - var skip = null; - - for (var i = 0; i < length; i++) - skip = skip == null ? filter[i].skip : skip > filter[i].skip ? filter[i].skip : skip; - - if (skip) { - for (var i = 0; i < length; i++) - filter[i].count = skip - 1; - } - - var reader = Fs.createReadStream(self.filename); - - var processing = function(docs) { - - for (var j = 0; j < docs.length; j++) { - - var json = docs[j]; - var cancel = length; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - - if (item.take && item.counter >= item.take) { - cancel--; - continue; - } - - item.count++; - - if (item.skip && item.skip > item.count) - continue; - - item.counter++; - item.response.push(json); - } - - if (!cancel) { - canceled = true; - return; - } - } - }; - - var buf = '['; - var bufcount = 0; - - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { - - if (value[0] !== '{') - return; - - var len = value.length - 1; - - if (self.$documentsize == null) - self.$documentsize = len; - else if (self.$documentsize > len) - self.$documentsize = len; - - buf += (bufcount ? ',' : '') + value.substring(0, len); - bufcount++; - - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = '['; - } - - if (canceled) { - reader.destroy && reader.destroy(); - return false; - } - - }, skip)); - - var finish = function() { - bufcount && processing(JSON.parse(buf + ']', jsonparser)); - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.callback(null, item.response); - } - self.next(0); - }; - - if (reader) - CLEANUP(reader, finish); - else - finish(); + fs.openread(function() { + fs.read(function() { + for (var i = 0; i < length; i++) + filter[i].callback && filter[i].callback(null, filter[i].repository, count); + self.next(0); + fs = null; + }); + }); return self; }; @@ -2053,6 +1893,11 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { Database.prototype.$views = function() { var self = this; + if (!self.views) { + self.next(0); + return; + } + self.step = 5; if (!self.pending_views) { @@ -2077,13 +1922,11 @@ Database.prototype.$views = function() { response.push({ response: [], name: views[i], compare: builder.compile(), filter: { repository: builder.$repository, options: builder.$options, arg: builder.$params, fn: builder.$functions }, builder: builder, count: 0, counter: 0, repository: {} }); } - var reader = Fs.createReadStream(self.filename); - - var buf = '['; - var bufcount = 0; + var fs = NoSQLStream(self.filename); var indexer = 0; - var processing = function(docs) { + fs.ondocuments = function() { + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); for (var i = 0; i < docs.length; i++) { var json = docs[i]; for (var j = 0; j < length; j++) { @@ -2102,66 +1945,45 @@ Database.prototype.$views = function() { } }; - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { - - indexer++; - - if (value[0] !== '{') - return; - - buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); - bufcount++; - - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = '['; - } - - })); - - CLEANUP(reader, function() { + fs.openread(function() { + fs.read(function() { + fs.close(F.errorcallback); + response.wait(function(item, next) { - if (bufcount) { - processing(JSON.parse(buf + ']', jsonparser)); - bufcount = 0; - buf = null; - } - - response.wait(function(item, next) { - - var builder = item.builder; + var builder = item.builder; - if (builder.$options.sort) { - if (builder.$options.sort.name) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - else if (builder.$options.sort === EMPTYOBJECT) - item.response.random(); - else - item.response.sort(builder.$options.sort); - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } + if (builder.$options.sort) { + if (builder.$options.sort.name) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + else if (builder.$options.sort === EMPTYOBJECT) + item.response.random(); + else + item.response.sort(builder.$options.sort); + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } - var filename = builder.$filename; - Fs.unlink(filename, function() { - item.response.limit(20, function(items, next) { - var builder = []; - for (var i = 0, length = items.length; i < length; i++) - builder.push(JSON.stringify(items[i])); - Fs.appendFile(filename, builder.join(NEWLINE) + NEWLINE, next); - }, function() { - // clears in-memory - self.inmemory[item.name] = undefined; - next(); + var filename = builder.$filename; + Fs.unlink(filename, function() { + item.response.limit(20, function(items, next) { + var builder = []; + for (var i = 0, length = items.length; i < length; i++) + builder.push(JSON.stringify(items[i])); + Fs.appendFile(filename, builder.join(NEWLINE) + NEWLINE, next); + }, function() { + // clears in-memory + self.inmemory[item.name] = undefined; + next(); + }); }); - }); - }, () => self.next(0), 5); + }, () => self.next(0), 5); + }); }); + }; Database.prototype.$views_inmemory = function() { @@ -2249,23 +2071,26 @@ Database.prototype.$remove = function() { return; } - var reader = Fs.createReadStream(self.filename); - var writer = Fs.createWriteStream(self.filenameTemp); - + var fs = new NoSQLStream(self.filename); var filter = self.pending_remove.splice(0); var length = filter.length; var change = false; - var buf = []; - var bufcount = 0; var indexer = 0; + var backup = false; for (var i = 0; i < length; i++) { var fil = filter[i]; fil.compare = fil.builder.compile(); fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + if (fil.backup) + backup = true; } - var processing = function(docs, raw) { + fs.ondocuments = function() { + + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); + var is = false; + var deleted = {}; for (var j = 0; j < docs.length; j++) { @@ -2277,9 +2102,9 @@ Database.prototype.$remove = function() { var item = filter[i]; var builder = item.builder; item.filter.index = indexer; - var output = item.compare(json, item.filter, indexer); if (output) { + item.count++; builder.$options.backup && builder.$backupdoc(output); removed = true; json = output; @@ -2288,44 +2113,55 @@ Database.prototype.$remove = function() { } if (removed) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(raw[j] + NEWLINE); - item.count++; - } - self.$events.remove && self.emit('remove', json); - change = true; - } else - writer.write(raw[j] + NEWLINE); - } - }; - reader && reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { + deleted[j] = 1; + var tmp = null; - if (value[0] !== '{') - return; + if (backup) { + for (var i = 0; i < length; i++) { + var item = filter[i]; + if (item.backup) { + if (!tmp) + tmp = JSON.stringify(docs[j]) + NEWLINE; + item.backup.write(tmp); + } + item.count++; + } + } + self.$events.remove && self.emit('remove', json); - buf.push(value.substring(0, value.length - 1)); - bufcount++; + if (!fs.changed) + fs.changed = true; + if (!is) + is = true; - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse('[' + buf.join(',') + ']', jsonparser), buf); - buf = []; + if (self.indexes && self.indexes.indexes.length) + self.indexes.remove(json); + } } - })); + if (is) { + var tmp = ''; + for (var j = 0; j < docs.length; j++) { + if (deleted[j] == null) + tmp += JSON.stringify(docs[j]) + NEWLINE; + } + tmp && fs.write(tmp); + } else + fs.write(fs.docscache); + }; var finish = function() { // No change - if (!change) { - Fs.unlink(self.filenameTemp, function() { + if (!fs.changed) { + fs.flush(function() { for (var i = 0; i < length; i++) { var item = filter[i]; item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } + fs = null; self.next(0); }); return; @@ -2340,7 +2176,8 @@ Database.prototype.$remove = function() { } self.step = 9; - Fs.rename(self.filenameTemp, self.filename, function() { + + fs.flush(function() { for (var i = 0; i < length; i++) { var item = filter[i]; @@ -2351,22 +2188,13 @@ Database.prototype.$remove = function() { setImmediate(function() { self.renaming = false; self.next(0); - change && setImmediate(views_refresh, self); + change && self.views && setImmediate(views_refresh, self); }); }); }; - CLEANUP(writer, finish); - - CLEANUP(reader, function() { - - if (bufcount) { - bufcount = 0; - processing(JSON.parse('[' + buf.join(',') + ']', jsonparser), buf); - buf = null; - } - - writer.end(); + fs.openupdate(function() { + fs.read(finish, true); }); }; @@ -2447,7 +2275,7 @@ Database.prototype.$drop = function() { self.pending_drops = false; var remove = [self.filename, self.filenameTemp]; - Object.keys(self.views).forEach(key => remove.push(self.views[key].$filename)); + self.views && Object.keys(self.views).forEach(key => remove.push(self.views[key].$filename)); try { Fs.readdirSync(self.binary.directory).forEach(function(filename) { @@ -2477,7 +2305,7 @@ DatabaseBuilder2.prototype.log = function(msg, user) { F.datetime = new Date(); self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { - self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.error()); + self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; } return self; @@ -2535,7 +2363,7 @@ DatabaseBuilder.prototype.log = function(msg, user) { F.datetime = new Date(); self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { - self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.error()); + self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; } return self; @@ -2762,7 +2590,7 @@ DatabaseBuilder.prototype.backup = function(user) { }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, F.error()); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, F.errorcallback); return this; }; @@ -4567,7 +4395,7 @@ Indexes.prototype.$index = function(index, value) { var key = ''; var number = false; - var num = value.length > 1 ? 2 : 3; + var num = 2; for (var i = 0; i < value.length; i++) { var val = value[i]; @@ -4576,9 +4404,9 @@ Indexes.prototype.$index = function(index, value) { case 'number': if (index.type === 'reverse') { val = val.toString(); - val = val.substring(val.length - num).padLeft(num); + val = val.substring(val.length - num).padLeft(num, '0'); } else - val = val.toString().substring(0, num).padLeft(num); + val = val.toString().substring(0, num).padLeft(num, '0'); number = true; break; case 'boolean': @@ -4589,9 +4417,9 @@ Indexes.prototype.$index = function(index, value) { val = +val; if (index.type === 'reverse') { val = val.toString(); - val = val.substring(val.length - num).padLeft(num); + val = val.substring(val.length - num).padLeft(num, '0'); } else - val = val.toString().substring(0, num).padLeft(num); + val = val.toString().substring(0, num).padLeft(num, '0'); number = true; } else { if (val.isUID()) { @@ -4730,12 +4558,23 @@ Indexes.prototype.reindex = function(callback) { var self = this; if (self.reindexing) { - callback(new Error('Reindex is running.')); + callback(new Error('Reindexing is running.')); return self; } + if (self.db.step === 1 || self.db.step === 2 || self.db.step === 3 || self.db.step === 7) { + // We need to wait + setTimeout(function(self, callback) { + self.reindex(callback); + }, 500, self, callback); + return self; + } + + self.db.pending_reindex = true; self.reindexing = true; + self.clear(function() { + self.db.$events['indexing-begin'] && self.db.emit('indexing-begin'); var chunker = U.chunker(self.db.name + '_reindex', 10000); self.db.streamer(function(doc) { chunker.write(doc); @@ -4747,7 +4586,9 @@ Indexes.prototype.reindex = function(callback) { self.insert(docs[i], true); self.$reindexingnext = next; }, function() { + self.db.$events['indexing-end'] && self.db.emit('indexing-end'); self.reindexing = false; + self.db.pending_reindex = false; callback && callback(null, count); }); }); @@ -5105,7 +4946,7 @@ Storage.prototype.mapreduce = function(name, fn, def) { Storage.prototype.$mapreducesave = function() { var self = this; - Fs.writeFile(self.$mapreducefile, JSON.stringify(self.$mapreduce, (k, v) => k !== 'reduce' ? v : undefined), F.error()); + Fs.writeFile(self.$mapreducefile, JSON.stringify(self.$mapreduce, (k, v) => k !== 'reduce' ? v : undefined), F.errorcallback); return self; }; @@ -5209,73 +5050,35 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { return; } - var reader = Fs.createReadStream(item.filename); + var reader = new NoSQLStream(item.filename); stats.current = item.date; stats.index = index; - var buf = '['; - var bufcount = 0; - var canceled = false; - - var processing = function(docs) { + reader.ondocuments = function() { + var docs = JSON.parse('[' + reader.docs + ']', jsonparser); for (var j = 0; j < docs.length; j++) { stats.documents++; var json = docs[j]; var end = mapreduce(json, repository, stats) === false; if (end) { - canceled = true; - return; + stats.canceled = true; + return false; } } }; - reader.on('data', framework_utils.streamer(NEWLINEBUF, function(value) { - - if (value[0] !== '{') - return; - - buf += (bufcount ? ',' : '') + value.substring(0, value.length - 1); - bufcount++; - - if (bufcount % JSONBUFFER === 0) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = '['; - } - - if (canceled) { - stats.canceled = true; - reader.destroy && reader.destroy(); - return false; - } - - })); - - var finish = function() { - - if (bufcount) { - bufcount = 0; - processing(JSON.parse(buf + ']', jsonparser)); - buf = null; - } - - stats.processed++; - - if (item.date === today) { - self.locked_writer--; - if (self.locked_writer <= 0 && self.pending.length) - self.insert(); - } - - setImmediate(next); - }; - - reader.on('error', finish); + reader.openread(function() { + reader.read(function() { + stats.processed++; + if (item.date === today) { + self.locked_writer--; + if (self.locked_writer <= 0 && self.pending.length) + self.insert(); + } + setImmediate(next); + }); + }); - if (reader) - CLEANUP(reader, finish); - else - finish(); }; storage.wait(function(item, next, index) { diff --git a/nosqlstream.js b/nosqlstream.js new file mode 100644 index 000000000..63708163a --- /dev/null +++ b/nosqlstream.js @@ -0,0 +1,445 @@ +// Copyright 2012-2018 (c) Peter Širka +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +/** + * @module NoSQL Stream + * @version 1.0.0 + */ + +require('./index'); + +const Fs = require('fs'); +const BUFFERSIZE = 1024 * 16; +const BUFFERDOCS = 10; +const NEWLINEBUFFER = framework_utils.createBuffer('\n', 'utf8'); +const NEWLINE = '\n'; +const DEFSTATS = { size: 0 }; + +function NoSQLStream(filename) { + this.filename = filename; + this.fd = null; + this.stats = DEFSTATS; + this.type = null; + this.bytesread = 0; + this.position = 0; + this.cache = [null, null]; + this.buffer = null; + // this.canceled = false; + // this.docs = ''; + // this.docscount = 0; + // this.docscache = ''; + // this.indexer = 0; + // this.fdwrite = null; +} + +NoSQLStream.prototype.openread = function(callback) { + var self = this; + self.type = 'r'; + self.position = 0; + self.open(callback); + return self; +}; + +NoSQLStream.prototype.openupdate = function(callback) { + var self = this; + self.type = 'r'; + self.open(function(err) { + + // File may not exist + if (err) { + callback(); + return; + } + + self.position = 0; + self.bufferstack = []; + self.changed = false; + + Fs.open(self.filename + '-tmp', 'w', function(err, fd) { + + if (err) { + self.close(NOOP); + throw err; + } + + self.fdwrite = fd; + callback && callback(); + }); + }); + + return self; +}; + +NoSQLStream.prototype.openinsert = function(callback) { + var self = this; + self.type = 'a'; + self.open(callback); + return self; +}; + +// For e.g. files on URL address +NoSQLStream.prototype.openstream = function(stream, callback) { + var self = this; + + self.docs = ''; + self.docscache = ''; + self.docscount = 0; + + self.stream = stream; + self.stream.on('end', function() { + if (self.docscount) { + self.ondocuments(); + self.docscount = 0; + self.docs = ''; + self.docscache = ''; + } + callback && callback(); + }); + + self.stream.on('data', function(chunk) { + + var beg = 0; + + if (self.buffer) { + self.cache[0] = self.buffer; + self.cache[1] = chunk; + self.buffer = Buffer.concat(self.cache); + + beg = self.buffer.length - 1; + + if (beg < 0) + beg = 0; + + } else + self.buffer = chunk; + + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + self.docs += (self.docs ? ',' : '') + self.buffer.toString('utf8', 0, index); + self.docscount++; + self.indexer++; + + if (self.docscount >= BUFFERDOCS) { + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = ''; + self.docscount = 0; + + if (self.canceled) { + self.close(); + self.stream.destroy && self.stream.destroy(); + return; + } + + } + + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } + }); + + return self; +}; + +NoSQLStream.prototype.open = function(callback) { + var self = this; + Fs.open(self.filename, self.type, function(err, fd) { + + if (err) { + callback && callback.call(err); + return; + } + + Fs.fstat(fd, function(err, stats) { + + self.docs = ''; + self.docscache = ''; + self.docscount = 0; + self.fd = fd; + self.stats = stats; + + // Appending + if (self.type === 'a') + self.position = stats.size; + else + self.position = 0; + + callback && callback(err); + }); + }); +}; + +NoSQLStream.prototype.close = function(callback) { + + var self = this; + + if (self.fd) { + + self.stream = null; + + Fs.close(self.fd, function(err) { + err && F.error(err); + callback && callback(); + }); + + if (self.buffer) { + self.buffer = null; + self.cache[0] = null; + self.cache[1] = null; + self.bytesread = 0; + } + + self.canceled = false; + self.fd = null; + self.type = null; + + } else if (callback) + callback(); + + return self; +}; + +NoSQLStream.prototype.write = function(docs) { + var self = this; + self.bufferstack.push(docs); + !self.writing && self.$write(); + return self; +}; + +NoSQLStream.prototype.$write = function() { + var self = this; + self.writing = true; + //var buf = framework_utils.createBuffer(self.bufferstack.shift(), 'utf8'); + Fs.write(self.fdwrite, self.bufferstack.shift(), 'utf8', function() { + self.bufferstack.length && self.$write(); + self.writing = false; + }); +}; + +NoSQLStream.prototype.flush = function(callback) { + var self = this; + if (self.writing) { + setTimeout((self, callback) => self.flush(callback), 100, self, callback); + } else { + self.close(function(err) { + + if (err) { + callback(err); + return; + } + + Fs.close(self.fdwrite, function(err) { + if (err) + callback(err); + else { + if (self.changed) + Fs.rename(self.filename + '-tmp', self.filename, callback); + else + Fs.unlink(self.filename + '-tmp', callback); + } + }); + + }); + } + return self; +}; + +NoSQLStream.prototype.read = function(callback, noclose) { + + var self = this; + var size = self.stats.size - self.position; + + if (!self.fd || size <= 0 || self.canceled) { + + if (self.docscount) { + self.ondocuments(); + self.docscount = 0; + self.docs = ''; + + if (self.fdwrite) + self.docscache = ''; + + } + + if (noclose) + callback && callback(); + else + self.close(err => callback && callback(err)); + + return; + } + + size = size < BUFFERSIZE ? size : BUFFERSIZE; + var buffer = framework_utils.createBufferSize(size); + Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { + + self.bytesread += size; + self.position += size; + + var beg = 0; + + if (self.buffer) { + self.cache[0] = self.buffer; + self.cache[1] = chunk; + + beg = self.buffer.length - 1; + + if (beg < 0) + beg = 0; + + self.buffer = Buffer.concat(self.cache); + + } else + self.buffer = chunk; + + + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index); + self.docs += (self.docs ? ',' : '') + tmp; + self.docscount++; + self.indexer++; + + if (self.fdwrite) + self.docscache += tmp + NEWLINE; + + if (self.docscount >= BUFFERDOCS) { + + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = ''; + self.docscount = 0; + + if (self.fdwrite) + self.docscache = ''; + + if (self.canceled) { + self.read(callback, noclose); + return; + } + + } + + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } + + self.read(callback, noclose); + }); +}; + +NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { + + var self = this; + + if (repeat == null) + self.position = self.stats.size; + + if (!self.fd || self.position <= 0 || self.canceled) { + + if (self.docscount) { + self.ondocuments(); + self.docs = ''; + self.docscount = 0; + } + + if (noclose) + callback && callback(); + else + self.close(err => callback && callback(err)); + + return; + } + + var size = self.stats.size - self.bytesread; + size = size < BUFFERSIZE ? size : BUFFERSIZE; + + self.position -= size; + + var buffer = framework_utils.createBufferSize(size); + + Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { + + self.bytesread += size; + + if (self.buffer) { + self.cache[0] = chunk; + self.cache[1] = self.buffer; + self.buffer = Buffer.concat(self.cache); + } else + self.buffer = chunk; + + var index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + + while (index !== -1) { + + self.docs += (self.docs ? ',' : '') + self.buffer.toString('utf8', index); + self.docscount++; + + if (self.docscount >= BUFFERDOCS) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = ''; + self.docscount = 0; + } + + if (self.canceled) { + self.readreverse(callback, noclose, true); + return; + } + + self.buffer = self.buffer.slice(0, index); + index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + if (index === -1) + break; + } + + if (self.position === 0 && self.buffer.length) { + self.docs += (self.docs ? ',' : '') + self.buffer.toString('utf8'); + self.docscount++; + if (self.docscount >= BUFFERDOCS) { + self.ondocuments(); + self.docs = ''; + self.docscount = 0; + } + } + + self.readreverse(callback, noclose, true); + }); +}; + +NoSQLStream.prototype.append = function(data, callback) { + var self = this; + Fs.write(self.fd, data, 'utf8', function(err) { + callback && callback(err); + }); + return self; +}; + +module.exports = NoSQLStream; \ No newline at end of file diff --git a/nosqlworker.js b/nosqlworker.js index f9493ffa3..37b2992c7 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -1,3 +1,29 @@ +// Copyright 2012-2018 (c) Peter Širka +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +/** + * @module NoSQL Worker + * @version 1.0.0 + */ + require(module.filename.replace(/nosqlworker\.js$/, 'index.js')); const RESFIND = { TYPE: 'find' }; From 06fb8a438bcf42ae7f6a13c02fce1900623d1e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 14:38:11 +0200 Subject: [PATCH 0186/1669] Improved `U.streamer()`. --- utils.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 03887c8e5..3ccb7f02d 100755 --- a/utils.js +++ b/utils.js @@ -1642,9 +1642,18 @@ exports.streamer = function(beg, end, callback, skip, stream, raw) { CONCAT[0] = buffer; CONCAT[1] = chunk; + + var f = 0; + + if (buffer.length) { + f = buffer.length - beg.length; + if (f < 0) + f = 0; + } + buffer = Buffer.concat(CONCAT); - var index = buffer.indexOf(beg); + var index = buffer.indexOf(beg, f); if (index === -1) return; @@ -1687,7 +1696,10 @@ exports.streamer = function(beg, end, callback, skip, stream, raw) { buffer = Buffer.concat(CONCAT); if (!is) { - bi = buffer.indexOf(beg); + var f = CONCAT[0].length - beg.length; + if (f < 0) + f = 0; + bi = buffer.indexOf(beg, f); if (bi === -1) return; is = true; From f5104ab8fbb7f57febed21fefe12ab6e3d4a5000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 14:39:48 +0200 Subject: [PATCH 0187/1669] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index c183c2ec2..18203779e 100755 --- a/changes.txt +++ b/changes.txt @@ -73,6 +73,7 @@ - improved: `debug` mode timing with improved consumption - improved: performance (+20%) NoSQL embedded database +- improved: reading performance (+5%) in `U.streamer()` ======= 2.9.4 (HOTFIX) From ce47a49fa621b607dde119d9e0d36551bd3a68dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 15:49:57 +0200 Subject: [PATCH 0188/1669] Fixed `reindexing`. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 9bc2aa606..b1fb49bcb 100755 --- a/nosql.js +++ b/nosql.js @@ -4758,7 +4758,7 @@ Indexes.prototype.flush = function() { self.changes.length && setImmediate(() => self.flush()); }; - var arr = self.changes.splice(0); + var arr = self.changes.splice(0, 50); for (var i = 0; i < arr.length; i++) { From 9473fa39273c8b1d6a244ec89b0f734e98aa40fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 18:06:41 +0200 Subject: [PATCH 0189/1669] Improved `res.binary()`. --- index.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/index.js b/index.js index cb6e45f54..085d539fd 100755 --- a/index.js +++ b/index.js @@ -14981,6 +14981,19 @@ function extend_response(PROTO) { }; PROTO.binary = function(body, type, encoding, download, headers) { + + if (typeof(encoding) === 'object') { + var tmp = encoding; + encoding = download; + download = headers; + headers = tmp; + } + + if (typeof(download) === 'object') { + headers = download; + download = headers; + } + this.options.type = type; this.options.body = body; this.options.encoding = encoding; From 32d6b58fe11463d64b7b210dec0d82e386188438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 20:22:01 +0200 Subject: [PATCH 0190/1669] Improved indexing + fixed new bugs in NoSQL. --- index.js | 4 +++ nosql.js | 79 ++++++++++++++++++++++++++++++++++++++++++++++++-- nosqlworker.js | 3 ++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 085d539fd..7765ddcd7 100755 --- a/index.js +++ b/index.js @@ -3136,6 +3136,10 @@ F.problem = F.wtf = function(message, name, uri, ip) { return F; }; +global.PRINTLN = function(msg) { + console.log('------>', '[' + new Date().format('yyyy-MM-dd HH:mm:ss') + ']', msg); +}; + /** * Registers a new change * @param {String} message diff --git a/nosql.js b/nosql.js index b1fb49bcb..fc2e49f41 100755 --- a/nosql.js +++ b/nosql.js @@ -50,6 +50,7 @@ const EXTENSION_MAPREDUCE = '.nosql-mapreduce'; const EXTENSION_BACKUP = '.nosql-backup'; const EXTENSION_META = '.meta'; const EXTENSION_COUNTER = '-counter2'; +const EXTENSION_INDEXES = '-indexes'; const BINARY_HEADER_LENGTH = 2000; const NEWLINE = '\n'; const EMPTYARRAY = []; @@ -218,7 +219,7 @@ exports.worker = function() { var CP = Counter.prototype; DP.once = DP.on = DP.emit = DP.removeListener = DP.removeAllListeners = CP.on = CP.once = CP.emit = CP.removeListener = CP.removeAllListeners = function() { - console.log('ERROR --> NoSQL events are not supported in fork mode.'); + PRINTLN('ERROR --> NoSQL events are not supported in fork mode.'); }; Database.prototype.find = function(view) { @@ -434,6 +435,11 @@ exports.worker = function() { return this; }; + Indexes.prototype.noreindex = function() { + notify(this.db, 'indexes.noreindex'); + return this; + }; + }; function Database(name, filename, readonly) { @@ -4378,16 +4384,52 @@ function Indexes(db, directory) { this.flushing = false; this.instances = {}; this.reindexing = false; + this.meta = { $version: 1 }; + try { + this.meta = Fs.readFileSync(this.db.filename + EXTENSION_INDEXES).toString('utf8').parseJSON(true) || {}; + } catch (e) {} } Indexes.prototype.create = function(name, properties, type) { + var self = this; + if (typeof(properties) === 'string') { type = properties; properties = null; } - !this.indexes.findItem('name', name) && this.indexes.push({ name: name, properties: properties ? properties : [name], type: type }); + var prop = properties ? properties : [name]; + var key = prop.join(',') + (type ? ('=' + type) : ''); + + !self.indexes.findItem('name', name) && self.indexes.push({ name: name, properties: prop, type: type }); + + var meta = self.meta[name]; + var reindex = false; + + if (meta) { + if (meta.key !== key) { + reindex = true; + meta.key = key; + } + } else { + self.meta[name] = { key: key, documents: 0 }; + reindex = true; + } + + reindex && setTimeout2(self.db.name + '_reindex', () => self.reindex(), 1000); + return self; +}; + +Indexes.prototype.noreindex = function() { + var self = this; + clearTimeout2(self.db.name + '_reindex'); + return self; +}; + +Indexes.prototype.$meta = function() { + var self = this; + Fs.writeFile(self.db.filename + EXTENSION_INDEXES, JSON.stringify(this.meta), NOOP); return this; }; @@ -4557,8 +4599,10 @@ Indexes.prototype.clear = function(callback) { Indexes.prototype.reindex = function(callback) { var self = this; + clearTimeout2(self.db.name + '_reindex'); + if (self.reindexing) { - callback(new Error('Reindexing is running.')); + callback && callback(new Error('Re-indexing is running.')); return self; } @@ -4572,6 +4616,20 @@ Indexes.prototype.reindex = function(callback) { self.db.pending_reindex = true; self.reindexing = true; + var now = Date.now(); + + PRINTLN('NoSQL embedded "{0}" re-indexing (beg)'.format(self.db.name)); + + var keys = Object.keys(self.meta); + + for (var i = 0; i < self.indexes.length; i++) + self.meta[self.indexes[i].name].documents = 0; + + // Clears non-exist indexes + for (var i = 0; i < keys.length; i++) { + if (self.indexes.findItem('name', keys[i]) == null) + delete self.meta[keys[i]]; + } self.clear(function() { self.db.$events['indexing-begin'] && self.db.emit('indexing-begin'); @@ -4586,10 +4644,13 @@ Indexes.prototype.reindex = function(callback) { self.insert(docs[i], true); self.$reindexingnext = next; }, function() { + self.$meta(); self.db.$events['indexing-end'] && self.db.emit('indexing-end'); self.reindexing = false; self.db.pending_reindex = false; + self.db.next(0); callback && callback(null, count); + PRINTLN('NoSQL embedded "{0}" re-indexing (end, {1}s)'.format(self.db.name, (((Date.now() - now) / 1000) >> 0))); }); }); }); @@ -4655,7 +4716,9 @@ Indexes.prototype.insert = function(doc, reindex) { var values = self.makeindex(index, doc); if (values.length) { + var key = self.$index(index, values); + if (!key) continue; @@ -4784,7 +4847,12 @@ Indexes.prototype.flush = function() { builder.where(item.properties[j], item.value[j]); } else if (item.insert) { + count++; + + if (self.meta[item.name]) + self.meta[item.name].documents++; + self.instances[item.key].insert(item.doc).callback(function() { if (self.instances[item.key].PENDING) self.instances[item.key].PENDING--; @@ -4793,7 +4861,12 @@ Indexes.prototype.flush = function() { }); } else { + count++; + + if (self.meta[item.name]) + self.meta[item.name].documents--; + var builder = self.instances[item.key].remove().callback(function() { if (self.instances[item.key].PENDING) self.instances[item.key].PENDING--; diff --git a/nosqlworker.js b/nosqlworker.js index 37b2992c7..49a3b87f7 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -249,5 +249,8 @@ process.on('message', function(msg) { process.send(RESINDEXESREINDEX); }); break; + case 'indexes.noreindex': + db.indexes.noreindex(); + break; } }); \ No newline at end of file From 190490896787d5f1dfb4a282a8999da30dfcbd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 20:26:11 +0200 Subject: [PATCH 0191/1669] Fixed views. --- nosql.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index fc2e49f41..378d3b79f 100755 --- a/nosql.js +++ b/nosql.js @@ -1928,7 +1928,7 @@ Database.prototype.$views = function() { response.push({ response: [], name: views[i], compare: builder.compile(), filter: { repository: builder.$repository, options: builder.$options, arg: builder.$params, fn: builder.$functions }, builder: builder, count: 0, counter: 0, repository: {} }); } - var fs = NoSQLStream(self.filename); + var fs = new NoSQLStream(self.filename); var indexer = 0; fs.ondocuments = function() { @@ -1953,7 +1953,6 @@ Database.prototype.$views = function() { fs.openread(function() { fs.read(function() { - fs.close(F.errorcallback); response.wait(function(item, next) { var builder = item.builder; From d4c135db517c3469589313a47eb2b20b5905c2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 21:27:12 +0200 Subject: [PATCH 0192/1669] Fixed cleaning NoSQL storage. --- nosql.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index 378d3b79f..251089f43 100755 --- a/nosql.js +++ b/nosql.js @@ -443,6 +443,7 @@ exports.worker = function() { }; function Database(name, filename, readonly) { + var self = this; var http = filename.substring(0, 6); self.readonly = http === 'http:/' || http === 'https:'; @@ -1601,7 +1602,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { }; var cb = function() { - fs.read(function() { + fs.readreverse(function() { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; @@ -4393,7 +4394,7 @@ Indexes.prototype.create = function(name, properties, type) { var self = this; - if (typeof(properties) === 'string') { + if (type == null && typeof(properties) === 'string') { type = properties; properties = null; } @@ -4443,7 +4444,9 @@ Indexes.prototype.$index = function(index, value) { switch (typeof(val)) { case 'number': - if (index.type === 'reverse') { + if (index.type === 'first') + val = val.toString()[0]; + else if (index.type === 'reverse') { val = val.toString(); val = val.substring(val.length - num).padLeft(num, '0'); } else @@ -4456,7 +4459,9 @@ Indexes.prototype.$index = function(index, value) { case 'string': if (REGNUMBER.test(val)) { val = +val; - if (index.type === 'reverse') { + if (index.type === 'first') + val = val.toString()[0]; + else if (index.type === 'reverse') { val = val.toString(); val = val.substring(val.length - num).padLeft(num, '0'); } else @@ -4470,7 +4475,9 @@ Indexes.prototype.$index = function(index, value) { val = val.toLowerCase().removeDiacritics().match(REGINDEXCHAR); if (val) { val = val.toString(); - if (index.type === 'reverse') + if (index.type === 'first') + val = val[0]; + else if (index.type === 'reverse') val = val.substring(val.length - 2); else val = val.substring(0, 2); @@ -5213,7 +5220,7 @@ Storage.prototype.clear = function(beg, end, callback) { }); }; - files.wait((filename, next) => remove(filename, next), function() { + files.wait((item, next) => remove(item.filename, next), function() { remove(self.$mapreducefile, () => callback && callback(null, count)); }); From b130df925f94352394d31cefdf820fbe8688a172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 21:27:16 +0200 Subject: [PATCH 0193/1669] Improved reading. --- nosqlstream.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nosqlstream.js b/nosqlstream.js index 63708163a..2ec41c830 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -27,8 +27,8 @@ require('./index'); const Fs = require('fs'); -const BUFFERSIZE = 1024 * 16; -const BUFFERDOCS = 10; +const BUFFERSIZE = 1024 * 32; +const BUFFERDOCS = 15; const NEWLINEBUFFER = framework_utils.createBuffer('\n', 'utf8'); const NEWLINE = '\n'; const DEFSTATS = { size: 0 }; @@ -123,7 +123,7 @@ NoSQLStream.prototype.openstream = function(stream, callback) { self.cache[1] = chunk; self.buffer = Buffer.concat(self.cache); - beg = self.buffer.length - 1; + beg = self.cache[0].length - 1; if (beg < 0) beg = 0; @@ -134,11 +134,14 @@ NoSQLStream.prototype.openstream = function(stream, callback) { var index = self.buffer.indexOf(NEWLINEBUFFER, beg); while (index !== -1) { - self.docs += (self.docs ? ',' : '') + self.buffer.toString('utf8', 0, index); + var tmp = self.buffer.toString('utf8', 0, index).trim(); + + self.docs += (self.docs ? ',' : '') + tmp; self.docscount++; self.indexer++; if (self.docscount >= BUFFERDOCS) { + if (self.ondocuments() === false) self.canceled = true; From aaa6453dfc4ef12060da9eced018079f9baae52c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 22:00:14 +0200 Subject: [PATCH 0194/1669] Fixed updating of indexes. --- nosql.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nosql.js b/nosql.js index 251089f43..63a844ff1 100755 --- a/nosql.js +++ b/nosql.js @@ -4469,7 +4469,7 @@ Indexes.prototype.$index = function(index, value) { number = true; } else { if (val.isUID()) { - val = val.substring(4, 6) + val.substring(2, 4) + val.substring(0, 2); + val = val.substring(0, 2) + val.substring(2, 4); number = true; } else { val = val.toLowerCase().removeDiacritics().match(REGINDEXCHAR); @@ -4756,16 +4756,19 @@ Indexes.prototype.update = function(doc, old) { var key = self.$index(index, values); if (!key) continue; + + var oldvalues = self.makeindex(index, old); + var oldkey = self.$index(index, oldvalues); + var item = self.findchanges(index, key, values); if (item) item.doc = doc; - else { - var oldvalues = self.makeindex(index, old); - var oldkey = self.$index(index, oldvalues); + else self.changes.push({ update: true, key: key, doc: doc, name: index.name, properties: index.properties, value: values }); - if (oldkey !== key && oldkey) - self.changes.push({ remove: true, key: oldkey, name: index.name, properties: index.properties, value: oldvalues }); - } + + if (oldkey !== key && oldkey) + self.changes.push({ remove: true, key: oldkey, name: index.name, properties: index.properties, value: oldvalues }); + } } From c69af48924a4164cbad5f456efebb9d7a6ba598e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Apr 2018 22:14:59 +0200 Subject: [PATCH 0195/1669] Extended `F.prototypes()`. --- index.js | 2 ++ nosql.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/index.js b/index.js index 7765ddcd7..2284c5e23 100755 --- a/index.js +++ b/index.js @@ -899,6 +899,8 @@ F.prototypes = function(fn) { proto.DatabaseBuilder = framework_nosql.DatabaseBuilder.prototype; proto.DatabaseBuilder2 = framework_nosql.DatabaseBuilder2.prototype; proto.DatabaseCounter = framework_nosql.DatabaseCounter.prototype; + proto.DatabaseIndexes = framework_nosql.DatabaseIndexes.prototype; + proto.DatabaseStorage = framework_nosql.DatabaseStorage.prototype; proto.ErrorBuilder = framework_builders.ErrorBuilder.prototype; proto.HttpFile = framework_internal.HttpFile.prototype; proto.HttpRequest = PROTOREQ; diff --git a/nosql.js b/nosql.js index 63a844ff1..6690fffcf 100755 --- a/nosql.js +++ b/nosql.js @@ -551,6 +551,8 @@ exports.DatabaseBuilder = DatabaseBuilder; exports.DatabaseBuilder2 = DatabaseBuilder2; exports.DatabaseCounter = Counter; exports.DatabaseBinary = Binary; +exports.DatabaseIndexes = Indexes; +exports.DatabaseStorage = Storage; exports.load = function(name, filename) { return new Database(name, filename); From 2af39aedd3bf6ae4a591429f283aab4f5fd1a312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Apr 2018 12:41:44 +0200 Subject: [PATCH 0196/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a2c0f14ad..0716dba42 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-39", + "version": "3.0.0-40", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 25cf43d165d384a2643c256c5e1ca3fcffb0d15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Apr 2018 20:06:31 +0200 Subject: [PATCH 0197/1669] Improved `U.get()` and `U.set()`. --- test/test-utils.js | 8 +++-- utils.js | 73 ++++++++++++++++++++++++++-------------------- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/test/test-utils.js b/test/test-utils.js index 3f602fe2c..23ff8004d 100755 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -397,6 +397,10 @@ function other() { utils.copy({ name: 'A', age: -1 }, obj); assert.ok(obj.name === 'A' && obj.age === -1, 'utils.copy(rewrite=true)'); + assert.ok(U.get(obj, 'arr').join(',') === '1,2,3,4', 'utils.get()'); + U.set(obj, 'address.city', 'Banská Bystrica'); + assert.ok(obj.address.city === 'Banská Bystrica', 'utils.set()'); + var a = utils.reduce(obj, ['name']); var b = utils.reduce(obj, ['name'], true); @@ -627,7 +631,7 @@ function other() { function Utils_Ls2_StringFilter() { var result; - var async = new Utils.Async(); + var async = new U.Async(); async.await('U.ls2', function(next) { U.ls2( @@ -650,7 +654,7 @@ function Utils_Ls2_StringFilter() { function Utils_Ls_RegExpFilter() { var result; - var async = new Utils.Async(); + var async = new U.Async(); async.await('U.ls', function(next) { U.ls( diff --git a/utils.js b/utils.js index 3ccb7f02d..d5a87a2b3 100755 --- a/utils.js +++ b/utils.js @@ -79,6 +79,7 @@ const NODEVERSION = parseFloat(process.version.toString().replace('v', '').repla const STREAMPIPE = { end: false }; const CT = 'Content-Type'; const CRC32TABLE = '00000000,77073096,EE0E612C,990951BA,076DC419,706AF48F,E963A535,9E6495A3,0EDB8832,79DCB8A4,E0D5E91E,97D2D988,09B64C2B,7EB17CBD,E7B82D07,90BF1D91,1DB71064,6AB020F2,F3B97148,84BE41DE,1ADAD47D,6DDDE4EB,F4D4B551,83D385C7,136C9856,646BA8C0,FD62F97A,8A65C9EC,14015C4F,63066CD9,FA0F3D63,8D080DF5,3B6E20C8,4C69105E,D56041E4,A2677172,3C03E4D1,4B04D447,D20D85FD,A50AB56B,35B5A8FA,42B2986C,DBBBC9D6,ACBCF940,32D86CE3,45DF5C75,DCD60DCF,ABD13D59,26D930AC,51DE003A,C8D75180,BFD06116,21B4F4B5,56B3C423,CFBA9599,B8BDA50F,2802B89E,5F058808,C60CD9B2,B10BE924,2F6F7C87,58684C11,C1611DAB,B6662D3D,76DC4190,01DB7106,98D220BC,EFD5102A,71B18589,06B6B51F,9FBFE4A5,E8B8D433,7807C9A2,0F00F934,9609A88E,E10E9818,7F6A0DBB,086D3D2D,91646C97,E6635C01,6B6B51F4,1C6C6162,856530D8,F262004E,6C0695ED,1B01A57B,8208F4C1,F50FC457,65B0D9C6,12B7E950,8BBEB8EA,FCB9887C,62DD1DDF,15DA2D49,8CD37CF3,FBD44C65,4DB26158,3AB551CE,A3BC0074,D4BB30E2,4ADFA541,3DD895D7,A4D1C46D,D3D6F4FB,4369E96A,346ED9FC,AD678846,DA60B8D0,44042D73,33031DE5,AA0A4C5F,DD0D7CC9,5005713C,270241AA,BE0B1010,C90C2086,5768B525,206F85B3,B966D409,CE61E49F,5EDEF90E,29D9C998,B0D09822,C7D7A8B4,59B33D17,2EB40D81,B7BD5C3B,C0BA6CAD,EDB88320,9ABFB3B6,03B6E20C,74B1D29A,EAD54739,9DD277AF,04DB2615,73DC1683,E3630B12,94643B84,0D6D6A3E,7A6A5AA8,E40ECF0B,9309FF9D,0A00AE27,7D079EB1,F00F9344,8708A3D2,1E01F268,6906C2FE,F762575D,806567CB,196C3671,6E6B06E7,FED41B76,89D32BE0,10DA7A5A,67DD4ACC,F9B9DF6F,8EBEEFF9,17B7BE43,60B08ED5,D6D6A3E8,A1D1937E,38D8C2C4,4FDFF252,D1BB67F1,A6BC5767,3FB506DD,48B2364B,D80D2BDA,AF0A1B4C,36034AF6,41047A60,DF60EFC3,A867DF55,316E8EEF,4669BE79,CB61B38C,BC66831A,256FD2A0,5268E236,CC0C7795,BB0B4703,220216B9,5505262F,C5BA3BBE,B2BD0B28,2BB45A92,5CB36A04,C2D7FFA7,B5D0CF31,2CD99E8B,5BDEAE1D,9B64C2B0,EC63F226,756AA39C,026D930A,9C0906A9,EB0E363F,72076785,05005713,95BF4A82,E2B87A14,7BB12BAE,0CB61B38,92D28E9B,E5D5BE0D,7CDCEFB7,0BDBDF21,86D3D2D4,F1D4E242,68DDB3F8,1FDA836E,81BE16CD,F6B9265B,6FB077E1,18B74777,88085AE6,FF0F6A70,66063BCA,11010B5C,8F659EFF,F862AE69,616BFFD3,166CCF45,A00AE278,D70DD2EE,4E048354,3903B3C2,A7672661,D06016F7,4969474D,3E6E77DB,AED16A4A,D9D65ADC,40DF0B66,37D83BF0,A9BCAE53,DEBB9EC5,47B2CF7F,30B5FFE9,BDBDF21C,CABAC28A,53B39330,24B4A3A6,BAD03605,CDD70693,54DE5729,23D967BF,B3667A2E,C4614AB8,5D681B02,2A6F2B94,B40BBE37,C30C8EA1,5A05DF1B,2D02EF8D'.split(',').map(s => parseInt(s, 16)); +const REGISARR = /\[\d+\]$/; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -5425,63 +5426,73 @@ exports.parseTheme = function(value) { return value === '?' ? F.config['default-theme'] : value; }; -exports.set = function(obj, path, value) { +global.SET = exports.set = function(obj, path, value) { var cachekey = 'S+' + path; if (F.temporary.other[cachekey]) return F.temporary.other[cachekey](obj, value); - var arr = path.split('.'); + var arr = parsepath(path); var builder = []; - var p = ''; - - for (var i = 0, length = arr.length; i < length; i++) { - p += (p !== '' ? '.' : '') + arr[i]; - var type = arr[i] instanceof Array ? '[]' : '{}'; - if (i !== length - 1) { - builder.push('if(typeof(w.' + p + ')!=="object"||w.' + p + '===null)w.' + p + '=' + type); - continue; - } - - if (type === '{}') - break; - - p = p.substring(0, p.lastIndexOf('[')); - builder.push('if(!(w.' + p + ' instanceof Array))w.' + p + '=' + type); - break; + for (var i = 0; i < arr.length - 1; i++) { + var type = arr[i + 1] ? (REGISARR.test(arr[i + 1]) ? '[]' : '{}') : '{}'; + builder.push('if(typeof(' + arr[i] + ')!==\'object\'||' + arr[i] + '==null)w.' + arr[i] + '=' + type + ';'); } - var fn = (new Function('w', 'a', 'b', builder.join(';') + ';w.' + path.replace(/'/, '\'') + '=a;return a')); + var fn = (new Function('w', 'a', 'b', builder.join(';') + ';var v=typeof(a)===\'function\'?a(MAIN.compiler.get(b)):a;w.' + arr[arr.length - 1] + '=v;return v')); F.temporary.other[cachekey] = fn; fn(obj, value, path); }; -exports.get = function(obj, path) { +global.GET = exports.get = function(obj, path) { var cachekey = 'G=' + path; if (F.temporary.other[cachekey]) return F.temporary.other[cachekey](obj); - var arr = path.split('.'); + var arr = parsepath(path); var builder = []; - var p = ''; - for (var i = 0, length = arr.length - 1; i < length; i++) { - var tmp = arr[i]; - var index = tmp.lastIndexOf('['); - if (index !== -1) - builder.push('if(!w.' + (p ? p + '.' : '') + tmp.substring(0, index) + ')return'); - p += (p !== '' ? '.' : '') + arr[i]; - builder.push('if(!w.' + p + ')return'); - } + for (var i = 0, length = arr.length - 1; i < length; i++) + builder.push('if(!w.' + arr[i] + ')return'); - var fn = (new Function('w', builder.join(';') + ';return w.' + path.replace(/'/, '\''))); + var fn = (new Function('w', builder.join(';') + ';return w.' + arr[arr.length - 1])); F.temporary.other[cachekey] = fn; return fn(obj); }; +function parsepath(path) { + + var arr = path.split('.'); + var builder = []; + var all = []; + + for (var i = 0; i < arr.length; i++) { + var p = arr[i]; + var index = p.indexOf('['); + if (index === -1) { + if (p.indexOf('-') === -1) { + all.push(p); + builder.push(all.join('.')); + } else { + var a = all.splice(all.length - 1); + all.push(a + '[\'' + p + '\']'); + builder.push(all.join('.')); + } + } else { + all.push(p.substring(0, index)); + builder.push(all.join('.')); + all.splice(all.length - 1); + all.push(p); + builder.push(all.join('.')); + } + } + + return builder; +} + global.Async = global.async = exports.async; global.sync = global.SYNCHRONIZE = exports.sync; global.sync2 = exports.sync2; From 141013c4e89152d3cd1fc048a495042330dcee30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Apr 2018 20:50:43 +0200 Subject: [PATCH 0198/1669] Fixed `U.get()` and `U.set()` again. --- utils.js | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/utils.js b/utils.js index d5a87a2b3..34f382cb6 100755 --- a/utils.js +++ b/utils.js @@ -5437,10 +5437,12 @@ global.SET = exports.set = function(obj, path, value) { for (var i = 0; i < arr.length - 1; i++) { var type = arr[i + 1] ? (REGISARR.test(arr[i + 1]) ? '[]' : '{}') : '{}'; - builder.push('if(typeof(' + arr[i] + ')!==\'object\'||' + arr[i] + '==null)w.' + arr[i] + '=' + type + ';'); + var p = 'w' + (arr[i][0] === '[' ? '' : '.') + arr[i]; + builder.push('if(typeof(' + p + ')!==\'object\'||' + p + '==null)' + p + '=' + type + ';'); } - var fn = (new Function('w', 'a', 'b', builder.join(';') + ';var v=typeof(a)===\'function\'?a(MAIN.compiler.get(b)):a;w.' + arr[arr.length - 1] + '=v;return v')); + var v = arr[arr.length - 1]; + var fn = (new Function('w', 'a', 'b', builder.join(';') + ';var v=typeof(a)===\'function\'?a(MAIN.compiler.get(b)):a;w' + (v[0] === '[' ? '' : '.') + v + '=v;return v')); F.temporary.other[cachekey] = fn; fn(obj, value, path); }; @@ -5456,9 +5458,10 @@ global.GET = exports.get = function(obj, path) { var builder = []; for (var i = 0, length = arr.length - 1; i < length; i++) - builder.push('if(!w.' + arr[i] + ')return'); + builder.push('if(!w' + (arr[i][0] === '[' ? '' : '.') + arr[i] + ')return'); - var fn = (new Function('w', builder.join(';') + ';return w.' + arr[arr.length - 1])); + var v = arr[arr.length - 1]; + var fn = (new Function('w', builder.join(';') + ';return w' + (v[0] === '[' ? '' : '.') + v)); F.temporary.other[cachekey] = fn; return fn(obj); }; @@ -5482,11 +5485,18 @@ function parsepath(path) { builder.push(all.join('.')); } } else { - all.push(p.substring(0, index)); - builder.push(all.join('.')); - all.splice(all.length - 1); - all.push(p); - builder.push(all.join('.')); + if (p.indexOf('-') === -1) { + all.push(p.substring(0, index)); + builder.push(all.join('.')); + all.splice(all.length - 1); + all.push(p); + builder.push(all.join('.')); + } else { + all.push('[\'' + p.substring(0, index) + '\']'); + builder.push(all.join('')); + all.push(p.substring(index)); + builder.push(all.join('')); + } } } From 517915ace87a053dc0df90730afb6a36e30c3f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Apr 2018 21:32:08 +0200 Subject: [PATCH 0199/1669] Added `Database.find2()` for reverse reading. --- changes.txt | 5 ++ nosql.js | 234 ++++++++++++++++++++++++++++++++++++++++++++----- nosqlworker.js | 10 +++ 3 files changed, 226 insertions(+), 23 deletions(-) diff --git a/changes.txt b/changes.txt index 18203779e..61ff3afa3 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - added: a new route flag type `&group` something like `roles` but groups aren't evaluated - added: `route.groups` with defined groups - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode +- added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) - added: new directory `schemas` with a new configuration item `directory-schemas' - added: new directory `operations` with a new configuration item `directory-operations' - added: `String.crc32([unsigned])` @@ -41,6 +42,8 @@ - added: `HTTPS()` global alias for `F.https()` - added: `LOAD()` global alias for `F.load()` - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` +- added: `GET(obj, path)` alias for `U.get()` +- added: `SET(obj, path, value)` alias for `U.set()` - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` @@ -70,6 +73,8 @@ - fixed: schema field can be changed dynamically - fixed: `String.arg()` - fixed: `controller.href()` with Array values +- fixed: `U.get()` a problem with path with `-` +- fixed: `U.set()` a problem with path with `-` - improved: `debug` mode timing with improved consumption - improved: performance (+20%) NoSQL embedded database diff --git a/nosql.js b/nosql.js index 6690fffcf..65bef6795 100755 --- a/nosql.js +++ b/nosql.js @@ -226,6 +226,10 @@ exports.worker = function() { return send(this, 'find', view).builder = new DatabaseBuilder(this); }; + Database.prototype.find2 = function(view) { + return send(this, 'find2', view).builder = new DatabaseBuilder(this); + }; + Database.prototype.top = function(max, view) { var builder = new DatabaseBuilder(this); builder.take(max); @@ -466,12 +470,13 @@ function Database(name, filename, readonly) { self.filenameTemp = filename + EXTENSION_TMP; self.directory = Path.dirname(filename); self.name = name; - self.pending_update = []; - self.pending_append = []; + self.pending_update = readonly ? EMPTYARRAY : []; + self.pending_append = readonly ? EMPTYARRAY : []; self.pending_reader = []; - self.pending_streamer = []; - self.pending_remove = []; self.pending_reader_view = readonly ? EMPTYARRAY : []; + self.pending_reader2 = readonly ? EMPTYARRAY : []; + self.pending_streamer = []; + self.pending_remove = readonly ? EMPTYARRAY : []; self.views = null; self.step = 0; self.pending_drops = false; @@ -840,7 +845,7 @@ Database.prototype.release = function() { return self; }; -Database.prototype.clear = Database.prototype.remove = function(filename) { +Database.prototype.remove = function(filename) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); @@ -869,21 +874,12 @@ Database.prototype.find = function(view) { return builder; }; -Database.prototype.load = function(take, skip, callback) { +Database.prototype.find2 = function() { var self = this; - - if (typeof(take) === 'function') { - callback = take; - take = 0; - skip = 0; - } else if (typeof(skip) === 'function') { - callback = skip; - skip = 0; - } - - self.pending_loader.push({ take: take, skip: skip, callback: callback, counter: 0, count: 0, response: [] }); - setImmediate(next_operation, self, 10); - return self; + var builder = new DatabaseBuilder(self); + self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 11); + return builder; }; Database.prototype.streamer = function(fn, repository, callback) { @@ -1005,6 +1001,11 @@ Database.prototype.next = function(type) { return; } + if (this.step !== 11 && this.pending_reader2.length) { + this.$reader3(); + return; + } + if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) { if (INMEMORY[this.name]) this.$append_inmemory(); @@ -1596,15 +1597,15 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { item.response = [output]; break; } - - if (first) - return false; } + + if (first) + return false; } }; var cb = function() { - fs.readreverse(function() { + fs.read(function() { for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; @@ -1662,6 +1663,193 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { return self; }; +Database.prototype.$reader3 = function() { + + var self = this; + var self = this; + + self.step = 11; + + if (!self.pending_reader2.length) { + self.next(0); + return self; + } + + var filter = self.pending_reader2.splice(0); + var length = filter.length; + var first = true; + var indexer = 0; + + for (var i = 0; i < length; i++) { + var fil = filter[i]; + if (!fil.builder.$options.first) + first = false; + fil.scalarcount = 0; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + } + + if (first && length > 1) + first = false; + + var fs = new NoSQLStream(self.filename); + + fs.ondocuments = function() { + + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); + var val; + + for (var j = 0; j < docs.length; j++) { + var json = docs[j]; + indexer++; + + var done = true; + + for (var i = 0; i < length; i++) { + + var item = filter[i]; + + if (item.done) + continue; + else if (done) + done = false; + + var builder = item.builder; + item.filter.index = indexer; + + var output = item.compare(json, item.filter, indexer); + if (!output) + continue; + + item.count++; + + if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) + continue; + + item.counter++; + + if (!builder.$inlinesort && !item.done) + item.done = builder.$options.take && builder.$options.take <= item.counter; + + if (item.type) + continue; + + switch (builder.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[builder.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[builder.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'max': + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'avg': + val = output[builder.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (builder.$inlinesort) + nosqlinlinesorter(item, builder, output); + else if (item.response) + item.response.push(output); + else + item.response = [output]; + break; + } + } + + if (first || done) + return false; + } + }; + + var cb = function() { + fs.readreverse(function() { + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + var output; + + if (builder.$options.scalar || !builder.$options.sort) { + + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || EMPTYARRAY; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + continue; + } + + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || EMPTYARRAY; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + builder.done(); + } + + fs = null; + self.next(0); + }); + }; + + fs.openread(cb); + return self; +}; + Database.prototype.$streamer = function() { var self = this; diff --git a/nosqlworker.js b/nosqlworker.js index 49a3b87f7..3e3f61b5f 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -69,6 +69,16 @@ process.on('message', function(msg) { process.send(RESFIND); }); break; + case 'find2': + db.find().parse(msg.data).callback(function(err, response, count, repository) { + RESFIND.err = err; + RESFIND.response = response; + RESFIND.count = count; + RESFIND.repository = repository; + RESFIND.id = msg.id; + process.send(RESFIND); + }); + break; case 'one': db.one(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, count, repository) { RESFIND.err = err; From 76722be16f132294e23ed40e1d4a67b1c87c78e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Apr 2018 21:57:14 +0200 Subject: [PATCH 0200/1669] Updated `Database.clear`. --- nosql.js | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 65bef6795..0738626a9 100755 --- a/nosql.js +++ b/nosql.js @@ -311,7 +311,12 @@ exports.worker = function() { return this; }; - Database.prototype.clear = Database.prototype.remove = function(filename) { + Database.prototype.clear = function(callback) { + send(this, 'clear').callback = callback; + return this; + }; + + Database.prototype.remove = function(filename) { return send(this, 'remove', filename).builder = new DatabaseBuilder(this); }; @@ -477,6 +482,7 @@ function Database(name, filename, readonly) { self.pending_reader2 = readonly ? EMPTYARRAY : []; self.pending_streamer = []; self.pending_remove = readonly ? EMPTYARRAY : []; + self.pending_clear = readonly ? EMPTYARRAY : []; self.views = null; self.step = 0; self.pending_drops = false; @@ -845,6 +851,13 @@ Database.prototype.release = function() { return self; }; +Database.prototype.clear = function(callback) { + var self = this; + self.pending_clear.push(callback || NOOP); + setImmediate(next_operation, self, 12); + return self; +}; + Database.prototype.remove = function(filename) { var self = this; self.readonly && self.throwReadonly(); @@ -974,6 +987,7 @@ Database.prototype.next = function(type) { } if (this.step < 2 && !this.pending_reindex) { + if (this.step !== 2 && this.pending_update.length) { if (INMEMORY[this.name]) this.$update_inmemory(); @@ -989,6 +1003,15 @@ Database.prototype.next = function(type) { this.$remove(); return; } + + if (this.step !== 12 && this.pending_clear.length) { + if (INMEMORY[this.name]) + this.$clear_inmemory(); + else + this.$clear(); + return; + } + } if (this.step !== 6 && this.pending_reader_view.length) { @@ -2394,6 +2417,34 @@ Database.prototype.$remove = function() { }); }; +Database.prototype.$clear = function() { + + var self = this; + self.step = 12; + + if (!self.pending_clear.length) { + self.next(0); + return; + } + + var filter = self.pending_clear.splice(0); + Fs.unlink(self.filename, function() { + if (self.indexes.length) { + self.indexes.clear(function() { + for (var i = 0; i < filter.length; i++) + filter[i](); + self.views && setImmediate(views_refresh, self); + self.next(0); + }); + } else { + for (var i = 0; i < filter.length; i++) + filter[i](); + self.views && setImmediate(views_refresh, self); + self.next(0); + } + }); +}; + Database.prototype.$remove_inmemory = function() { var self = this; @@ -2456,7 +2507,31 @@ Database.prototype.$remove_inmemory = function() { } self.next(0); - change && setImmediate(views_refresh, self); + change && self.views && setImmediate(views_refresh, self); + }); +}; + +Database.prototype.$clear_inmemory = function() { + + var self = this; + self.step = 12; + + if (!self.pending_clear.length) { + self.next(0); + return self; + } + + var filter = self.pending_clear.splice(0); + return self.$inmemory('#', function() { + + self.inmemory['#'] = []; + self.$save('#'); + + for (var i = 0; i < length; i++) + filter[i](null); + + self.next(0); + self.views && setImmediate(views_refresh, self); }); }; From 07705cf74167936d1679a38781d07f01a89ceae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Apr 2018 22:08:18 +0200 Subject: [PATCH 0201/1669] Fixed `Database.one()`. --- nosql.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nosql.js b/nosql.js index 0738626a9..57e03951b 100755 --- a/nosql.js +++ b/nosql.js @@ -1620,10 +1620,10 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { item.response = [output]; break; } - } - if (first) - return false; + if (first) + return false; + } } }; @@ -1689,7 +1689,6 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { Database.prototype.$reader3 = function() { var self = this; - var self = this; self.step = 11; @@ -1811,9 +1810,12 @@ Database.prototype.$reader3 = function() { item.response = [output]; break; } + + if (first) + return false; } - if (first || done) + if (done) return false; } }; From 9bd65aa6b5f82dc6e91352986a1268b456ce2e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Apr 2018 10:59:09 +0200 Subject: [PATCH 0202/1669] Improved `NoSQL.binary`. --- changes.txt | 1 + nosql.js | 392 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 333 insertions(+), 60 deletions(-) diff --git a/changes.txt b/changes.txt index 61ff3afa3..601b51588 100755 --- a/changes.txt +++ b/changes.txt @@ -45,6 +45,7 @@ - added: `GET(obj, path)` alias for `U.get()` - added: `SET(obj, path, value)` alias for `U.set()` +- updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` - updated: `ROUTE()`, extended syntax for schemas, for example: `Schema --> @name` (more in docs.) diff --git a/nosql.js b/nosql.js index 57e03951b..d67376714 100755 --- a/nosql.js +++ b/nosql.js @@ -60,6 +60,10 @@ const FLAGS_READ = ['get']; const COUNTER_MMA = [0, 0]; const REGNUMBER = /^\d+$/; const REGINDEXCHAR = /[a-z]{1,2}/; +const DIRECTORYLENGTH = 9; +const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; +const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; +const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); @@ -700,13 +704,18 @@ Database.prototype.restore = function(filename, callback) { F.restore(filename, F.path.root(), function(err, response) { self.type = 0; - if (!err) { self.$meta(); - self.refresh(); - self.storage && self.storage.refresh(); + if (self.indexes && self.indexes.indexes.length) { + self.indexes.reindex(function() { + self.refresh(); + self.storage && self.storage.refresh(); + }); + } else { + self.refresh(); + self.storage && self.storage.refresh(); + } } - callback && callback(err, response); }); @@ -4300,12 +4309,50 @@ Counter.prototype.clear = function(callback) { }; function Binary(db, directory) { + this.db = db; this.directory = directory; - this.exists = false; this.$events = {}; + this.metafile = directory + 'meta.json'; + this.meta = { $version: 1, index: 0, count: 0, free: [], updated: F.datetime }; + this.cachekey = 'nobin_' + db.name + '_'; + + try { + var json = Fs.readFileSync(this.metafile, 'utf8').toString(); + if (json.length) { + var config = JSON.parse(json, jsonparser); + this.meta.index = config.index; + this.meta.count = config.count; + this.meta.free = config.free || []; + this.meta.updated = config.updated || F.datetime; + } + } catch(e) {} } +Binary.prototype.$save = function() { + var self = this; + self.check(); + self.meta.updated = F.datetime; + Fs.writeFile(self.metafile, JSON.stringify(self.meta), F.error()); + return self; +}; + +Binary.prototype.$directoryindex = function(index) { + return Math.floor(index / 1000) + 1; +}; + +Binary.prototype.$directory = function(index, dir) { + var self = this; + var id = (dir ? index : self.$directoryindex(index)).toString().padLeft(DIRECTORYLENGTH, '0'); + var length = id.length; + var directory = ''; + + for (var i = 0; i < length; i++) + directory += (i % 3 === 0 && i > 0 ? '-' : '') + id[i]; + + return Path.join(self.directory, directory); +}; + Binary.prototype.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { @@ -4380,20 +4427,18 @@ Binary.prototype.insert = function(name, buffer, callback) { var reader = Fs.createReadStream(name); CLEANUP(reader); - return self.insert_stream(null, framework_utils.getName(name), type, reader, callback); + return self.insertstream(null, framework_utils.getName(name), type, reader, callback); } if (typeof(buffer) === 'string') buffer = framework_utils.createBuffer(buffer, 'base64'); else if (buffer.resume) - return self.insert_stream(null, name, type, buffer, callback); + return self.insertstream(null, name, type, buffer, callback); var size = buffer.length; var dimension; var ext = framework_utils.getExtension(name); - self.check(); - switch (ext) { case 'gif': dimension = framework_image.measureGIF(buffer); @@ -4413,51 +4458,145 @@ Binary.prototype.insert = function(name, buffer, callback) { if (!dimension) dimension = { width: 0, height: 0 }; - var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, created: F.created }; + var time = F.datetime.format('yyyyMMdd'); + var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, date: time }; var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); header.write(JSON.stringify(h)); - var id = F.datetime.format('yyMMddHHmm') + 'T' + framework_utils.GUID(5); - var key = self.db.name + '#' + id; - var stream = Fs.createWriteStream(Path.join(self.directory, key + EXTENSION_BINARY)); + var id; + + if (self.meta.free.length) { + id = self.meta.free.shift(); + } else { + self.meta.index++; + id = self.meta.index; + } + + self.meta.count++; + + var path = self.$directory(id); + self.check(path); + self.$save(); + + var filename = id.toString().padLeft(DIRECTORYLENGTH, '0'); + var stream = Fs.createWriteStream(Path.join(path, filename + EXTENSION_BINARY)); stream.write(header, 'binary'); stream.end(buffer); CLEANUP(stream); + + id = 'B' + time + 'T' + filename; callback && callback(null, id, h); self.$events.insert && self.emit('insert', id, h); + return id; }; -Binary.prototype.insert_stream = function(id, name, type, stream, callback) { +Binary.prototype.insertstream = function(id, name, type, stream, callback) { var self = this; - self.check(); - - var h = { name: name, size: 0, type: type, width: 0, height: 0 }; + var time = F.datetime.format('yyyyMMdd'); + var h = { name: name, size: 0, type: type, width: 0, height: 0, date: time }; var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); + header.fill(' '); header.write(JSON.stringify(h)); - if (!id) - id = F.datetime.format('yyMMddHHmm') + 'T' + framework_utils.GUID(5); + var isnew = false; + var cacheid = id; - var key = self.db.name + '#' + id; - var writer = Fs.createWriteStream(framework_utils.join(self.directory, key + EXTENSION_BINARY)); + if (id) { + // check if it's new implementation + if (id > 0) { + isnew = true; + } else if (id[0] === 'B' || id[0] === 'b') { + isnew = true; + id = +id.substring(id.length - DIRECTORYLENGTH); + } + } else { + isnew = true; + if (self.meta.free.length) { + id = self.meta.free.shift(); + } else { + self.meta.index++; + id = self.meta.index; + } + self.meta.count++; + self.$save(); + } + var filepath; + var filename; + + if (isnew) { + var path = self.$directory(id); + self.check(path); + filename = id.toString().padLeft(DIRECTORYLENGTH, '0'); + filepath = Path.join(path, filename + EXTENSION_BINARY); + } else + filepath = framework_utils.join(self.directory, self.db.name + '#' + id + EXTENSION_BINARY); + + var writer = Fs.createWriteStream(filepath); writer.write(header, 'binary'); + + var ext = framework_utils.getExtension(name); + var dimension = null; + + IMAGES[ext] && stream.once('data', function(buffer) { + switch (ext) { + case 'gif': + dimension = framework_image.measureGIF(buffer); + break; + case 'png': + dimension = framework_image.measurePNG(buffer); + break; + case 'jpg': + case 'jpeg': + dimension = framework_image.measureJPG(buffer); + break; + case 'svg': + dimension = framework_image.measureSVG(buffer); + break; + } + }); + stream.pipe(writer); - CLEANUP(writer); - callback && callback(null, id, h); - self.$events.insert && self.emit('insert', id, h); - return id; + + if (isnew) + id = 'B' + time + 'T' + filename; + + CLEANUP(writer, function() { + + if (dimension) { + h.width = dimension.width; + h.height = dimension.height; + } + + h.size = writer.bytesWritten; + + Fs.open(filepath, 'a', function(err, fd) { + if (!err) { + var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); + header.fill(' '); + header.write(JSON.stringify(h)); + Fs.write(fd, header, 0, header.length, 0, () => Fs.close(fd, NOOP)); + } + }); + + callback && callback(null, cacheid || id, h); + self.$events.insert && self.emit('insert', cacheid || id, h); + }); + + return cacheid || id; }; Binary.prototype.update = function(id, name, buffer, callback) { var type = framework_utils.getContentType(framework_utils.getExtension(name)); + var self = this; var isfn = typeof(buffer) === 'function'; + if (isfn || !buffer) { if (isfn) { @@ -4467,21 +4606,41 @@ Binary.prototype.update = function(id, name, buffer, callback) { var reader = Fs.createReadStream(name); CLEANUP(reader); - return self.insert_stream(id, framework_utils.getName(name), type, reader, callback); + return self.insertstream(id, framework_utils.getName(name), type, reader, callback); } if (typeof(buffer) === 'string') buffer = framework_utils.createBuffer(buffer, 'base64'); if (buffer.resume) - return this.insert_stream(id, name, type, buffer, callback); + return self.insertstream(id, name, type, buffer, callback); - var self = this; + var isnew = false; + var time = F.datetime.format('yyyyMMdd'); var size = buffer.length; - var dimension; var ext = framework_utils.getExtension(name); - - self.check(); + var dimension; + var filepath; + var filename; + var cacheid = id; + + // check if it's new implementation + if (id > 0) + isnew = true; + else if (id[0] === 'B' || id[0] === 'b') { + isnew = true; + id = +id.substring(id.length - DIRECTORYLENGTH); + } + + if (isnew) { + var path = self.$directory(id); + self.check(path); + filename = id.toString().padLeft(DIRECTORYLENGTH, '0'); + filepath = Path.join(path, filename + EXTENSION_BINARY); + } else { + self.check(); + filepath = framework_utils.join(self.directory, self.db.name + '#' + id + EXTENSION_BINARY); + } switch (ext) { case 'gif': @@ -4502,38 +4661,47 @@ Binary.prototype.update = function(id, name, buffer, callback) { if (!dimension) dimension = { width: 0, height: 0 }; - var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, created: F.datetime }; + var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, date: time }; var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); - var key = self.db.name + '#' + id; header.fill(' '); header.write(JSON.stringify(h)); - var stream = Fs.createWriteStream(framework_utils.join(self.directory, key + EXTENSION_BINARY)); + var stream = Fs.createWriteStream(filepath); stream.write(header, 'binary'); stream.end(buffer); CLEANUP(stream); - callback && callback(null, id, h); - self.$events.insert && self.emit('insert', id, h); - return id; + callback && callback(null, cacheid, h); + self.$events.insert && self.emit('insert', cacheid, h); + return cacheid; }; Binary.prototype.read = function(id, callback) { var self = this; - - self.check(); - - if (id.indexOf('#') === -1) + var isnew = false; + + if (id > 0) + isnew = true; + else if (id[0] === 'B' || id[0] === 'b') { + id = +id.substring(id.length - DIRECTORYLENGTH); + isnew = true; + } else if (id.indexOf('#') === -1) id = self.db.name + '#' + id; - var filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); - var stream = Fs.createReadStream(filename, { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }); + var filename; + + if (isnew) { + var path = self.$directory(id); + filename = Path.join(path, id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + } else + filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); + var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); stream.on('data', function(buffer) { var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, ''); - stream = Fs.createReadStream(filename, { start: BINARY_HEADER_LENGTH }); + stream = Fs.createReadStream(filename, BINARYREADDATA); callback(null, stream, JSON.parse(json, jsonparser)); CLEANUP(stream); }); @@ -4544,35 +4712,61 @@ Binary.prototype.read = function(id, callback) { Binary.prototype.remove = function(id, callback) { var self = this; - var key = id; + var cacheid = id; + var isnew = false; + var filename; + + if (id > 0) + isnew = true; + else if (id[0] === 'B' || id[0] === 'b') { + isnew = true; + id = +id.substring(id.length - DIRECTORYLENGTH); + } else if (id.indexOf('#') === -1) + id = self.db.name + '#' + id; - self.check(); + if (isnew) { + var path = self.$directory(id); + filename = Path.join(path, id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + } else + filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); + + Fs.unlink(filename, function(err) { + + if (isnew && !err) { + self.meta.count--; + self.meta.free.push(id); + self.$save(); + } - if (key.indexOf('#') === -1) - key = self.db.name + '#' + key; + callback && callback(null, err ? false : true); + }); - var filename = framework_utils.join(self.directory, key + EXTENSION_BINARY); - Fs.unlink(filename, (err) => callback && callback(null, err ? false : true)); - self.$events.remove && self.emit('remove', id); + self.$events.remove && self.emit('remove', cacheid); return self; }; -Binary.prototype.check = function() { +Binary.prototype.check = function(path) { var self = this; - if (self.exists) + var key = self.cachekey + (path == null ? 'root' : path.substring(path.length - (DIRECTORYLENGTH + 2))); + + if (F.temporary.other[key]) return self; - self.exists = true; + if (path != null && !F.temporary.other[self.cachekey + 'root']) + self.check(); + + F.temporary.other[key] = true; try { - Fs.mkdirSync(self.directory); + Fs.mkdirSync(path ? path : self.directory); } catch (err) {} return self; }; Binary.prototype.clear = function(callback) { + var self = this; Fs.readdir(self.directory, function(err, response) { @@ -4581,20 +4775,85 @@ Binary.prototype.clear = function(callback) { return callback(err); var pending = []; + var directories =[]; var key = self.db.name + '#'; var l = key.length; var target = framework_utils.join(self.directory); - for (var i = 0, length = response.length; i < length; i++) - response[i].substring(0, l) === key && pending.push(target + '/' + response[i]); + for (var i = 0, length = response.length; i < length; i++) { + var p = response[i]; + if (p.substring(0, l) === key) + pending.push(target + '/' + p); + else if (p[3] === '-' && p[7] === '-') + directories.push(target + '/' + p); + } + pending.push(target + '/meta.json'); self.$events.clear && self.emit('clear', pending.length); - F.unlink(pending, callback); + pending.length && F.unlink(pending, F.errorhandling); + directories.wait(function(path, next) { + Fs.readdir(path, function(err, files) { + for (var i = 0; i < files.length; i++) + files[i] = path + '/' + files[i]; + F.unlink(files, () => Fs.unlink(path, next)); + }); + }, callback); }); return self; }; +Binary.prototype.browse = function(directory, callback) { + var self = this; + + if (typeof(directory) === 'function') { + Fs.readdir(self.directory, function(err, files) { + var dirs = []; + for (var i = 0; i < files.length; i++) { + var p = files[i]; + if (p[3] === '-' && p[7] === '-') + dirs.push(p); + } + directory(null, dirs); + }); + } else { + Fs.readdir(Path.join(self.directory, directory), function(err, response) { + + var target = framework_utils.join(self.directory, directory); + var output = []; + var le = EXTENSION_BINARY.length; + + response.wait(function(item, next) { + Fs.stat(target + '/' + item, function(err, stat) { + + if (err) + return next(); + + var stream = Fs.createReadStream(target + '/' + item, BINARYREADMETA); + + stream.on('data', function(buffer) { + var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, '').parseJSON(true); + if (json) { + json.id = item.substring(0, item.length - le); + json.id = +json.id.substring(json.id.length - DIRECTORYLENGTH); + json.ctime = stat.ctime; + json.mtime = stat.mtime; + output.push(json); + } + }); + + CLEANUP(stream, next); + + }); + + }, () => callback(null, output), 2); + + }); + } + + return self; +}; + Binary.prototype.all = function(callback) { var self = this; @@ -4622,7 +4881,7 @@ Binary.prototype.all = function(callback) { if (err) return next(); - var stream = Fs.createReadStream(target + '/' + item, { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }); + var stream = Fs.createReadStream(target + '/' + item, BINARYREADMETA); stream.on('data', function(buffer) { var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, '').parseJSON(true); @@ -4637,7 +4896,20 @@ Binary.prototype.all = function(callback) { CLEANUP(stream, next); }); - }, () => callback(null, output), 2); + + }, function() { + if (self.meta.count) { + self.browse(function(err, directories) { + directories.wait(function(item, next) { + self.browse(item, function(err, files) { + files.length && output.push.apply(output, files); + next(); + }); + }, () => callback(null, output)); + }); + } else + callback(null, output); + }, 2); }); return self; From 45d27adb6f864421539d601e1d34d7432d8839b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Apr 2018 10:59:34 +0200 Subject: [PATCH 0203/1669] Added new alias `NOSQLBINARY`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 2284c5e23..8dd0ed5e1 100755 --- a/index.js +++ b/index.js @@ -284,7 +284,7 @@ var U = global.Utils = global.utils = global.U = global.framework_utils; global.Mail = framework_mail; global.WTF = (message, name, uri) => F.problem(message, name, uri); -global.NOBIN = (name) => F.nosql(name).binary; +global.NOBIN = global.NOSQLBINARY = (name) => F.nosql(name).binary; global.NOSQLSTORAGE = (name) => F.nosql(name).storage; global.NOSQLINDEXES = (name) => F.nosql(name).indexes; global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; From 1fec7314aea1e4d3e5b92372a285fbb1cf33971a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Apr 2018 11:25:35 +0200 Subject: [PATCH 0204/1669] Remove `GET()` and `SET()` globals. --- changes.txt | 2 -- utils.js | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 601b51588..274eec744 100755 --- a/changes.txt +++ b/changes.txt @@ -42,8 +42,6 @@ - added: `HTTPS()` global alias for `F.https()` - added: `LOAD()` global alias for `F.load()` - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` -- added: `GET(obj, path)` alias for `U.get()` -- added: `SET(obj, path, value)` alias for `U.set()` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/utils.js b/utils.js index 34f382cb6..6c2cefc90 100755 --- a/utils.js +++ b/utils.js @@ -5426,7 +5426,7 @@ exports.parseTheme = function(value) { return value === '?' ? F.config['default-theme'] : value; }; -global.SET = exports.set = function(obj, path, value) { +exports.set = function(obj, path, value) { var cachekey = 'S+' + path; if (F.temporary.other[cachekey]) @@ -5447,7 +5447,7 @@ global.SET = exports.set = function(obj, path, value) { fn(obj, value, path); }; -global.GET = exports.get = function(obj, path) { +exports.get = function(obj, path) { var cachekey = 'G=' + path; From 79ab24ec64e0d879b2c3055c4abe4842702af17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Apr 2018 11:27:15 +0200 Subject: [PATCH 0205/1669] Removed `HTTP()` and `HTTPS()` globals. --- changes.txt | 3 --- index.js | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index 274eec744..727eaddf6 100755 --- a/changes.txt +++ b/changes.txt @@ -38,9 +38,6 @@ - added: `$INSERT(schema, model, [options], [callback], [controller])` performs `schema.insert()` - added: `$UPDATE(schema, model, [options], [callback], [controller])` performs `schema.update()` - added: `$REMOVE(schema, [options], [callback], [controller])` performs `schema.remove()` -- added: `HTTP()` global alias for `F.http()` -- added: `HTTPS()` global alias for `F.https()` -- added: `LOAD()` global alias for `F.load()` - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory diff --git a/index.js b/index.js index 8dd0ed5e1..f52f37d13 100755 --- a/index.js +++ b/index.js @@ -6648,7 +6648,7 @@ function connection_tunning(socket) { * @param {Function(listen)} middleware A middleware for manual calling of HTTP listener * @return {Framework} */ -global.HTTP = F.http = function(mode, options, middleware) { +F.http = function(mode, options, middleware) { F.consoledebug('begin'); if (typeof(options) === 'function') { @@ -6675,7 +6675,7 @@ global.HTTP = F.http = function(mode, options, middleware) { * @param {Function(listen)} middleware A middleware for manual calling of HTTP listener * @return {Framework} */ -global.HTTPS = F.https = function(mode, options, middleware) { +F.https = function(mode, options, middleware) { F.consoledebug('begin'); var http = require('http'); From bf200d1f15d7e5557e9ef1d2f3344836e5db783d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Apr 2018 12:16:08 +0200 Subject: [PATCH 0206/1669] Fixed `SchemaOptions.response()`. --- builders.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index e8ea6dbb8..72723609f 100755 --- a/builders.js +++ b/builders.js @@ -2497,7 +2497,7 @@ function async_continue(self) { } SchemaInstance.prototype.$response = function(index) { - return this.$$index == undefined ? this.$$result : this.$$result[index === 'prev' ? this.$$index : index]; + return index == undefined ? this.$$result : this.$$result[index === 'prev' ? (this.$$result.length - 1) : index]; }; SchemaInstance.prototype.$repository = function(name, value) { @@ -2528,7 +2528,7 @@ SchemaInstance.prototype.$callback = function(callback) { return this; }; -SchemaInstance.prototype.$response = SchemaInstance.prototype.$output = function() { +SchemaInstance.prototype.$output = function() { this.$$index = this.$$result.length; return this; }; From f13187b4c3f91ac0344328eaa8f73270bc93ee98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Apr 2018 17:39:20 +0200 Subject: [PATCH 0207/1669] Fixed refreshing of meta info in binary files. --- nosql.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index d67376714..c618df662 100755 --- a/nosql.js +++ b/nosql.js @@ -706,6 +706,7 @@ Database.prototype.restore = function(filename, callback) { self.type = 0; if (!err) { self.$meta(); + self.binary.$refresh(); if (self.indexes && self.indexes.indexes.length) { self.indexes.reindex(function() { self.refresh(); @@ -4309,14 +4310,19 @@ Counter.prototype.clear = function(callback) { }; function Binary(db, directory) { - this.db = db; this.directory = directory; this.$events = {}; this.metafile = directory + 'meta.json'; - this.meta = { $version: 1, index: 0, count: 0, free: [], updated: F.datetime }; + this.meta = { $version: 1, updated: F.datetime }; this.cachekey = 'nobin_' + db.name + '_'; + this.$refresh(); +} +Binary.prototype.$refresh = function() { + this.meta.index = 0; + this.meta.count = 0; + this.meta.free = []; try { var json = Fs.readFileSync(this.metafile, 'utf8').toString(); if (json.length) { @@ -4327,7 +4333,7 @@ function Binary(db, directory) { this.meta.updated = config.updated || F.datetime; } } catch(e) {} -} +}; Binary.prototype.$save = function() { var self = this; From 3f2b805cdd4d938997ec1c027b9050029d17765c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 12 Apr 2018 16:02:54 +0200 Subject: [PATCH 0208/1669] Improved updating/removing in NoSQL embedded database. --- changes.txt | 2 + index.js | 11 ++ nosql.js | 352 +++++++++++++++++++++++++------------------------ nosqlstream.js | 264 +++++++++++++++++++++++++------------ 4 files changed, 373 insertions(+), 256 deletions(-) diff --git a/changes.txt b/changes.txt index 727eaddf6..f3c2d75dd 100755 --- a/changes.txt +++ b/changes.txt @@ -30,6 +30,8 @@ - added: `U.hash('crc32')` and `U.hash('crc32unsigned')` - added: config `nosql-worker' for enabling worker for NoSQL embedded database (default: `false`) - added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array +- added: config `nosql-cleaner` for cleaning databases from removed documents (default: `1440` === 24 hours) +- added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` diff --git a/index.js b/index.js index f52f37d13..8746f073b 100755 --- a/index.js +++ b/index.js @@ -671,6 +671,8 @@ function Framework() { 'disable-clear-temporary-directory': false, 'nosql-worker': false, 'nosql-inmemory': null, // String Array + 'nosql-cleaner': 1440, + 'nosql-logger': true, // Used in F.service() // All values are in minutes @@ -748,6 +750,7 @@ function Framework() { this.workers = {}; this.databases = {}; + this.databasescleaner = {}; this.directory = HEADERS.workers.cwd = directory; this.isLE = Os.endianness ? Os.endianness() === 'LE' : true; this.isHTTPS = false; @@ -6991,6 +6994,12 @@ F.service = function(count) { // Update expires date count % 1000 === 0 && (DATE_EXPIRES = F.datetime.add('y', 1).toUTCString()); + if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { + var keys = Object.keys(F.databasescleaner); + for (var i = 0; i < keys.length; i++) + NOSQL(keys[i]).clean(); + } + F.$events.service && F.emit('service', count); if (F.config['allow-debug']) { @@ -8564,6 +8573,7 @@ F.$configure_configs = function(arr, rewrite) { case 'default-maximum-file-descriptors': case 'default-interval-clear-dnscache': case 'default-dependency-timeout': + case 'nosql-cleaner': obj[name] = U.parseInt(value); break; case 'default-image-consumption': @@ -8621,6 +8631,7 @@ F.$configure_configs = function(arr, rewrite) { case 'trace': case 'allow-cache-snapshot': case 'nosql-worker': + case 'nosql-logger': obj[name] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; diff --git a/nosql.js b/nosql.js index c618df662..c138dbc1a 100755 --- a/nosql.js +++ b/nosql.js @@ -64,6 +64,7 @@ const DIRECTORYLENGTH = 9; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; +const CLEANDBTICKS = 86400000 * 2; // 48 hours const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); @@ -479,14 +480,15 @@ function Database(name, filename, readonly) { self.filenameTemp = filename + EXTENSION_TMP; self.directory = Path.dirname(filename); self.name = name; - self.pending_update = readonly ? EMPTYARRAY : []; - self.pending_append = readonly ? EMPTYARRAY : []; + self.pending_update = []; + self.pending_append = []; self.pending_reader = []; + self.pending_remove = []; self.pending_reader_view = readonly ? EMPTYARRAY : []; - self.pending_reader2 = readonly ? EMPTYARRAY : []; + self.pending_reader2 = []; self.pending_streamer = []; - self.pending_remove = readonly ? EMPTYARRAY : []; - self.pending_clear = readonly ? EMPTYARRAY : []; + self.pending_clean = []; + self.pending_clear = []; self.views = null; self.step = 0; self.pending_drops = false; @@ -499,7 +501,6 @@ function Database(name, filename, readonly) { self.$timeoutmeta; self.$events = {}; self.$free = true; - self.renaming = false; } Database.prototype.emit = function(name, a, b, c, d, e, f, g) { @@ -868,6 +869,13 @@ Database.prototype.clear = function(callback) { return self; }; +Database.prototype.clean = function(callback) { + var self = this; + self.pending_clean.push(callback || NOOP); + setImmediate(next_operation, self, 13); + return self; +}; + Database.prototype.remove = function(filename) { var self = this; self.readonly && self.throwReadonly(); @@ -987,41 +995,36 @@ Database.prototype.view = function(name) { Database.prototype.next = function(type) { - // 9: renaming file (by updating/removing) - if (type && this.step === 9) + if (type && this.step) return; - if (this.renaming) { - !type && (this.step = 0); + if (this.step !== 2 && this.pending_update.length) { + if (INMEMORY[this.name]) + this.$update_inmemory(); + else + this.$update(); return; } - if (this.step < 2 && !this.pending_reindex) { - - if (this.step !== 2 && this.pending_update.length) { - if (INMEMORY[this.name]) - this.$update_inmemory(); - else - this.$update(); - return; - } - - if (this.step !== 3 && this.pending_remove.length) { - if (INMEMORY[this.name]) - this.$remove_inmemory(); - else - this.$remove(); - return; - } + if (this.step !== 3 && this.pending_remove.length) { + if (INMEMORY[this.name]) + this.$remove_inmemory(); + else + this.$remove(); + return; + } - if (this.step !== 12 && this.pending_clear.length) { - if (INMEMORY[this.name]) - this.$clear_inmemory(); - else - this.$clear(); - return; - } + if (this.step !== 12 && this.pending_clear.length) { + if (INMEMORY[this.name]) + this.$clear_inmemory(); + else + this.$clear(); + return; + } + if (this.step !== 13 && this.pending_clean.length) { + this.$clean(); + return; } if (this.step !== 6 && this.pending_reader_view.length) { @@ -1039,6 +1042,11 @@ Database.prototype.next = function(type) { return; } + if (this.step !== 7 && !this.pending_reindex && this.pending_drops) { + this.$drop(); + return; + } + if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) { if (INMEMORY[this.name]) this.$append_inmemory(); @@ -1047,11 +1055,6 @@ Database.prototype.next = function(type) { return; } - if (this.step !== 7 && !this.pending_reindex && this.pending_drops) { - this.$drop(); - return; - } - if (this.step !== 5 && this.pending_views) { if (INMEMORY[this.name]) this.$views_inmemory(); @@ -1233,30 +1236,40 @@ Database.prototype.$update = function() { return self; } + var filter = self.pending_update.splice(0); + var length = filter.length; + var backup = false; + + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + if (fil.backup || fil.builder.$options.backup) + backup = true; + } + var indexer = 0; var fs = new NoSQLStream(self.filename); fs.ondocuments = function() { + var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - var updated = false; for (var a = 0; a < docs.length; a++) { - + var doc = docs[a]; indexer++; - var doc = docs[a]; var is = false; var copy = self.indexes && self.indexes.indexes.length ? CLONE(doc) : null; + var rec = fs.docsbuffer[a]; for (var i = 0; i < length; i++) { var item = filter[i]; - var builder = item.builder; item.filter.index = indexer; var output = item.compare(doc, item.filter, indexer); if (output) { - builder.$options.backup && builder.$backupdoc(output); if (item.keys) { for (var j = 0, jl = item.keys.length; j < jl; j++) { var val = item.doc[item.keys[j]]; @@ -1274,72 +1287,49 @@ Database.prototype.$update = function() { var e = item.keys ? 'modify' : 'update'; self.$events[e] && self.emit(e, output); item.count++; - if (!fs.changed) - fs.changed = true; - docs[a] = output; + doc = output; is = true; - if (!updated) - updated = true; } } - if (is && self.indexes && self.indexes.indexes.length) - self.indexes.update(doc, copy); - } - - if (updated) { - var tmp = ''; - for (var i = 0; i < docs.length; i++) - tmp += JSON.stringify(docs[i]) + NEWLINE; - fs.write(tmp); - } else - fs.write(fs.docscache); - }; + if (is) { - var filter = self.pending_update.splice(0); - var length = filter.length; + if (self.indexes && self.indexes.indexes.length) + self.indexes.update(doc, copy); - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } + if (backup) { + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.backup && item.backup.write(rec.doc + NEWLINE); + item.builder.$options.backup && item.builder.$backupdoc(rec.doc); + } + } - var finish = function() { + var upd = JSON.stringify(doc); + var was = true; - if (!fs.changed) { + if (rec.doc.length === upd.length) { + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + was = false; + } + } - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert); - self.insert(item.insert).$callback = item.builder.$callback; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + if (was) { + var tmp = '-' + rec.doc.substring(1) + NEWLINE; + fs.write(tmp, rec.position); + fs.write2(upd + NEWLINE); } } - - fs.flush(function() { - fs = null; - self.next(0); - }); - return; - } - - self.renaming = true; - - // Maybe is reading? - if (self.step && self.step !== 9 && self.step !== 2) { - self.next(0); - return setTimeout(finish, 100); } + }; - self.step = 9; - fs.flush(function(err) { + var finish = function() { + fs.flush(function() { - err && F.error(err); - self.renaming = false; + if (self.indexes) + F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; for (var i = 0; i < length; i++) { var item = filter[i]; @@ -1359,7 +1349,7 @@ Database.prototype.$update = function() { }; fs.openupdate(function() { - fs.read(finish, true); + fs.readupdate(finish, true); }); return self; @@ -2313,119 +2303,72 @@ Database.prototype.$remove = function() { var fil = filter[i]; fil.compare = fil.builder.compile(); fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup) + if (fil.backup || fil.builder.$options.backup) backup = true; } fs.ondocuments = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - var is = false; - var deleted = {}; for (var j = 0; j < docs.length; j++) { indexer++; var removed = false; - var json = docs[j]; + var doc = docs[j]; + var rec = fs.docsbuffer[j]; for (var i = 0; i < length; i++) { var item = filter[i]; - var builder = item.builder; item.filter.index = indexer; - var output = item.compare(json, item.filter, indexer); + var output = item.compare(doc, item.filter, indexer); if (output) { - item.count++; - builder.$options.backup && builder.$backupdoc(output); removed = true; - json = output; + doc = output; break; } } if (removed) { - deleted[j] = 1; - var tmp = null; - if (backup) { for (var i = 0; i < length; i++) { var item = filter[i]; - if (item.backup) { - if (!tmp) - tmp = JSON.stringify(docs[j]) + NEWLINE; - item.backup.write(tmp); - } - item.count++; + item.backup && item.backup.write(rec.doc + NEWLINE); + item.builder.$options.backup && item.builder.$backupdoc(rec.doc); } } - self.$events.remove && self.emit('remove', json); - if (!fs.changed) - fs.changed = true; - if (!is) - is = true; + item.count++; + self.$events.remove && self.emit('remove', doc); if (self.indexes && self.indexes.indexes.length) - self.indexes.remove(json); - } - } + self.indexes.remove(doc); - if (is) { - var tmp = ''; - for (var j = 0; j < docs.length; j++) { - if (deleted[j] == null) - tmp += JSON.stringify(docs[j]) + NEWLINE; + fs.write('-' + rec.doc.substring(1) + NEWLINE, rec.position); } - tmp && fs.write(tmp); - } else - fs.write(fs.docscache); + } }; var finish = function() { - - // No change - if (!fs.changed) { - fs.flush(function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - fs = null; - self.next(0); - }); - return; - } - - self.renaming = true; - - // Maybe is reading? - if (self.step && self.step !== 9 && self.step !== 3) { - self.next(0); - return setTimeout(finish, 100); - } - - self.step = 9; - fs.flush(function() { + if (self.indexes) + F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; + for (var i = 0; i < length; i++) { var item = filter[i]; item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } - - setImmediate(function() { - self.renaming = false; - self.next(0); - change && self.views && setImmediate(views_refresh, self); - }); + fs = null; + self.next(0); + change && self.views && setImmediate(views_refresh, self); }); }; fs.openupdate(function() { - fs.read(finish, true); + fs.readupdate(finish, true); }); }; @@ -2457,6 +2400,42 @@ Database.prototype.$clear = function() { }); }; +Database.prototype.$clean = function() { + + var self = this; + self.step = 13; + + if (!self.clean.length) { + self.next(0); + return; + } + + var filter = self.pending_clean.splice(0); + var length = filter.length; + var now = Date.now(); + + F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" cleaning (beg)'.format(self.name)); + + var fs = new NoSQLStream(self.filename); + var writer = Fs.createWriteStream(self.filename + '-tmp'); + + fs.divider = NEWLINE; + fs.ondocuments = () => writer.write(fs.docs + NEWLINE); + + writer.on('finish', function() { + Fs.rename(self.filename + '-tmp', self.filename, function() { + F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); + for (var i = 0; i < length; i++) + filter[i](); + self.$events.clean && self.emit('clean'); + self.next(0); + fs = null; + }); + }); + + fs.openread(() => fs.read(() => writer.end())); +}; + Database.prototype.$remove_inmemory = function() { var self = this; @@ -2873,7 +2852,7 @@ DatabaseBuilder.prototype.backup = function(user) { }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + JSON.stringify(doc) + NEWLINE, F.errorcallback); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); return this; }; @@ -4958,7 +4937,7 @@ Indexes.prototype.create = function(name, properties, type) { meta.key = key; } } else { - self.meta[name] = { key: key, documents: 0 }; + self.meta[name] = { key: key, documents: 0, changes: 0, cleaned: F.datetime.getTime() }; reindex = true; } @@ -4974,6 +4953,7 @@ Indexes.prototype.noreindex = function() { Indexes.prototype.$meta = function() { var self = this; + self.$metachanged = false; Fs.writeFile(self.db.filename + EXTENSION_INDEXES, JSON.stringify(this.meta), NOOP); return this; }; @@ -5169,12 +5149,16 @@ Indexes.prototype.reindex = function(callback) { self.reindexing = true; var now = Date.now(); - PRINTLN('NoSQL embedded "{0}" re-indexing (beg)'.format(self.db.name)); + F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" re-indexing (beg)'.format(self.db.name)); var keys = Object.keys(self.meta); + var ticks = F.datetime.getTime(); - for (var i = 0; i < self.indexes.length; i++) - self.meta[self.indexes[i].name].documents = 0; + for (var i = 0; i < self.indexes.length; i++) { + var item = self.meta[self.indexes[i].name]; + item.documents = 0; + item.cleaned = ticks; + } // Clears non-exist indexes for (var i = 0; i < keys.length; i++) { @@ -5201,7 +5185,7 @@ Indexes.prototype.reindex = function(callback) { self.db.pending_reindex = false; self.db.next(0); callback && callback(null, count); - PRINTLN('NoSQL embedded "{0}" re-indexing (end, {1}s)'.format(self.db.name, (((Date.now() - now) / 1000) >> 0))); + F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" re-indexing (end, {1}s)'.format(self.db.name, (((Date.now() - now) / 1000) >> 0))); }); }); }); @@ -5305,6 +5289,10 @@ Indexes.prototype.update = function(doc, old) { var oldvalues = self.makeindex(index, old); var oldkey = self.$index(index, oldvalues); + // Because of cleaning + self.meta[index.name].changes++; + self.$metachanged = true; + var item = self.findchanges(index, key, values); if (item) item.doc = doc; @@ -5321,7 +5309,6 @@ Indexes.prototype.update = function(doc, old) { return self; }; - Indexes.prototype.remove = function(doc) { var self = this; @@ -5335,6 +5322,10 @@ Indexes.prototype.remove = function(doc) { if (!key) continue; + // Because of cleaning + self.meta[index.name].changes++; + self.$metachanged = true; + var item = self.findchanges(index, key, values); if (!item) self.changes.push({ remove: true, key: key, name: index.name, properties: index.properties, value: values }); @@ -5363,6 +5354,7 @@ Indexes.prototype.flush = function() { return; self.flushing = false; + self.$metachanged && self.$meta(); self.$free(); if (!self.changes.length) { @@ -5376,6 +5368,7 @@ Indexes.prototype.flush = function() { }; var arr = self.changes.splice(0, 50); + var ticks = F.datetime.getTime() - CLEANDBTICKS; for (var i = 0; i < arr.length; i++) { @@ -5386,6 +5379,7 @@ Indexes.prototype.flush = function() { } else { self.instances[item.key] = new Database(item.key, self.directory + item.key, true); self.instances[item.key].PENDING = 0; + self.instances[item.key].CLEANDB = self.meta[item.name].changes > 0 && self.meta[item.name].cleaned < ticks ? item.name : null; } if (item.update) { @@ -5439,8 +5433,22 @@ Indexes.prototype.$free = function() { var keys = Object.keys(self.instances); for (var i = 0; i < keys.length; i++) { var db = self.instances[keys[i]]; - if (!db && db.PENDING) - delete self.instances[keys[i]]; + if (!db || !db.PENDING) { + // Cleans removed/changed documents + if (db && db.CLEANDB) { + db.PENDING++; + db.clean(function() { + db.PENDING--; + var a = self.meta[db.CLEANDB]; + a.cleaned = F.datetime.getTime(); + a.changes = 0; + db.CLEANDB = null; + self.$meta(); + delete self.instances[keys[i]]; + }); + } else + delete self.instances[keys[i]]; + } } return self; }; diff --git a/nosqlstream.js b/nosqlstream.js index 2ec41c830..fb340bbd5 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -30,7 +30,6 @@ const Fs = require('fs'); const BUFFERSIZE = 1024 * 32; const BUFFERDOCS = 15; const NEWLINEBUFFER = framework_utils.createBuffer('\n', 'utf8'); -const NEWLINE = '\n'; const DEFSTATS = { size: 0 }; function NoSQLStream(filename) { @@ -39,15 +38,15 @@ function NoSQLStream(filename) { this.stats = DEFSTATS; this.type = null; this.bytesread = 0; + this.ticks = 0; this.position = 0; this.cache = [null, null]; this.buffer = null; + this.divider = ','; // this.canceled = false; // this.docs = ''; // this.docscount = 0; - // this.docscache = ''; // this.indexer = 0; - // this.fdwrite = null; } NoSQLStream.prototype.openread = function(callback) { @@ -60,7 +59,7 @@ NoSQLStream.prototype.openread = function(callback) { NoSQLStream.prototype.openupdate = function(callback) { var self = this; - self.type = 'r'; + self.type = 'r+'; self.open(function(err) { // File may not exist @@ -70,19 +69,13 @@ NoSQLStream.prototype.openupdate = function(callback) { } self.position = 0; + self.positionappend = self.stats.size; + self.positionupdate = 0; self.bufferstack = []; - self.changed = false; - - Fs.open(self.filename + '-tmp', 'w', function(err, fd) { - - if (err) { - self.close(NOOP); - throw err; - } + self.bufferstacknew = []; + self.docsbuffer = []; - self.fdwrite = fd; - callback && callback(); - }); + callback && callback(); }); return self; @@ -97,10 +90,10 @@ NoSQLStream.prototype.openinsert = function(callback) { // For e.g. files on URL address NoSQLStream.prototype.openstream = function(stream, callback) { + var self = this; self.docs = ''; - self.docscache = ''; self.docscount = 0; self.stream = stream; @@ -109,7 +102,6 @@ NoSQLStream.prototype.openstream = function(stream, callback) { self.ondocuments(); self.docscount = 0; self.docs = ''; - self.docscache = ''; } callback && callback(); }); @@ -136,7 +128,7 @@ NoSQLStream.prototype.openstream = function(stream, callback) { var tmp = self.buffer.toString('utf8', 0, index).trim(); - self.docs += (self.docs ? ',' : '') + tmp; + self.docs += (self.docs ? self.divider : '') + tmp; self.docscount++; self.indexer++; @@ -176,19 +168,11 @@ NoSQLStream.prototype.open = function(callback) { } Fs.fstat(fd, function(err, stats) { - self.docs = ''; - self.docscache = ''; self.docscount = 0; self.fd = fd; self.stats = stats; - - // Appending - if (self.type === 'a') - self.position = stats.size; - else - self.position = 0; - + self.position = 0; callback && callback(err); }); }); @@ -217,6 +201,8 @@ NoSQLStream.prototype.close = function(callback) { self.canceled = false; self.fd = null; self.type = null; + self.docscache = null; + self.docs = null; } else if (callback) callback(); @@ -224,48 +210,59 @@ NoSQLStream.prototype.close = function(callback) { return self; }; -NoSQLStream.prototype.write = function(docs) { +NoSQLStream.prototype.write = function(doc, position) { var self = this; - self.bufferstack.push(docs); + self.bufferstack.push({ position: position, data: doc }); + !self.writing && self.$write(); + return self; +}; + +NoSQLStream.prototype.write2 = function(doc) { + var self = this; + self.bufferstacknew.push(U.createBuffer(doc)); !self.writing && self.$write(); return self; }; NoSQLStream.prototype.$write = function() { var self = this; - self.writing = true; - //var buf = framework_utils.createBuffer(self.bufferstack.shift(), 'utf8'); - Fs.write(self.fdwrite, self.bufferstack.shift(), 'utf8', function() { - self.bufferstack.length && self.$write(); - self.writing = false; - }); + if (self.bufferstacknew.length && self.bufferstack.length) { + self.writing = true; + var buf = self.bufferstacknew.splice(0, 5); + buf = buf.length > 1 ? Buffer.concat(buf) : buf[0]; + Fs.write(self.fd, buf, 0, buf.length, self.positionappend, function(err, size) { + self.positionappend += size; + var item = self.bufferstack.shift(); + Fs.write(self.fd, item.data, item.position, 'utf8', function() { + self.writing = false; + self.$write(); + }); + }); + } else if (self.bufferstacknew.length) { + self.writing = true; + var buf = self.bufferstacknew.splice(0, 5); + buf = buf.length > 1 ? Buffer.concat(buf) : buf[0]; + Fs.write(self.fd, buf, 0, buf.length, self.positionappend, function(err, size) { + self.positionappend += size; + self.writing = false; + self.$write(); + }); + } else if (self.bufferstack.length) { + self.writing = true; + var item = self.bufferstack.shift(); + Fs.write(self.fd, item.data, item.position, 'utf8', function() { + self.writing = false; + self.$write(); + }); + } }; NoSQLStream.prototype.flush = function(callback) { var self = this; if (self.writing) { setTimeout((self, callback) => self.flush(callback), 100, self, callback); - } else { - self.close(function(err) { - - if (err) { - callback(err); - return; - } - - Fs.close(self.fdwrite, function(err) { - if (err) - callback(err); - else { - if (self.changed) - Fs.rename(self.filename + '-tmp', self.filename, callback); - else - Fs.unlink(self.filename + '-tmp', callback); - } - }); - - }); - } + } else + self.close(callback || NOOP); return self; }; @@ -280,10 +277,6 @@ NoSQLStream.prototype.read = function(callback, noclose) { self.ondocuments(); self.docscount = 0; self.docs = ''; - - if (self.fdwrite) - self.docscache = ''; - } if (noclose) @@ -296,9 +289,9 @@ NoSQLStream.prototype.read = function(callback, noclose) { size = size < BUFFERSIZE ? size : BUFFERSIZE; var buffer = framework_utils.createBufferSize(size); + Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { - self.bytesread += size; self.position += size; var beg = 0; @@ -317,18 +310,22 @@ NoSQLStream.prototype.read = function(callback, noclose) { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); while (index !== -1) { var tmp = self.buffer.toString('utf8', 0, index); - self.docs += (self.docs ? ',' : '') + tmp; + if (tmp[0] === '-') { + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + continue; + } + + self.docs += (self.docs ? self.divider : '') + tmp; self.docscount++; self.indexer++; - if (self.fdwrite) - self.docscache += tmp + NEWLINE; - if (self.docscount >= BUFFERDOCS) { if (self.ondocuments() === false) @@ -337,14 +334,10 @@ NoSQLStream.prototype.read = function(callback, noclose) { self.docs = ''; self.docscount = 0; - if (self.fdwrite) - self.docscache = ''; - if (self.canceled) { self.read(callback, noclose); return; } - } self.buffer = self.buffer.slice(index + 1); @@ -353,10 +346,23 @@ NoSQLStream.prototype.read = function(callback, noclose) { break; } - self.read(callback, noclose); + self.ticks++; + + if (self.ticks % 5 === 0) + setImmediate(readnext, self, callback, noclose); + else + self.read(callback, noclose); }); }; +function readnext(self, callback, close) { + self.read(callback, close); +} + +function readnextreverse(self, callback, close) { + self.readreverse(callback, close, true); +} + NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { var self = this; @@ -384,7 +390,6 @@ NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { size = size < BUFFERSIZE ? size : BUFFERSIZE; self.position -= size; - var buffer = framework_utils.createBufferSize(size); Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { @@ -402,7 +407,16 @@ NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { while (index !== -1) { - self.docs += (self.docs ? ',' : '') + self.buffer.toString('utf8', index); + var tmp = self.buffer.toString('utf8', index); + if (tmp[1] === '-') { + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + continue; + } + + self.docs += (self.docs ? self.divider : '') + tmp; self.docscount++; if (self.docscount >= BUFFERDOCS) { @@ -424,25 +438,107 @@ NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { } if (self.position === 0 && self.buffer.length) { - self.docs += (self.docs ? ',' : '') + self.buffer.toString('utf8'); - self.docscount++; - if (self.docscount >= BUFFERDOCS) { - self.ondocuments(); - self.docs = ''; - self.docscount = 0; + var tmp = self.buffer.toString('utf8'); + if (tmp[0] !== '"') { + self.docs += (self.docs ? self.divider : '') + tmp; + self.docscount++; + if (self.docscount >= BUFFERDOCS) { + self.ondocuments(); + self.docs = ''; + self.docscount = 0; + } } } - self.readreverse(callback, noclose, true); + self.ticks++; + + if (self.ticks % 5 === 0) + setImmediate(readnextreverse, self, callback, noclose); + else + self.readreverse(callback, noclose, true); }); }; -NoSQLStream.prototype.append = function(data, callback) { +NoSQLStream.prototype.readupdate = function(callback, noclose) { + var self = this; - Fs.write(self.fd, data, 'utf8', function(err) { - callback && callback(err); + var size = self.stats.size - self.position; + + if (!self.fd || size <= 0 || self.canceled) { + + if (self.docsbuffer.length) { + self.ondocuments(); + self.docsbuffer = []; + self.docs = ''; + } + + if (noclose) + callback && callback(); + else + self.close(err => callback && callback(err)); + + return; + } + + size = size < BUFFERSIZE ? size : BUFFERSIZE; + var buffer = framework_utils.createBufferSize(size); + + Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { + + self.position += size; + + var beg = 0; + + if (self.buffer) { + self.cache[0] = self.buffer; + self.cache[1] = chunk; + + beg = self.buffer.length - 1; + + if (beg < 0) + beg = 0; + + self.buffer = Buffer.concat(self.cache); + + } else + self.buffer = chunk; + + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index); + + if (tmp[0] !== '-') { + self.docs += (self.docs ? ',' : '') + tmp; + self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); + self.docscount++; + if (self.docsbuffer.length >= BUFFERDOCS) { + self.ondocuments(); + self.docsbuffer = []; + self.docs = ''; + } + } + + self.positionupdate += Buffer.byteLength(tmp, 'utf8') + 1; + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + + if (index === -1) + break; + } + + if (self.bufferstack.length || self.bufferstacknew.length) { + var fn = function() { + if (self.bufferstack.length || self.bufferstacknew.length) + setImmediate(fn); + else + self.readupdate(callback, noclose); + }; + setImmediate(fn); + } else + self.readupdate(callback, noclose); }); - return self; }; module.exports = NoSQLStream; \ No newline at end of file From 31861f528e66728b832908b013b4fa459c17769a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 12 Apr 2018 20:41:27 +0200 Subject: [PATCH 0209/1669] Fixed sorting. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index c138dbc1a..fffca6dd7 100755 --- a/nosql.js +++ b/nosql.js @@ -3090,7 +3090,7 @@ DatabaseBuilder.prototype.repository = function(key, value) { DatabaseBuilder.prototype.compile = function() { var self = this; var raw = self.$code.join(''); - var code = 'var repository=$F.repository;var options=$F.options;var arg=$F.arg;var fn=$F.fn;var $is=false;var $tmp;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + 'if(!options.fields)return doc;var $doc={};for(var $i=0;$i Date: Fri, 13 Apr 2018 10:14:13 +0200 Subject: [PATCH 0210/1669] Improved reading of database. --- nosql.js | 181 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 118 insertions(+), 63 deletions(-) diff --git a/nosql.js b/nosql.js index fffca6dd7..dcf6d21e8 100755 --- a/nosql.js +++ b/nosql.js @@ -501,6 +501,8 @@ function Database(name, filename, readonly) { self.$timeoutmeta; self.$events = {}; self.$free = true; + self.$writting = false; + self.$reading = false; } Database.prototype.emit = function(name, a, b, c, d, e, f, g) { @@ -993,79 +995,105 @@ Database.prototype.view = function(name) { return builder; }; +// 1 append +// 2 update +// 3 remove +// 4 reader +// 5 views +// 6 reader views +// 7 drop +// 8 backup +// 9 restore +// 10 streamer +// 11 reader reverse +// 12 clear +// 13 clean + +const NEXTWAIT = { 7: true, 8: true, 9: true, 12: true, 13: true }; + Database.prototype.next = function(type) { - if (type && this.step) + if (type && NEXTWAIT[this.step]) return; - if (this.step !== 2 && this.pending_update.length) { - if (INMEMORY[this.name]) - this.$update_inmemory(); - else - this.$update(); - return; - } + if (!this.$writting && !this.$reading) { - if (this.step !== 3 && this.pending_remove.length) { - if (INMEMORY[this.name]) - this.$remove_inmemory(); - else - this.$remove(); - return; - } + if (this.step !== 12 && this.pending_clear.length) { + if (INMEMORY[this.name]) + this.$clear_inmemory(); + else + this.$clear(); + return; + } - if (this.step !== 12 && this.pending_clear.length) { - if (INMEMORY[this.name]) - this.$clear_inmemory(); - else - this.$clear(); - return; - } + if (this.step !== 13 && this.pending_clean.length) { + this.$clean(); + return; + } - if (this.step !== 13 && this.pending_clean.length) { - this.$clean(); - return; + if (this.step !== 7 && !this.pending_reindex && this.pending_drops) { + this.$drop(); + return; + } } - if (this.step !== 6 && this.pending_reader_view.length) { - this.$readerview(); - return; - } + if (!this.$writting) { - if (this.step !== 4 && this.pending_reader.length) { - this.$reader(); - return; - } + if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) { + if (INMEMORY[this.name]) + this.$append_inmemory(); + else + this.$append(); + return; + } - if (this.step !== 11 && this.pending_reader2.length) { - this.$reader3(); - return; - } + if (this.step !== 2 && !this.$writting && this.pending_update.length) { + if (INMEMORY[this.name]) + this.$update_inmemory(); + else + this.$update(); + return; + } - if (this.step !== 7 && !this.pending_reindex && this.pending_drops) { - this.$drop(); - return; - } + if (this.step !== 3 && !this.$writting && this.pending_remove.length) { + if (INMEMORY[this.name]) + this.$remove_inmemory(); + else + this.$remove(); + return; + } - if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) { - if (INMEMORY[this.name]) - this.$append_inmemory(); - else - this.$append(); - return; } - if (this.step !== 5 && this.pending_views) { - if (INMEMORY[this.name]) - this.$views_inmemory(); - else - this.$views(); - return; - } + if (!this.$reading) { - if (this.step !== 10 && this.pending_streamer.length) { - this.$streamer(); - return; + if (this.step !== 4 && this.pending_reader.length) { + this.$reader(); + return; + } + + if (this.step !== 11 && this.pending_reader2.length) { + this.$reader3(); + return; + } + + if (this.step !== 10 && this.pending_streamer.length) { + this.$streamer(); + return; + } + + if (this.step !== 6 && this.pending_reader_view.length) { + this.$readerview(); + return; + } + + if (this.step !== 5 && this.pending_views) { + if (INMEMORY[this.name]) + this.$views_inmemory(); + else + this.$views(); + return; + } } if (this.step !== type) { @@ -1170,6 +1198,8 @@ Database.prototype.$append = function() { return; } + self.$writting = true; + self.pending_append.splice(0).limit(JSONBUFFER, function(items, next) { var json = ''; @@ -1197,6 +1227,7 @@ Database.prototype.$append = function() { }; function next_append(self) { + self.$writting = false; self.next(0); self.views && setImmediate(views_refresh, self); } @@ -1236,6 +1267,8 @@ Database.prototype.$update = function() { return self; } + self.$writting = true; + var filter = self.pending_update.splice(0); var length = filter.length; var backup = false; @@ -1343,6 +1376,7 @@ Database.prototype.$update = function() { } fs = null; + self.$writting = false; self.next(0); self.views && setImmediate(views_refresh, self); }); @@ -1457,10 +1491,15 @@ Database.prototype.$reader = function() { } var list = self.pending_reader.splice(0); - if (INMEMORY[self.name]) + if (INMEMORY[self.name]) { self.$reader2_inmemory('#', list, () => self.next(0)); - else - self.$reader2(self.filename, list, () => self.next(0)); + } else { + self.$reading = true; + self.$reader2(self.filename, list, function() { + self.$reading = false; + self.next(0); + }); + } return self; }; @@ -1697,6 +1736,8 @@ Database.prototype.$reader3 = function() { return self; } + self.$reading = true; + var filter = self.pending_reader2.splice(0); var length = filter.length; var first = true; @@ -1866,6 +1907,7 @@ Database.prototype.$reader3 = function() { builder.done(); } + self.$reading = false; fs = null; self.next(0); }); @@ -1885,6 +1927,8 @@ Database.prototype.$streamer = function() { return self; } + self.$reading = true; + var filter = self.pending_streamer.splice(0); var length = filter.length; var count = 0; @@ -1904,6 +1948,7 @@ Database.prototype.$streamer = function() { fs.read(function() { for (var i = 0; i < length; i++) filter[i].callback && filter[i].callback(null, filter[i].repository, count); + self.$reading = false; self.next(0); fs = null; }); @@ -2147,6 +2192,8 @@ Database.prototype.$views = function() { var fs = new NoSQLStream(self.filename); var indexer = 0; + self.$reading = true; + fs.ondocuments = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); for (var i = 0; i < docs.length; i++) { @@ -2201,7 +2248,10 @@ Database.prototype.$views = function() { next(); }); }); - }, () => self.next(0), 5); + }, function() { + self.$reading = false; + self.next(0); + }, 5); }); }); @@ -2292,6 +2342,8 @@ Database.prototype.$remove = function() { return; } + self.$writting = true; + var fs = new NoSQLStream(self.filename); var filter = self.pending_remove.splice(0); var length = filter.length; @@ -2362,6 +2414,7 @@ Database.prototype.$remove = function() { item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } fs = null; + self.$writting = false; self.next(0); change && self.views && setImmediate(views_refresh, self); }); @@ -2420,7 +2473,9 @@ Database.prototype.$clean = function() { var writer = Fs.createWriteStream(self.filename + '-tmp'); fs.divider = NEWLINE; - fs.ondocuments = () => writer.write(fs.docs + NEWLINE); + fs.ondocuments = function() { + writer.write(fs.docs + NEWLINE); + }; writer.on('finish', function() { Fs.rename(self.filename + '-tmp', self.filename, function() { @@ -5867,4 +5922,4 @@ function errorhandling(err, builder, response) { function jsonparser(key, value) { return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; -} +} \ No newline at end of file From 6e9c95efba021817cf3a16f29181a6c4d7857380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 13 Apr 2018 10:28:37 +0200 Subject: [PATCH 0211/1669] Improved handling error in `String.parseConfig()`. --- changes.txt | 1 + utils.js | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index f3c2d75dd..4b55287d2 100755 --- a/changes.txt +++ b/changes.txt @@ -54,6 +54,7 @@ - updated: `schema.define('name', null)` removes a schema field - updated: Chunker supports `compression`, default `true` - updated: Chunker supports `autoremove` processed files in `each()` or `read()` method, default `true` +- updated: `String.parseConfig(def, [onError])` can handle errors better - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/utils.js b/utils.js index 6c2cefc90..4f1fae76c 100755 --- a/utils.js +++ b/utils.js @@ -3160,12 +3160,19 @@ String.prototype.localeCompare2 = function(value) { /** * Parse configuration from a string * @param {Object} def + * @onerr {Function} error handling * @return {Object} */ -String.prototype.parseConfig = function(def) { +String.prototype.parseConfig = function(def, onerr) { + + if (typeof(def) === 'function') { + onerr = def; + def = null; + } + var arr = this.split('\n'); var length = arr.length; - var obj = exports.extend({}, def); + var obj = def ? exports.extend({}, def) : {}; var subtype; var name; var index; @@ -3214,7 +3221,14 @@ String.prototype.parseConfig = function(def) { case 'eval': case 'object': case 'array': - obj[name] = new Function('return ' + value)(); + try { + obj[name] = new Function('return ' + value)(); + } catch (e) { + if (onerr) + onerr(e, arr[i]); + else + throw new Error('A value of "{0}" can\'t be converted to "{1}": '.format(name, subtype) + e.toString()); + } break; case 'json': obj[name] = value.parseJSON(true); From 7bede8436d88fc74efc45b0a2362bbf0f08fec80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 13 Apr 2018 14:13:43 +0200 Subject: [PATCH 0212/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0716dba42..915243978 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-40", + "version": "3.0.0-41", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From f0d5b802f331bb9585590ee8976d32cfd48d3d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 13 Apr 2018 21:02:04 +0200 Subject: [PATCH 0213/1669] Fixed `route` property in `controller`. --- index.js | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index 8746f073b..ede485500 100755 --- a/index.js +++ b/index.js @@ -9646,6 +9646,7 @@ function Controller(name, req, res, currentView) { if (req) { this.language = req.$language; this.req = req; + this.route = req.$total_route; } else this.req = EMPTYREQUEST; @@ -9693,27 +9694,23 @@ Controller.prototype = { }, get schema() { - return this.req.$total_route.schema[0] === 'default' ? this.req.$total_route.schema[1] : this.req.$total_route.schema.join('/'); + return this.route.schema[0] === 'default' ? this.route.schema[1] : this.route.schema.join('/'); }, get workflow() { - return this.req.$total_route.schema_workflow; + return this.route.schema_workflow; }, get sseID() { return this.req.headers['last-event-id'] || null; }, - get route() { - return this.req.$total_route; - }, - get options() { - return this.req.$total_route.options; + return this.route.options; }, get flags() { - return this.req.$total_route.flags; + return this.route.flags; }, get path() { @@ -9824,7 +9821,7 @@ Controller.prototype = { get params() { if (this.$params) return this.$params; - var route = this.req.$total_route; + var route = this.route; var names = route.paramnames; if (names) { var obj = {}; @@ -10001,7 +9998,7 @@ Controller.prototype.$async = function(callback, index) { }; Controller.prototype.getSchema = function() { - var route = this.req.$total_route; + var route = this.route; if (!route.schema || !route.schema[1]) throw new Error('The controller\'s route does not define any schema.'); var schema = GETSCHEMA(route.schema[0], route.schema[1]); @@ -10310,7 +10307,7 @@ Controller.prototype.transfer = function(url, flags) { // Hidden variable self.req.$path = framework_internal.routeSplit(url, true); - self.req.$total_route = selected; + self.route = self.req.$total_route = selected; self.req.$total_execute(404); return true; }; @@ -12287,7 +12284,7 @@ Controller.prototype.sse = function(data, eventname, id, retry) { self.type = 1; if (retry === undefined) - retry = self.req.$total_route.timeout; + retry = self.route.timeout; self.req.$total_success(); self.req.on('close', () => self.close()); From bf4ed0eff7c4646507acb432ac1081bbd62cb8d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Apr 2018 10:28:05 +0200 Subject: [PATCH 0214/1669] Added `wait` for cleaning DB. --- index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index ede485500..b9fa3e5d9 100755 --- a/index.js +++ b/index.js @@ -6996,8 +6996,9 @@ F.service = function(count) { if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { var keys = Object.keys(F.databasescleaner); - for (var i = 0; i < keys.length; i++) - NOSQL(keys[i]).clean(); + keys.wait(function(item, next) { + NOSQL(item).clean(next); + }); } F.$events.service && F.emit('service', count); From df5cbf5246296381bf82766859a7de76e60bd3e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Apr 2018 10:41:52 +0200 Subject: [PATCH 0215/1669] Improved NoSQL DB. --- changes.txt | 1 + nosql.js | 415 ++++++++++++++++-------------- nosqlstream.js | 686 +++++++++++++++++++++++++++---------------------- nosqlworker.js | 9 + 4 files changed, 619 insertions(+), 492 deletions(-) diff --git a/changes.txt b/changes.txt index 4b55287d2..4dd69312a 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,7 @@ - added: `route.groups` with defined groups - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) +- added: `Database.lock(callback(next))` locks all internal DB operations - added: new directory `schemas` with a new configuration item `directory-schemas' - added: new directory `operations` with a new configuration item `directory-operations' - added: `String.crc32([unsigned])` diff --git a/nosql.js b/nosql.js index dcf6d21e8..8b53cced9 100755 --- a/nosql.js +++ b/nosql.js @@ -107,7 +107,6 @@ exports.worker = function() { if (item && item.time) { var diff = time - item.time; if (diff >= 60000) { - delete FORKCALLBACKS[key]; var err = new Error('NoSQL worker timeout.'); switch (item.type) { @@ -122,6 +121,10 @@ exports.worker = function() { case 'remove': item.builder && item.builder.$callback(err, EMPTYOBJECT, EMPTYOBJECT); break; + case 'clean': + case 'clear': + item.callback && item.callback(err); + break; default: item.callback && item.callback(err, EMPTYOBJECT, EMPTYOBJECT); break; @@ -174,6 +177,11 @@ exports.worker = function() { var obj = FORKCALLBACKS[msg.id]; obj && obj.callback && obj.callback(msg.err, msg.response, msg.repository); break; + case 'callback': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.callback && obj.callback(msg.err); + break; + } delete FORKCALLBACKS[msg.id]; }); @@ -321,6 +329,11 @@ exports.worker = function() { return this; }; + Database.prototype.clean = function(callback) { + send(this, 'clean').callback = callback; + return this; + }; + Database.prototype.remove = function(filename) { return send(this, 'remove', filename).builder = new DatabaseBuilder(this); }; @@ -489,6 +502,7 @@ function Database(name, filename, readonly) { self.pending_streamer = []; self.pending_clean = []; self.pending_clear = []; + self.pending_locks = []; self.views = null; self.step = 0; self.pending_drops = false; @@ -878,6 +892,13 @@ Database.prototype.clean = function(callback) { return self; }; +Database.prototype.lock = function(callback) { + var self = this; + self.pending_locks.push(callback || NOOP); + setImmediate(next_operation, self, 14); + return self; +}; + Database.prototype.remove = function(filename) { var self = this; self.readonly && self.throwReadonly(); @@ -909,6 +930,10 @@ Database.prototype.find = function(view) { Database.prototype.find2 = function() { var self = this; + + if (self.readonly) + return self.find(); + var builder = new DatabaseBuilder(self); self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); setImmediate(next_operation, self, 11); @@ -1008,8 +1033,9 @@ Database.prototype.view = function(name) { // 11 reader reverse // 12 clear // 13 clean +// 14 locks -const NEXTWAIT = { 7: true, 8: true, 9: true, 12: true, 13: true }; +const NEXTWAIT = { 7: true, 8: true, 9: true, 12: true, 13: true, 14: true }; Database.prototype.next = function(type) { @@ -1035,6 +1061,11 @@ Database.prototype.next = function(type) { this.$drop(); return; } + + if (this.step !== 14 && this.pending_locks.length) { + this.$lock(); + return; + } } if (!this.$writting) { @@ -1304,15 +1335,16 @@ Database.prototype.$update = function() { if (output) { if (item.keys) { - for (var j = 0, jl = item.keys.length; j < jl; j++) { - var val = item.doc[item.keys[j]]; + for (var j = 0; j < item.keys.length; j++) { + var key = item.keys[j]; + var val = item.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - output[item.keys[j]] = val(output[item.keys[j]], output); + output[key] = val(output[key], output); else - output[item.keys[j]] = val; + output[key] = val; } else - output[item.keys[j]] = undefined; + output[key] = undefined; } } else output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; @@ -1358,34 +1390,29 @@ Database.prototype.$update = function() { } }; - var finish = function() { - fs.flush(function() { + fs.$callback = function() { - if (self.indexes) - F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; + if (self.indexes) + F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert); - self.insert(item.insert).$callback = item.builder.$callback; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } + for (var i = 0; i < length; i++) { + var item = filter[i]; + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert); + self.insert(item.insert).$callback = item.builder.$callback; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); } + } - fs = null; - self.$writting = false; - self.next(0); - self.views && setImmediate(views_refresh, self); - }); + fs = null; + self.$writting = false; + self.next(0); + self.views && setImmediate(views_refresh, self); }; - fs.openupdate(function() { - fs.readupdate(finish, true); - }); - + fs.openupdate(); return self; }; @@ -1666,61 +1693,59 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { } }; - var cb = function() { - fs.read(function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else - output = item.response || EMPTYARRAY; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } + fs.$callback = function() { + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + var output; - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } + if (builder.$options.scalar || !builder.$options.sort) { - if (builder.$options.first) + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); + continue; } - fs = null; - callback(); - }); + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || EMPTYARRAY; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + builder.done(); + } + + fs = null; + callback(); }; if (reader) - fs.openstream(reader, cb); + fs.openstream(reader); else - fs.openread(cb); + fs.openread(); return self; }; @@ -1861,59 +1886,58 @@ Database.prototype.$reader3 = function() { } }; - var cb = function() { - fs.readreverse(function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; + fs.$callback = function() { - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else - output = item.response || EMPTYARRAY; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + var output; - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } + if (builder.$options.scalar || !builder.$options.sort) { - if (builder.$options.first) + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else output = item.response || EMPTYARRAY; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); + continue; } - self.$reading = false; - fs = null; - self.next(0); - }); + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || EMPTYARRAY; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + builder.done(); + } + + self.$reading = false; + fs = null; + self.next(0); }; - fs.openread(cb); + fs.openreadreverse(); return self; }; @@ -1944,16 +1968,15 @@ Database.prototype.$streamer = function() { } }; - fs.openread(function() { - fs.read(function() { - for (var i = 0; i < length; i++) - filter[i].callback && filter[i].callback(null, filter[i].repository, count); - self.$reading = false; - self.next(0); - fs = null; - }); - }); + fs.$callback = function() { + for (var i = 0; i < length; i++) + filter[i].callback && filter[i].callback(null, filter[i].repository, count); + self.$reading = false; + self.next(0); + fs = null; + }; + fs.openread(); return self; }; @@ -2214,47 +2237,46 @@ Database.prototype.$views = function() { } }; - fs.openread(function() { - fs.read(function() { - response.wait(function(item, next) { + fs.$callback = function() { + response.wait(function(item, next) { - var builder = item.builder; + var builder = item.builder; - if (builder.$options.sort) { - if (builder.$options.sort.name) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - else if (builder.$options.sort === EMPTYOBJECT) - item.response.random(); - else - item.response.sort(builder.$options.sort); - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } + if (builder.$options.sort) { + if (builder.$options.sort.name) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + else if (builder.$options.sort === EMPTYOBJECT) + item.response.random(); + else + item.response.sort(builder.$options.sort); + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } - var filename = builder.$filename; - Fs.unlink(filename, function() { - item.response.limit(20, function(items, next) { - var builder = []; - for (var i = 0, length = items.length; i < length; i++) - builder.push(JSON.stringify(items[i])); - Fs.appendFile(filename, builder.join(NEWLINE) + NEWLINE, next); - }, function() { - // clears in-memory - self.inmemory[item.name] = undefined; - next(); - }); + var filename = builder.$filename; + Fs.unlink(filename, function() { + item.response.limit(20, function(items, next) { + var builder = []; + for (var i = 0, length = items.length; i < length; i++) + builder.push(JSON.stringify(items[i])); + Fs.appendFile(filename, builder.join(NEWLINE) + NEWLINE, next); + }, function() { + // clears in-memory + self.inmemory[item.name] = undefined; + next(); }); - }, function() { - self.$reading = false; - self.next(0); - }, 5); - }); - }); + }); + }, function() { + self.$reading = false; + self.next(0); + }, 5); + }; + fs.openread(); }; Database.prototype.$views_inmemory = function() { @@ -2402,27 +2424,24 @@ Database.prototype.$remove = function() { } }; - var finish = function() { - fs.flush(function() { + fs.$callback = function() { - if (self.indexes) - F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; + if (self.indexes) + F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - fs = null; - self.$writting = false; - self.next(0); - change && self.views && setImmediate(views_refresh, self); - }); + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + } + + fs = null; + self.$writting = false; + self.next(0); + change && self.views && setImmediate(views_refresh, self); }; - fs.openupdate(function() { - fs.readupdate(finish, true); - }); + fs.openupdate(); }; Database.prototype.$clear = function() { @@ -2458,7 +2477,7 @@ Database.prototype.$clean = function() { var self = this; self.step = 13; - if (!self.clean.length) { + if (!self.pending_clean.length) { self.next(0); return; } @@ -2473,10 +2492,15 @@ Database.prototype.$clean = function() { var writer = Fs.createWriteStream(self.filename + '-tmp'); fs.divider = NEWLINE; + fs.ondocuments = function() { writer.write(fs.docs + NEWLINE); }; + fs.$callback = function() { + writer.end(); + }; + writer.on('finish', function() { Fs.rename(self.filename + '-tmp', self.filename, function() { F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); @@ -2488,7 +2512,25 @@ Database.prototype.$clean = function() { }); }); - fs.openread(() => fs.read(() => writer.end())); + fs.openread(); +}; + +Database.prototype.$lock = function() { + + var self = this; + self.step = 14; + + if (!self.pending_locks.length) { + self.next(0); + return; + } + + var filter = self.pending_locks.splice(0); + filter.wait(function(fn, next) { + fn.call(self, next); + }, function() { + self.next(0); + }); }; Database.prototype.$remove_inmemory = function() { @@ -5757,18 +5799,17 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { } }; - reader.openread(function() { - reader.read(function() { - stats.processed++; - if (item.date === today) { - self.locked_writer--; - if (self.locked_writer <= 0 && self.pending.length) - self.insert(); - } - setImmediate(next); - }); - }); + reader.$callback = function() { + stats.processed++; + if (item.date === today) { + self.locked_writer--; + if (self.locked_writer <= 0 && self.pending.length) + self.insert(); + } + setImmediate(next); + }; + reader.openread(); }; storage.wait(function(item, next, index) { diff --git a/nosqlstream.js b/nosqlstream.js index fb340bbd5..8982920c9 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -49,248 +49,15 @@ function NoSQLStream(filename) { // this.indexer = 0; } -NoSQLStream.prototype.openread = function(callback) { - var self = this; - self.type = 'r'; - self.position = 0; - self.open(callback); - return self; -}; - -NoSQLStream.prototype.openupdate = function(callback) { - var self = this; - self.type = 'r+'; - self.open(function(err) { - - // File may not exist - if (err) { - callback(); - return; - } - - self.position = 0; - self.positionappend = self.stats.size; - self.positionupdate = 0; - self.bufferstack = []; - self.bufferstacknew = []; - self.docsbuffer = []; - - callback && callback(); - }); - - return self; -}; - -NoSQLStream.prototype.openinsert = function(callback) { - var self = this; - self.type = 'a'; - self.open(callback); - return self; -}; - -// For e.g. files on URL address -NoSQLStream.prototype.openstream = function(stream, callback) { - - var self = this; - - self.docs = ''; - self.docscount = 0; - - self.stream = stream; - self.stream.on('end', function() { - if (self.docscount) { - self.ondocuments(); - self.docscount = 0; - self.docs = ''; - } - callback && callback(); - }); - - self.stream.on('data', function(chunk) { - - var beg = 0; - - if (self.buffer) { - self.cache[0] = self.buffer; - self.cache[1] = chunk; - self.buffer = Buffer.concat(self.cache); - - beg = self.cache[0].length - 1; - - if (beg < 0) - beg = 0; - - } else - self.buffer = chunk; - - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); - while (index !== -1) { - - var tmp = self.buffer.toString('utf8', 0, index).trim(); - - self.docs += (self.docs ? self.divider : '') + tmp; - self.docscount++; - self.indexer++; - - if (self.docscount >= BUFFERDOCS) { - - if (self.ondocuments() === false) - self.canceled = true; - - self.docs = ''; - self.docscount = 0; - - if (self.canceled) { - self.close(); - self.stream.destroy && self.stream.destroy(); - return; - } - - } - - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; - } - }); - - return self; -}; - -NoSQLStream.prototype.open = function(callback) { - var self = this; - Fs.open(self.filename, self.type, function(err, fd) { - - if (err) { - callback && callback.call(err); - return; - } - - Fs.fstat(fd, function(err, stats) { - self.docs = ''; - self.docscount = 0; - self.fd = fd; - self.stats = stats; - self.position = 0; - callback && callback(err); - }); - }); -}; - -NoSQLStream.prototype.close = function(callback) { - - var self = this; - - if (self.fd) { - - self.stream = null; - - Fs.close(self.fd, function(err) { - err && F.error(err); - callback && callback(); - }); - - if (self.buffer) { - self.buffer = null; - self.cache[0] = null; - self.cache[1] = null; - self.bytesread = 0; - } - - self.canceled = false; - self.fd = null; - self.type = null; - self.docscache = null; - self.docs = null; - - } else if (callback) - callback(); - - return self; -}; - -NoSQLStream.prototype.write = function(doc, position) { - var self = this; - self.bufferstack.push({ position: position, data: doc }); - !self.writing && self.$write(); - return self; -}; - -NoSQLStream.prototype.write2 = function(doc) { - var self = this; - self.bufferstacknew.push(U.createBuffer(doc)); - !self.writing && self.$write(); - return self; -}; - -NoSQLStream.prototype.$write = function() { - var self = this; - if (self.bufferstacknew.length && self.bufferstack.length) { - self.writing = true; - var buf = self.bufferstacknew.splice(0, 5); - buf = buf.length > 1 ? Buffer.concat(buf) : buf[0]; - Fs.write(self.fd, buf, 0, buf.length, self.positionappend, function(err, size) { - self.positionappend += size; - var item = self.bufferstack.shift(); - Fs.write(self.fd, item.data, item.position, 'utf8', function() { - self.writing = false; - self.$write(); - }); - }); - } else if (self.bufferstacknew.length) { - self.writing = true; - var buf = self.bufferstacknew.splice(0, 5); - buf = buf.length > 1 ? Buffer.concat(buf) : buf[0]; - Fs.write(self.fd, buf, 0, buf.length, self.positionappend, function(err, size) { - self.positionappend += size; - self.writing = false; - self.$write(); - }); - } else if (self.bufferstack.length) { - self.writing = true; - var item = self.bufferstack.shift(); - Fs.write(self.fd, item.data, item.position, 'utf8', function() { - self.writing = false; - self.$write(); - }); - } -}; +NoSQLStream.prototype.readhelpers = function() { -NoSQLStream.prototype.flush = function(callback) { var self = this; - if (self.writing) { - setTimeout((self, callback) => self.flush(callback), 100, self, callback); - } else - self.close(callback || NOOP); - return self; -}; -NoSQLStream.prototype.read = function(callback, noclose) { + self.cb_read = function() { + self.read(); + }; - var self = this; - var size = self.stats.size - self.position; - - if (!self.fd || size <= 0 || self.canceled) { - - if (self.docscount) { - self.ondocuments(); - self.docscount = 0; - self.docs = ''; - } - - if (noclose) - callback && callback(); - else - self.close(err => callback && callback(err)); - - return; - } - - size = size < BUFFERSIZE ? size : BUFFERSIZE; - var buffer = framework_utils.createBufferSize(size); - - Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { + self.cb_readbuffer = function(err, size, chunk) { self.position += size; @@ -335,7 +102,7 @@ NoSQLStream.prototype.read = function(callback, noclose) { self.docscount = 0; if (self.canceled) { - self.read(callback, noclose); + self.read(self.$callback, self.$noclose); return; } } @@ -349,50 +116,20 @@ NoSQLStream.prototype.read = function(callback, noclose) { self.ticks++; if (self.ticks % 5 === 0) - setImmediate(readnext, self, callback, noclose); + setImmediate(self.cb_readticks); else - self.read(callback, noclose); - }); -}; - -function readnext(self, callback, close) { - self.read(callback, close); -} + self.read(); + }; -function readnextreverse(self, callback, close) { - self.readreverse(callback, close, true); -} - -NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { - - var self = this; + self.cb_readticks = function() { + self.read(); + }; - if (repeat == null) - self.position = self.stats.size; + self.cb_readreverse = function() { + self.readreverse2(); + }; - if (!self.fd || self.position <= 0 || self.canceled) { - - if (self.docscount) { - self.ondocuments(); - self.docs = ''; - self.docscount = 0; - } - - if (noclose) - callback && callback(); - else - self.close(err => callback && callback(err)); - - return; - } - - var size = self.stats.size - self.bytesread; - size = size < BUFFERSIZE ? size : BUFFERSIZE; - - self.position -= size; - var buffer = framework_utils.createBufferSize(size); - - Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { + self.cb_readreversebuffer = function(err, size, chunk) { self.bytesread += size; @@ -427,7 +164,7 @@ NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { } if (self.canceled) { - self.readreverse(callback, noclose, true); + self.readreverse2(); return; } @@ -453,37 +190,125 @@ NoSQLStream.prototype.readreverse = function(callback, noclose, repeat) { self.ticks++; if (self.ticks % 5 === 0) - setImmediate(readnextreverse, self, callback, noclose); + setImmediate(self.cb_readreverseticks); else - self.readreverse(callback, noclose, true); - }); + self.readreverse2(); + }; + + self.cb_readreverseticks = function() { + self.readreverse2(); + }; + + self.cb_readstream = function(chunk) { + + var beg = 0; + + if (self.buffer) { + + self.cache[0] = self.buffer; + self.cache[1] = chunk; + self.buffer = Buffer.concat(self.cache); + + beg = self.cache[0].length - 1; + + if (beg < 0) + beg = 0; + + } else + self.buffer = chunk; + + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index).trim(); + + self.docs += (self.docs ? self.divider : '') + tmp; + self.docscount++; + self.indexer++; + + if (self.docscount >= BUFFERDOCS) { + + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = ''; + self.docscount = 0; + + if (self.canceled) { + self.stream.destroy && self.stream.destroy(); + return; + } + + } + + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } + }; + }; -NoSQLStream.prototype.readupdate = function(callback, noclose) { +NoSQLStream.prototype.writehelpers = function() { var self = this; - var size = self.stats.size - self.position; - - if (!self.fd || size <= 0 || self.canceled) { - if (self.docsbuffer.length) { - self.ondocuments(); - self.docsbuffer = []; - self.docs = ''; + self.cb_writeAddUpdAdd = function(err, size) { + if (err) { + console.log('ERROR --> NoSQLstream.writer (add)', err); + self.canceled = true; + self.bufferstacknew.length = 0; + self.bufferstack.length = 0; + self.writing = false; + } else { + self.positionappend += size; + var item = self.bufferstack.shift(); + Fs.write(self.fd, item.data, item.position, 'utf8', self.cb_writeAddUpdUpd); } + }; - if (noclose) - callback && callback(); - else - self.close(err => callback && callback(err)); + self.cb_writeAddUpdUpd = function(err) { - return; - } + self.writing = false; + + if (err) { + console.log('ERROR --> NoSQLstream.writer (upd)', err); + self.canceled = true; + self.bufferstack.length = 0; + self.bufferstacknew.length = 0; + } else + self.$write(); + }; + + self.cb_writeAdd = function(err, size) { + + self.writing = false; + self.positionappend += size; + + if (err) { + console.log('ERROR --> NoSQLstream.writer (add)', err); + self.canceled = true; + } else + self.$write(); + }; + + self.cb_writeUpd = function(err) { + + self.writing = false; + + if (err) { + console.log('ERROR --> NoSQLstream.writer (upd)', err); + self.canceled = true; + } else + self.$write(); + }; - size = size < BUFFERSIZE ? size : BUFFERSIZE; - var buffer = framework_utils.createBufferSize(size); + self.cb_flush = function() { + self.flush(); + }; - Fs.read(self.fd, buffer, 0, size, self.position, function(err, size, chunk) { + self.cb_readwritebuffer = function(err, size, chunk) { self.position += size; @@ -506,9 +331,7 @@ NoSQLStream.prototype.readupdate = function(callback, noclose) { var index = self.buffer.indexOf(NEWLINEBUFFER, beg); while (index !== -1) { - var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] !== '-') { self.docs += (self.docs ? ',' : '') + tmp; self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); @@ -523,22 +346,275 @@ NoSQLStream.prototype.readupdate = function(callback, noclose) { self.positionupdate += Buffer.byteLength(tmp, 'utf8') + 1; self.buffer = self.buffer.slice(index + 1); index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) break; } if (self.bufferstack.length || self.bufferstacknew.length) { + // @TODO: add helper var fn = function() { if (self.bufferstack.length || self.bufferstacknew.length) setImmediate(fn); else - self.readupdate(callback, noclose); + self.readupdate(); }; setImmediate(fn); } else - self.readupdate(callback, noclose); + self.readupdate(); + }; +}; + +NoSQLStream.prototype.openread = function() { + var self = this; + self.type = 'r'; + self.position = 0; + self.open(); + return self; +}; + +NoSQLStream.prototype.openreadreverse = function() { + var self = this; + self.type = 'r'; + self.position = 0; + self.$reverse = true; + self.open(); + return self; +}; + +NoSQLStream.prototype.openupdate = function() { + var self = this; + self.type = 'r+'; + Fs.open(self.filename, self.type, function(err, fd) { + + if (err) { + self.$callback(err); + return; + } + + Fs.fstat(fd, function(err, stats) { + + self.docs = ''; + self.docscount = 0; + + if (err) { + Fs.close(fd, NOOP); + self.$callback(err); + return; + } + + self.docs = ''; + self.docscount = 0; + self.fd = fd; + self.stats = stats; + self.position = 0; + self.positionappend = self.stats.size; + self.positionupdate = 0; + self.bufferstack = []; + self.bufferstacknew = []; + self.docsbuffer = []; + self.writehelpers(); + self.readupdate(); + }); + }); + + return self; +}; + +// For e.g. files on URL address +NoSQLStream.prototype.openstream = function(stream) { + + var self = this; + + var close = function() { + if (self.docscount) { + self.ondocuments(); + self.docscount = 0; + self.docs = ''; + } + self.$callback && self.$callback(); + self.$callback = null; + }; + + self.docs = ''; + self.docscount = 0; + self.readhelpers(); + self.stream = stream; + self.stream.on('error', close); + self.stream.on('end', close); + self.stream.on('data', self.cb_readstream); + return self; +}; + +NoSQLStream.prototype.open = function() { + var self = this; + Fs.open(self.filename, self.type, function(err, fd) { + + if (err) { + self.$callback(err); + return; + } + + Fs.fstat(fd, function(err, stats) { + self.docs = ''; + self.docscount = 0; + self.fd = fd; + self.stats = stats; + self.position = 0; + + if (err) { + Fs.close(fd, NOOP); + self.$callback(err); + return; + } + + self.readhelpers(); + + if (self.$reverse) + self.readreverse(); + else + self.read(); + }); }); }; +NoSQLStream.prototype.close = function() { + + var self = this; + + if (self.fd) { + + self.stream = null; + + Fs.close(self.fd, function(err) { + err && F.error(err); + self.$callback && self.$callback(); + }); + + if (self.buffer) { + self.buffer = null; + self.cache[0] = null; + self.cache[1] = null; + self.bytesread = 0; + } + + self.canceled = false; + self.fd = null; + self.type = null; + self.docscache = null; + self.docs = null; + + } else if (self.$callback) + self.$callback && self.$callback(); + + return self; +}; + +NoSQLStream.prototype.write = function(doc, position) { + var self = this; + self.bufferstack.push({ position: position, data: doc }); + !self.writing && self.$write(); + return self; +}; + +NoSQLStream.prototype.write2 = function(doc) { + var self = this; + self.bufferstacknew.push(U.createBuffer(doc)); + !self.writing && self.$write(); + return self; +}; + +NoSQLStream.prototype.$write = function() { + var self = this; + if (self.bufferstacknew.length && self.bufferstack.length) { + self.writing = true; + var buf = self.bufferstacknew.splice(0, 5); + buf = buf.length > 1 ? Buffer.concat(buf) : buf[0]; + Fs.write(self.fd, buf, 0, buf.length, self.positionappend, self.cb_writeAddUpdAdd); + } else if (self.bufferstacknew.length) { + self.writing = true; + var buf = self.bufferstacknew.splice(0, 5); + buf = buf.length > 1 ? Buffer.concat(buf) : buf[0]; + Fs.write(self.fd, buf, 0, buf.length, self.positionappend, self.cb_writeAdd); + } else if (self.bufferstack.length) { + self.writing = true; + var item = self.bufferstack.shift(); + Fs.write(self.fd, item.data, item.position, 'utf8', self.cb_writeUpd); + } +}; + +NoSQLStream.prototype.flush = function() { + var self = this; + if (self.writing) + setTimeout(self.cb_flush, 100); + else + self.close(); + return self; +}; + +NoSQLStream.prototype.read = function() { + + var self = this; + var size = self.stats.size - self.position; + + if (!self.fd || size <= 0 || self.canceled) { + + if (self.docscount) { + self.ondocuments(); + self.docscount = 0; + self.docs = ''; + } + + self.close(); + + } else { + size = size < BUFFERSIZE ? size : BUFFERSIZE; + var buffer = framework_utils.createBufferSize(size); + Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readbuffer); + } +}; + +NoSQLStream.prototype.readreverse = function() { + var self = this; + self.position = self.stats.size; + self.readreverse2(); + return self; +}; + +NoSQLStream.prototype.readreverse2 = function() { + var self = this; + if (!self.fd || self.position <= 0 || self.canceled) { + if (self.docscount) { + self.ondocuments(); + self.docs = ''; + self.docscount = 0; + } + self.close(); + } else { + var size = self.stats.size - self.bytesread; + size = size < BUFFERSIZE ? size : BUFFERSIZE; + self.position -= size; + var buffer = framework_utils.createBufferSize(size); + Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readreversebuffer); + } +}; + +NoSQLStream.prototype.readupdate = function() { + var self = this; + var size = self.stats.size - self.position; + if (!self.fd || size <= 0 || self.canceled) { + + if (self.docsbuffer.length) { + self.ondocuments(); + self.docsbuffer = []; + self.docs = ''; + } + + self.flush(); + } else { + size = size < BUFFERSIZE ? size : BUFFERSIZE; + var buffer = framework_utils.createBufferSize(size); + Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readwritebuffer); + } +}; + module.exports = NoSQLStream; \ No newline at end of file diff --git a/nosqlworker.js b/nosqlworker.js index 3e3f61b5f..c845d0a34 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -33,6 +33,7 @@ const RESUPDATE = { TYPE: 'update' }; const RESBACKUP = { TYPE: 'backup' }; const RESRESTORE = { TYPE: 'restore' }; const RESREMOVE = { TYPE: 'remove' }; +const RESCALLBACK = { TYPE: 'callback' }; const RESCOUNTERREAD = { TYPE: 'counter.read' }; const RESCOUNTERSTATS = { TYPE: 'counter.stats' }; const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; @@ -262,5 +263,13 @@ process.on('message', function(msg) { case 'indexes.noreindex': db.indexes.noreindex(); break; + case 'clean': + case 'clear': + db[msg.TYPE](function(err) { + RESCALLBACK.id = msg.id; + RESCALLBACK.err = err; + process.send(RESCALLBACK); + }); + break; } }); \ No newline at end of file From 516f77ee19d29cc767811564e290ee5276ba250f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Apr 2018 10:42:17 +0200 Subject: [PATCH 0216/1669] Added comments. --- nosqlstream.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nosqlstream.js b/nosqlstream.js index 8982920c9..c4f98cc74 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -49,6 +49,7 @@ function NoSQLStream(filename) { // this.indexer = 0; } +// Because of performance NoSQLStream.prototype.readhelpers = function() { var self = this; @@ -250,6 +251,7 @@ NoSQLStream.prototype.readhelpers = function() { }; +// Because of performance NoSQLStream.prototype.writehelpers = function() { var self = this; From 41b88706aeba766eb3881d22d69304c79255e032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Apr 2018 10:50:04 +0200 Subject: [PATCH 0217/1669] Improved NoSQL writer. --- nosqlstream.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nosqlstream.js b/nosqlstream.js index c4f98cc74..3e3e92009 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -352,16 +352,16 @@ NoSQLStream.prototype.writehelpers = function() { break; } - if (self.bufferstack.length || self.bufferstacknew.length) { - // @TODO: add helper - var fn = function() { - if (self.bufferstack.length || self.bufferstacknew.length) - setImmediate(fn); - else - self.readupdate(); - }; - setImmediate(fn); - } else + if (self.bufferstack.length || self.bufferstacknew.length) + setImmediate(self.cb_writeticks); + else + self.readupdate(); + }; + + self.cb_writeticks = function() { + if (self.bufferstack.length || self.bufferstacknew.length) + setImmediate(self.cb_writeticks); + else self.readupdate(); }; }; From 13d29905cd1c847b18e61105590b0fe153f6bef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Apr 2018 23:31:23 +0200 Subject: [PATCH 0218/1669] Fixed joins. --- nosql.js | 238 ++++++++++++++++++++++++++++--------------------- nosqlworker.js | 2 +- 2 files changed, 139 insertions(+), 101 deletions(-) diff --git a/nosql.js b/nosql.js index 8b53cced9..d859d56b9 100755 --- a/nosql.js +++ b/nosql.js @@ -235,12 +235,20 @@ exports.worker = function() { PRINTLN('ERROR --> NoSQL events are not supported in fork mode.'); }; - Database.prototype.find = function(view) { - return send(this, 'find', view).builder = new DatabaseBuilder(this); + Database.prototype.find = function(view, builder) { + if (builder) + builder.db = this; + else + builder = new DatabaseBuilder(this); + return send(this, 'find', view).builder = builder; }; - Database.prototype.find2 = function(view) { - return send(this, 'find2', view).builder = new DatabaseBuilder(this); + Database.prototype.find2 = function(builder) { + if (builder) + builder.db = this; + else + builder = new DatabaseBuilder(this); + return send(this, 'find2').builder = builder; }; Database.prototype.top = function(max, view) { @@ -913,9 +921,13 @@ Database.prototype.remove = function(filename) { return builder; }; -Database.prototype.find = function(view) { +Database.prototype.find = function(view, builder) { var self = this; - var builder = new DatabaseBuilder(self); + + if (builder) + builder.db = self; + else + builder = new DatabaseBuilder(self); if (view) { self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); @@ -928,13 +940,17 @@ Database.prototype.find = function(view) { return builder; }; -Database.prototype.find2 = function() { +Database.prototype.find2 = function(builder) { var self = this; + if (builder) + builder.db = self; + else + builder = new DatabaseBuilder(self); + if (self.readonly) - return self.find(); + return self.find(null, builder); - var builder = new DatabaseBuilder(self); self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); setImmediate(next_operation, self, 11); return builder; @@ -2693,7 +2709,6 @@ function DatabaseBuilder(db) { this.$scope = 0; // this.$fields; // this.$join; - // this.$joincount; this.$callback = NOOP; // this.$scalar; // this.$scalarfield; @@ -2728,6 +2743,69 @@ DatabaseBuilder.prototype.log = function(msg, user) { return self; }; +DatabaseBuilder.prototype.$callbackjoin = function(callback) { + var self = this; + + Object.keys(self.$join).wait(function(key, next) { + var join = self.$join[key]; + var response = self.$response; + var unique = []; + + for (var i = 0; i < response.length; i++) { + var item = response[i]; + var val = item[join.b]; + if (val !== undefined) + if (unique.indexOf(val) === -1) + unique.push(val); + } + + var db = NOSQL(join.name); + + if (join.scalartype) { + join.items = []; + join.count = unique.length; + for (var i = 0; i < unique.length; i++) { + (function(val) { + var builder = db.scalar(join.scalartype, join.scalarfield, join.view).callback(function(err, response) { + join.items.push({ id: val, response: response }); + join.count--; + if (join.count === 0) { + join.count = -1; + next(); + } + }); + + if (join.builder.$counter) { + builder.$counter = join.builder.$counter; + builder.$code = join.builder.$code.slice(0); + U.extend_headers2(builder.$options, join.builder.$options); + builder.$repository = join.builder.$repository; + builder.$params = CLONE(join.builder.$params); + } + + builder.$take = join.builder.$take; + builder.$skip = join.builder.$skip; + builder.$filter = join.builder.$filter; + builder.$scope = join.builder.$scope; + builder.where(join.a, val); + + })(unique[i]); + } + + } else { + join.builder.$options.fields && join.builder.$options.fields.push(join.a); + join.builder.$callback = function(err, docs) { + join.items = docs; + next(); + }; + db.find(join.view, join.builder).in(join.a, unique); + } + + }, callback); + + return self; +}; + DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository) { var self = this; @@ -2737,86 +2815,48 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository return self.$callback(err, response, count, repository); } - if (self.$joincount) { - setImmediate(() => self.$callback2(err, response, count, repository)); - return self; - } + self.$response = response; + self.$callbackjoin(function() { - var keys = Object.keys(self.$join); - var jl = keys.length; + var keys = Object.keys(self.$join); - if (response instanceof Array) { - for (var i = 0, length = response.length; i < length; i++) { - var item = response[i]; + var jl = keys.length; + if (response instanceof Array) { + for (var i = 0, length = response.length; i < length; i++) { + var item = response[i]; + for (var j = 0; j < jl; j++) { + var join = self.$join[keys[j]]; + item[join.field] = join.scalartype ? findScalar(join.items, item[join.b]) : join.first ? findItem(join.items, join.a, item[join.b]) : findItems(join.items, join.a, item[join.b]); + } + } + } else if (response) { for (var j = 0; j < jl; j++) { var join = self.$join[keys[j]]; - item[join.field] = join.scalar ? scalar(join.items, join.scalar, join.scalarfield, join.a, join.b != null ? item[join.b] : undefined) : join.first ? findItem(join.items, join.a, item[join.b], join.scalar, join.scalarfield) : findItems(join.items, join.a, item[join.b]); + response[join.field] = join.scalartype ? findScalar(join.items, item[join.b]) : join.first ? findItem(join.items, join.a, response[join.b]) : findItems(join.items, join.a, response[join.b]); } } - } else if (response) { - for (var j = 0; j < jl; j++) { - var join = self.$join[keys[j]]; - response[join.field] = join.scalar ? scalar(join.items, join.scalar, join.scalarfield, join.a, join.b != null ? response[join.b] : undefined) : join.first ? findItem(join.items, join.a, response[join.b], join.scalar, join.scalarfield) : findItems(join.items, join.a, response[join.b]); - } - } - self.$options.log && self.log(); - self.$callback(err, response, count, repository); - self.$done && setImmediate(self.$done); + self.$options.log && self.log(); + self.$callback(err, response, count, repository); + self.$done && setImmediate(self.$done); + }); + return self; }; -function scalar(items, type, field, where, value) { - - if (type === 'count' && !where) - return items.length; - - var val = type !== 'min' && type !== 'max' ? type === 'group' ? {} : 0 : null; - var count = 0; - +function findItem(items, field, value) { for (var i = 0, length = items.length; i < length; i++) { - var item = items[i]; - - if (where && item[where] !== value) - continue; - - switch (type) { - case 'count': - val++; - break; - case 'sum': - val += item[field] || 0; - break; - case 'avg': - val += item[field] || 0; - count++; - break; - case 'min': - val = val === null ? item[field] : (val > item[field] ? item[field] : val); - break; - case 'group': - if (val[item[field]]) - val[item[field]]++; - else - val[item[field]] = 1; - break; - case 'max': - val = val === null ? item[field] : (val < item[field] ? item[field] : val); - break; - } + if (items[i][field] === value) + return items[i]; } - - if (type === 'avg') - val = val / count; - - return val || 0; } -function findItem(items, field, value) { +function findScalar(items, value) { for (var i = 0, length = items.length; i < length; i++) { - if (items[i][field] === value) - return items[i]; + if (items[i].id === value) + return items[i].response || null; } + return null; } function findItems(items, field, value) { @@ -2831,24 +2871,25 @@ function findItems(items, field, value) { DatabaseBuilder.prototype.join = function(field, name, view) { var self = this; - if (!self.$join) { + if (!self.$join) self.$join = {}; - self.$joincount = 0; - } var key = name + '.' + (view || '') + '.' + field; var join = self.$join[key]; if (join) return join; - self.$join[key] = {}; - self.$join[key].field = field; - self.$join[key].pending = true; - self.$joincount++; - - join = NOSQL(name).find(view); + var item = self.$join[key] = {}; + item.field = field; + item.name = name; + item.view = view; + item.builder = join = new DatabaseBuilder(self.db); join.on = function(a, b) { + + if (self.$options.fields) + self.$options.fields.push(b); + self.$join[key].a = a; self.$join[key].b = b; return join; @@ -2856,32 +2897,26 @@ DatabaseBuilder.prototype.join = function(field, name, view) { join.$where = self.where; - join.where = function(a, b, c) { - return c === undefined && typeof(b) === 'string' ? join.on(a, b) : join.$where(a, b, c); + // join.where = function(a, b, c) { + // return c === undefined && typeof(b) === 'string' ? join.on(a, b) : join.$where(a, b, c); + // }; + + join.first = function() { + item.first = true; + return join; }; join.scalar = function(type, field) { - self.$join[key].scalar = type; - self.$join[key].scalarfield = field; + item.scalartype = type; + item.scalarfield = field; return join; }; - join.first = function() { - self.$join[key].first = true; + join.callback = function(a, b) { + self.callback(a, b); return join; }; - join.callback(function(err, docs) { - self.$join[key].pending = false; - self.$join[key].items = docs; - self.$joincount--; - }); - - setImmediate(function() { - join.$fields && join.fields(self.$join[key].b); - join.$fields && self.$join[key].scalarfield && join.fields(self.$join[key].scalarfield); - }); - return join; }; @@ -2909,6 +2944,7 @@ DatabaseBuilder.prototype.filter = function(fn) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$counter++; return self; }; @@ -2927,6 +2963,7 @@ DatabaseBuilder.prototype.contains = function(name) { code = 'if(!$is){' + code + '}'; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$counter++; return self; }; @@ -2937,6 +2974,7 @@ DatabaseBuilder.prototype.empty = function(name) { code = 'if(!$is){' + code + '}'; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$counter++; return self; }; diff --git a/nosqlworker.js b/nosqlworker.js index c845d0a34..7b3e294dc 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -71,7 +71,7 @@ process.on('message', function(msg) { }); break; case 'find2': - db.find().parse(msg.data).callback(function(err, response, count, repository) { + db.find2().parse(msg.data).callback(function(err, response, count, repository) { RESFIND.err = err; RESFIND.response = response; RESFIND.count = count; From 885411ba5f611d2e9be0f9700d1660486cad87dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Apr 2018 11:15:05 +0200 Subject: [PATCH 0219/1669] Removed useless comments. --- builders.js | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/builders.js b/builders.js index 72723609f..6d5248f44 100755 --- a/builders.js +++ b/builders.js @@ -1399,37 +1399,6 @@ SchemaBuilderEntity.prototype.validate = function(model, resourcePrefix, resourc path = ''; framework_utils.validate_builder.call(self, model, builder, self.name, self.parent.collection, self.name, index, filter, path); - - /* - if (!self.dependencies) - return builder; - - for (var i = 0, length = self.dependencies.length; i < length; i++) { - var key = self.dependencies[i]; - var schema = self.schema[key]; - var s = self.parent.collection[schema.raw]; - - if (!s) { - F.error(new Error('Schema "{0}" not found (validation).'.format(schema.raw))); - continue; - } - - - if (schema.isArray) { - var arr = model[key]; - for (var j = 0, jl = arr.length; j < jl; j++) { - if (model[key][j] != null || schema.required) { - if (!schema.can || schema.can(model)) - s.validate(model[key][j], resourcePrefix, resourceName, builder, filter, path + key + '[' + j + ']', j); - } - } - } else if (model[key] != null || schema.required) { - if (!schema.can || schema.can(model)) - s.validate(model[key], resourcePrefix, resourceName, builder, filter, path + key, -1); - } - } - */ - return builder; }; @@ -1452,7 +1421,7 @@ SchemaBuilderEntity.prototype.Create = function() { * @return {Object} */ SchemaBuilderEntity.prototype.$make = function(obj) { - return obj; // TODO remove + return obj; }; SchemaBuilderEntity.prototype.$prepare = function(obj, callback) { From f293fdc46c38100ddf02b8712312a541ce5dc641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Apr 2018 11:15:17 +0200 Subject: [PATCH 0220/1669] Improved `F.usage()` by adding `nosqlcleaner`. --- index.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index b9fa3e5d9..0a67afbf2 100755 --- a/index.js +++ b/index.js @@ -5372,6 +5372,7 @@ F.usage = function(detailed) { var staticRange = Object.keys(F.temporary.range); var redirects = Object.keys(F.routes.redirects); var output = {}; + var nosqlcleaner = Object.keys(F.databasescleaner); output.framework = { id: F.id, @@ -5417,7 +5418,8 @@ F.usage = function(detailed) { streaming: staticRange.length, modificator: F.modificators ? F.modificators.length : 0, viewphrases: $VIEWCACHE.length, - uptodates: F.uptodates ? F.uptodates.length : 0 + uptodates: F.uptodates ? F.uptodates.length : 0, + nosqlcleaner: nosqlcleaner.length }; output.routing = { @@ -5463,17 +5465,18 @@ F.usage = function(detailed) { output.models.push({ name: key, usage: item.usage ? item.usage() : null }); } - output.uptodates = F.uptodates; - output.helpers = helpers; output.cache = cache; - output.resources = resources; - output.errors = F.errors; - output.problems = F.problems; output.changes = F.changes; - output.traces = F.traces; + output.errors = F.errors; output.files = staticFiles; - output.streaming = staticRange; + output.helpers = helpers; + output.nosqlcleaner = nosqlcleaner; output.other = Object.keys(F.temporary.other); + output.problems = F.problems; + output.resources = resources; + output.streaming = staticRange; + output.traces = F.traces; + output.uptodates = F.uptodates; return output; }; From fcfdfc9afddf5561760776c852e5d65cd3857ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 08:58:54 +0200 Subject: [PATCH 0221/1669] Added `DatabaseBuilder.query()`. --- changes.txt | 1 + nosql.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/changes.txt b/changes.txt index 4dd69312a..d8340d1ae 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - added: a new route flag type `&group` something like `roles` but groups aren't evaluated - added: `route.groups` with defined groups - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode +- added: `DatabaseBuilder.query(code)` can contain a raw JS condition in the form e.g. `doc.age > 18 && doc.age < 33` - added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) - added: `Database.lock(callback(next))` locks all internal DB operations - added: new directory `schemas` with a new configuration item `directory-schemas' diff --git a/nosql.js b/nosql.js index d859d56b9..0b46b2ace 100755 --- a/nosql.js +++ b/nosql.js @@ -3024,6 +3024,20 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { return self; }; +DatabaseBuilder.prototype.query = function(code) { + var self = this; + + code = '$is=(' + code + ');'; + + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + self.$counter++; + return self; +}; + DatabaseBuilder.prototype.month = function(name, operator, value) { var self = this; var key = 'dm' + (self.$counter++); From f3ad342955399fa748179dc2d6110778372df5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 10:33:04 +0200 Subject: [PATCH 0222/1669] Improved `exports.parseJSON()`. --- utils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/utils.js b/utils.js index 4f1fae76c..5b19610ee 100755 --- a/utils.js +++ b/utils.js @@ -2264,7 +2264,6 @@ exports.parseJSON = function(value, date) { try { return JSON.parse(value, date ? jsonparser : undefined); } catch(e) { - return null; } }; From d6d3b21b2c8c7fc506ce284e34e285d436afaccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 10:33:12 +0200 Subject: [PATCH 0223/1669] Improved bundles. --- bundles.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bundles.js b/bundles.js index 5707550d1..cfb9e276c 100755 --- a/bundles.js +++ b/bundles.js @@ -159,10 +159,7 @@ function cleanFiles(callback) { blacklist[F.config['directory-private']] = 1; blacklist[F.config['directory-databases']] = 1; - var meta = {}; - try { - meta = U.parseJSON(Fs.readFileSync(Path.join(path, 'bundle.json')).toString('utf8'), true); - } catch (e) {} + var meta = U.parseJSON(Fs.readFileSync(Path.join(path, 'bundle.json')).toString('utf8'), true) || {}; if (meta.files && meta.files.length) { for (var i = 0, length = meta.files.length; i < length; i++) { From 681b4e58721a6a5887cb2627fe912082ed0bf62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 10:40:03 +0200 Subject: [PATCH 0224/1669] Improved middleware. --- changes.txt | 1 + index.js | 198 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 145 insertions(+), 54 deletions(-) diff --git a/changes.txt b/changes.txt index d8340d1ae..becaffea4 100755 --- a/changes.txt +++ b/changes.txt @@ -57,6 +57,7 @@ - updated: Chunker supports `compression`, default `true` - updated: Chunker supports `autoremove` processed files in `each()` or `read()` method, default `true` - updated: `String.parseConfig(def, [onError])` can handle errors better +- updated: `middleware`, now Total.js supports new declaration `F.middleware(function($) {})` - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index 0a67afbf2..eab89bf02 100755 --- a/index.js +++ b/index.js @@ -51,6 +51,7 @@ const REG_ROBOT = /search|agent|bot|crawler|spider/i; const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.]+\.(jpg|js|css|png|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; +const REG_NEWIMPL = /^(async\s)?function(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; const REG_RANGE = /bytes=/; const REG_EMPTY = /\s/g; const REG_ACCEPTCLEANER = /\s|\./g; @@ -910,6 +911,7 @@ F.prototypes = function(fn) { proto.HttpResponse = PROTORES; proto.Image = framework_image.Image.prototype; proto.Message = Mail.Message.prototype; + proto.MiddlewareOptions = MiddlewareOptions; proto.OperationOptions = framework_builders.OperationOptions.prototype; proto.Page = framework_builders.Page.prototype; proto.Pagination = framework_builders.Pagination.prototype; @@ -2840,7 +2842,7 @@ F.websocket = function(url, funcInitialize, flags, length) { F.initwebsocket = function() { if (F.routes.websockets.length && F.config['allow-websocket'] && F.server) { - F.server.on('upgrade', F._upgrade); + F.server.on('upgrade', F.$upgrade); F.initwebsocket = null; } }; @@ -3658,6 +3660,9 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.routes.middleware[name] = typeof(declaration) === 'function' ? declaration : eval(declaration); F._length_middleware = Object.keys(F.routes.middleware).length; + if (REG_NEWIMPL.test(F.routes.middleware[name].toString())) + F.routes.middleware[name].$newversion = true; + next && next(); callback && callback(null, name); @@ -7102,6 +7107,8 @@ F.listener = function(req, res) { }; function requestcontinue_middleware(req, res) { + if (req.$total_middleware) + req.$total_middleware = null; F.$requestcontinue(req, res, req.headers); } @@ -7444,7 +7451,7 @@ F.$cors = function(req, res, fn, arg) { * @param {Socket} socket * @param {Buffer} head */ -F._upgrade = function(req, socket, head) { +F.$upgrade = function(req, socket, head) { if ((req.headers.upgrade || '').toLowerCase() !== 'websocket') return; @@ -7481,6 +7488,8 @@ F._upgrade = function(req, socket, head) { }; function websocketcontinue_middleware(req) { + if (req.$total_middleware) + req.$total_middleware = null; F.$websocketcontinue(req, req.$wspath, req.headers); } @@ -7530,6 +7539,9 @@ F.$websocketcontinue_process = function(route, req, path) { var next = function() { + if (req.$total_middleware) + req.$total_middleware = null; + if (F.connections[id]) { socket.upgrade(F.connections[id]); return; @@ -7898,19 +7910,7 @@ F.decrypt = function(value, key, jsonConvert) { jsonConvert = true; var response = (value || '').decrypt(F.config.secret + '=' + key); - if (!response) - return null; - - if (jsonConvert) { - if (response.isJSON()) { - try { - return response.parseJSON(true); - } catch (ex) {} - } - return null; - } - - return response; + return response ? (jsonConvert ? (response.isJSON() ? response.parseJSON(true) : null) : response) : null; }; /** @@ -9386,9 +9386,9 @@ FrameworkCache.prototype.init = function() { clearInterval(self.interval); self.interval = setInterval(() => F.cache.recycle(), 1000 * 60); if (F.config['allow-cache-snapshot']) - self.load(() => self.loadPersist()); + self.load(() => self.loadpersistent()); else - self.loadPersist(); + self.loadpersistent(); return self; }; @@ -9428,7 +9428,7 @@ FrameworkCache.prototype.savePersist = function() { return this; }; -FrameworkCache.prototype.loadPersist = function(callback) { +FrameworkCache.prototype.loadpersistent = function(callback) { var self = this; Fs.readFile(F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'framework_cachepersist.jsoncache'), function(err, data) { if (!err) { @@ -9626,6 +9626,8 @@ function subscribe_timeout(req) { } function subscribe_timeout_middleware(req) { + if (req.$total_middleware) + req.$total_middleware = null; req.$total_execute2(); } @@ -10117,6 +10119,10 @@ Controller.prototype.middleware = function(names, options, callback) { options = EMPTYOBJECT; var self = this; + + if (self.req.$total_middleware) + self.req.$total_middleware = null; + async_middleware(0, self.req, self.res, names, () => callback && callback(), options, self); return self; }; @@ -14633,42 +14639,45 @@ function extend_request(PROTO) { for (var i = 0; i < F._length_files; i++) { var file = F.routes.files[i]; - try { + // try { - if (file.extensions && !file.extensions[req.extension]) - continue; + if (file.extensions && !file.extensions[req.extension]) + continue; - if (file.url) { - var skip = false; - var length = file.url.length; + if (file.url) { + var skip = false; + var length = file.url.length; - if (!file.wildcard && !file.fixedfile && length !== req.path.length - 1) - continue; - - for (var j = 0; j < length; j++) { - if (file.url[j] === req.path[j]) - continue; - skip = true; - break; - } + if (!file.wildcard && !file.fixedfile && length !== req.path.length - 1) + continue; - if (skip) + for (var j = 0; j < length; j++) { + if (file.url[j] === req.path[j]) continue; + skip = true; + break; + } - } else if (file.onValidate && !file.onValidate.call(F, req, res, true)) + if (skip) continue; - if (file.middleware) - req.$total_endfilemiddleware(file); - else - file.execute.call(F, req, res, false); - return; + } else if (file.onValidate && !file.onValidate(req, res, true)) + continue; + + if (file.middleware) + req.$total_endfilemiddleware(file); + else + file.execute(req, res, false); + return; + + /* } catch (err) { F.error(err, file.controller, req.uri); res.throw500(); return; } + */ } res.continue(); @@ -14723,8 +14732,12 @@ function extend_request(PROTO) { } function total_endmiddleware(req) { + + if (req.total_middleware) + req.total_middleware = null; + try { - req.$total_filemiddleware.execute.call(F, req, req.res, false); + req.$total_filemiddleware.execute(req, req.res, false); } catch (err) { F.error(err, req.$total_filemiddleware.controller + ' :: ' + req.$total_filemiddleware.name, req.uri); req.res.throw500(); @@ -16064,6 +16077,57 @@ function createTemporaryKey(req) { return (req.uri ? req.uri.pathname : req).replace(REG_TEMPORARY, '-').substring(1); } +function MiddlewareOptions() {} + +MiddlewareOptions.prototype = { + + get user() { + return this.req.user; + }, + + get session() { + return this.req.session; + }, + + get language() { + return this.req.$language; + }, + + get ip() { + return this.req.ip; + }, + + get id() { + return this.controller ? this.controller.id : null; + }, + + get params() { + return this.controller ? this.controller.params : null; + }, + + get files() { + return this.req.files; + }, + + get body() { + return this.req.body; + }, + + get query() { + return this.req.query; + } +}; + +MiddlewareOptions.prototype.callback = MiddlewareOptions.prototype.resume = function() { + this.next(); + return this; +}; + +MiddlewareOptions.prototype.cancel = function() { + this.next(false); + return this; +}; + process.on('SIGTERM', () => F.stop()); process.on('SIGINT', () => F.stop()); process.on('exit', () => F.stop()); @@ -16163,22 +16227,48 @@ function async_middleware(index, req, res, middleware, callback, options, contro return async_middleware(index, req, res, middleware, callback, options, controller); } - var output = item.call(framework, req, res, function(err) { - - if (err === false) { - req.$total_route && req.$total_success(); - callback = null; - return; + var output; + + if (item.$newversion) { + var opt = req.$total_middleware; + if (!index || !opt) { + opt = req.$total_middleware = new MiddlewareOptions(); + opt.req = req; + opt.res = res; + opt.middleware = middleware; + opt.options = options; + opt.controller = controller; + opt.callback2 = callback; + opt.next = function(err) { + var mid = req.$total_middleware; + if (err === false) { + req.$total_route && req.$total_success(); + req.$total_middleware = null; + callback = null; + } else if (err instanceof Error || err instanceof ErrorBuilder) { + res.throw500(err); + req.$total_middleware = null; + callback = null; + } else + async_middleware(mid.index, mid.req, mid.res, mid.middleware, mid.callback2, mid.options, mid.controller); + }; } - if (err instanceof Error || err instanceof ErrorBuilder) { - res.throw500(err); - callback = null; - return; - } + opt.index = index; + output = item(opt); - async_middleware(index, req, res, middleware, callback, options, controller); - }, options, controller); + } else { + output = item.call(framework, req, res, function(err) { + if (err === false) { + req.$total_route && req.$total_success(); + callback = null; + } else if (err instanceof Error || err instanceof ErrorBuilder) { + res.throw500(err); + callback = null; + } else + async_middleware(index, req, res, middleware, callback, options, controller); + }, options, controller); + } if (output !== false) return; From f2d22334469adcab5ad192e8a1ae95065427ff9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 10:40:26 +0200 Subject: [PATCH 0225/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 915243978..c774ad484 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-41", + "version": "3.0.0-42", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From f283e8549c68c80d084409463eef43bd8685b70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 11:03:12 +0200 Subject: [PATCH 0226/1669] Updated code. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index eab89bf02..789d8932b 100755 --- a/index.js +++ b/index.js @@ -16118,7 +16118,7 @@ MiddlewareOptions.prototype = { } }; -MiddlewareOptions.prototype.callback = MiddlewareOptions.prototype.resume = function() { +MiddlewareOptions.prototype.callback = function() { this.next(); return this; }; From d582c58e6970e1964d3d05a837f4942b42aa22fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Apr 2018 20:57:09 +0200 Subject: [PATCH 0227/1669] Added `Number.round()`. --- changes.txt | 1 + utils.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index becaffea4..512ac0d37 100755 --- a/changes.txt +++ b/changes.txt @@ -43,6 +43,7 @@ - added: `$UPDATE(schema, model, [options], [callback], [controller])` performs `schema.update()` - added: `$REMOVE(schema, [options], [callback], [controller])` performs `schema.remove()` - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` +- added: `Number.round([precision])` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/utils.js b/utils.js index 5b19610ee..15c8b1282 100755 --- a/utils.js +++ b/utils.js @@ -3805,6 +3805,11 @@ Number.prototype.padRight = function(max, c) { return this.toString().padRight(max, c || '0'); }; +Number.prototype.round = function(precision) { + var m = Math.pow(10, precision) || 1; + return Math.round(this * m) / m; +}; + /** * Async decrements * @param {Function(index, next)} fn From 2e330476539b817f95a3e9ac4acf2268fabcfe15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 18 Apr 2018 07:48:19 +0200 Subject: [PATCH 0228/1669] Added cleaning of DB. --- nosqlworker.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nosqlworker.js b/nosqlworker.js index 7b3e294dc..28b0acfcf 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -48,6 +48,14 @@ function killprocess() { process.exit(0); } +// One day cleaner +setInterval(function() { + var keys = Object.keys(F.databasescleaner); + keys.length && keys.wait(function(item, next) { + NOSQL(item).clean(next); + }); +}, 60000 * 60 * 24); + process.on('disconnect', killprocess); process.on('close', killprocess); process.on('exit', killprocess); From 04c8caad21355d8d349aa2fca9e5539e425daab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 18 Apr 2018 09:38:03 +0200 Subject: [PATCH 0229/1669] Added `soundex` index. --- nosql.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index 0b46b2ace..2097d154a 100755 --- a/nosql.js +++ b/nosql.js @@ -5149,12 +5149,19 @@ Indexes.prototype.$index = function(index, value) { val = val.toLowerCase().removeDiacritics().match(REGINDEXCHAR); if (val) { val = val.toString(); - if (index.type === 'first') - val = val[0]; - else if (index.type === 'reverse') - val = val.substring(val.length - 2); - else - val = val.substring(0, 2); + switch (index.type) { + case 'first': + val = val[0]; + break; + case 'reverse': + val = val.substring(val.length - 2); + break; + case 'soundex': + val = val.soundex(); + break; + default: + val = val.substring(0, 2); + } } } } From 9bb4389ceb68bf8811fd62215a7cc6583a3f78fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 18 Apr 2018 10:20:58 +0200 Subject: [PATCH 0230/1669] Improved code. --- nosql.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nosql.js b/nosql.js index 2097d154a..652be3b5c 100755 --- a/nosql.js +++ b/nosql.js @@ -2895,12 +2895,6 @@ DatabaseBuilder.prototype.join = function(field, name, view) { return join; }; - join.$where = self.where; - - // join.where = function(a, b, c) { - // return c === undefined && typeof(b) === 'string' ? join.on(a, b) : join.$where(a, b, c); - // }; - join.first = function() { item.first = true; return join; From 118331b0ca340123b42ec9aa126624dd0ce27a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 18 Apr 2018 16:35:07 +0200 Subject: [PATCH 0231/1669] Removed useless code. --- utils.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils.js b/utils.js index 15c8b1282..4cd72f14c 100755 --- a/utils.js +++ b/utils.js @@ -2550,8 +2550,6 @@ Date.prototype.extend = function(date) { var dt = new Date(this); var match = date.match(regexpDATE); - console.log(match); - if (!match) return dt; From df327160b150b570b6fae228f51863d5a6c8e01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Apr 2018 09:20:55 +0200 Subject: [PATCH 0232/1669] Improved `UID()`. --- changes.txt | 1 + index.js | 31 ++++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 512ac0d37..dcfb5d86e 100755 --- a/changes.txt +++ b/changes.txt @@ -44,6 +44,7 @@ - added: `$REMOVE(schema, [options], [callback], [controller])` performs `schema.remove()` - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` - added: `Number.round([precision])` +- added: `UID([type])` supports custom types, e.g. `UID('users')` or `UID('orders')` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index 789d8932b..e7cf7e671 100755 --- a/index.js +++ b/index.js @@ -307,7 +307,24 @@ global.ROUTING = (name) => F.routing(name); global.SCHEDULE = (date, each, fn, param) => F.schedule(date, each, fn, param); global.FINISHED = framework_internal.onFinished; global.DESTROY = framework_internal.destroyStream; -global.UID = () => UIDGENERATOR.date + (++UIDGENERATOR.index).padLeft(4, '0') + UIDGENERATOR.instance + (UIDGENERATOR.index % 2 ? 1 : 0); + +global.UID = function(type) { + + var index; + + if (type) { + if (UIDGENERATOR.types[type]) + index = UIDGENERATOR.types[type] = UIDGENERATOR.types[type] + 1; + else { + UIDGENERATOR.multiple = true; + index = UIDGENERATOR.types[type] = 1; + } + } else + index = UIDGENERATOR.index++; + + return UIDGENERATOR.date + index.padLeft(4, '0') + UIDGENERATOR.instance + (index % 2 ? 1 : 0); +}; + global.ROUTE = (a, b, c, d, e) => F.route(a, b, c, d, e); global.WEBSOCKET = (a, b, c, d) => F.websocket(a, b, c, d); global.FILE = (a, b, c) => F.file(a, b, c); @@ -544,7 +561,7 @@ var DATE_EXPIRES = new Date().add('y', 1).toUTCString(); const WEBSOCKET_COMPRESS = U.createBuffer([0x00, 0x00, 0xFF, 0xFF]); const WEBSOCKET_COMPRESS_OPTIONS = { windowBits: Zlib.Z_DEFAULT_WINDOWBITS }; -const UIDGENERATOR = { date: new Date().format('yyMMddHHmm'), instance: 'abcdefghijklmnoprstuwxy'.split('').random().join('').substring(0, 3), index: 1 }; +const UIDGENERATOR = { date: new Date().format('yyMMddHHmm'), instance: 'abcdefghijklmnoprstuwxy'.split('').random().join('').substring(0, 3), index: 1, types: {} }; const EMPTYBUFFER = U.createBufferSize(0); global.EMPTYBUFFER = EMPTYBUFFER; @@ -6892,6 +6909,14 @@ F.service = function(count) { UIDGENERATOR.date = F.datetime.format('yyMMddHHmm'); UIDGENERATOR.index = 1; + var keys; + + if (UIDGENERATOR.multiple) { + keys = Object.keys(UIDGENERATOR.types); + for (var i = 0; i < keys.length; i++) + UIDGENERATOR.types[keys[i]] = 0; + } + var releasegc = false; // clears temporary memory for non-exist files @@ -7003,7 +7028,7 @@ F.service = function(count) { count % 1000 === 0 && (DATE_EXPIRES = F.datetime.add('y', 1).toUTCString()); if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { - var keys = Object.keys(F.databasescleaner); + keys = Object.keys(F.databasescleaner); keys.wait(function(item, next) { NOSQL(item).clean(next); }); From 4d3610942ec93a04d0610c338b9d28aa576bd55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Apr 2018 09:21:06 +0200 Subject: [PATCH 0233/1669] Improved NoSQL updates. --- nosql.js | 21 +++++++++++++++++++-- nosqlstream.js | 8 +++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index 652be3b5c..3bbd666f9 100755 --- a/nosql.js +++ b/nosql.js @@ -1319,6 +1319,7 @@ Database.prototype.$update = function() { var filter = self.pending_update.splice(0); var length = filter.length; var backup = false; + var filters = 0; for (var i = 0; i < length; i++) { var fil = filter[i]; @@ -1336,20 +1337,30 @@ Database.prototype.$update = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); for (var a = 0; a < docs.length; a++) { - var doc = docs[a]; + indexer++; + var doc = docs[a]; var is = false; var copy = self.indexes && self.indexes.indexes.length ? CLONE(doc) : null; var rec = fs.docsbuffer[a]; for (var i = 0; i < length; i++) { + var item = filter[i]; + if (item.skip) + continue; item.filter.index = indexer; - var output = item.compare(doc, item.filter, indexer); + var output = item.compare(doc, item.filter, indexer); if (output) { + + if (item.filter.options.first) { + item.skip = true; + filters++; + } + if (item.keys) { for (var j = 0; j < item.keys.length; j++) { var key = item.keys[j]; @@ -1387,6 +1398,9 @@ Database.prototype.$update = function() { } var upd = JSON.stringify(doc); + if (upd === rec.doc) + continue; + var was = true; if (rec.doc.length === upd.length) { @@ -1403,6 +1417,9 @@ Database.prototype.$update = function() { fs.write2(upd + NEWLINE); } } + + if (filters === length) + return false; } }; diff --git a/nosqlstream.js b/nosqlstream.js index 3e3e92009..d0d542b40 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -339,9 +339,15 @@ NoSQLStream.prototype.writehelpers = function() { self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); self.docscount++; if (self.docsbuffer.length >= BUFFERDOCS) { - self.ondocuments(); + + if (self.ondocuments() === false) + self.canceled = true; + self.docsbuffer = []; self.docs = ''; + + if (self.canceled) + break; } } From 607d0a7e42915d20786769bc1f6946cb4ae430c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Apr 2018 09:27:51 +0200 Subject: [PATCH 0234/1669] Added new alias. --- changes.txt | 1 + utils.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index dcfb5d86e..9e881c9ec 100755 --- a/changes.txt +++ b/changes.txt @@ -45,6 +45,7 @@ - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` - added: `Number.round([precision])` - added: `UID([type])` supports custom types, e.g. `UID('users')` or `UID('orders')` +- added: `REQUEST()` alias to `U.request()` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/utils.js b/utils.js index 4cd72f14c..ac50bf549 100755 --- a/utils.js +++ b/utils.js @@ -423,7 +423,7 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng * @param {Number} timeout Request timeout. * return {Boolean} */ -exports.request = function(url, flags, data, callback, cookies, headers, encoding, timeout, files) { +global.REQUEST = exports.request = function(url, flags, data, callback, cookies, headers, encoding, timeout, files) { // No data (data is optional argument) if (typeof(data) === 'function') { From 3bd4dc185414c0c1014d4f4c1f746ebcec4caf46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Apr 2018 09:36:49 +0200 Subject: [PATCH 0235/1669] Added `NOW` global alias to `F.datetime`. --- builders.js | 4 +- changes.txt | 3 +- index.js | 126 ++++++++++++++++++++++++++++------------------------ mail.js | 2 +- nosql.js | 46 +++++++++---------- utils.js | 2 +- 6 files changed, 96 insertions(+), 87 deletions(-) diff --git a/builders.js b/builders.js index 6d5248f44..9fba9a3e9 100755 --- a/builders.js +++ b/builders.js @@ -1479,7 +1479,7 @@ SchemaBuilderEntity.prototype.default = function() { break; // date case 5: - item[property] = type.isArray ? [] : F.datetime; + item[property] = type.isArray ? [] : NOW; break; // schema case 7: @@ -4227,7 +4227,7 @@ RESTBuilder.prototype.exec = function(callback) { output.headers = headers; output.hostname = hostname; output.cache = false; - output.datetime = F.datetime; + output.datetime = NOW; if (self.$schema) { diff --git a/changes.txt b/changes.txt index 9e881c9ec..14b74d534 100755 --- a/changes.txt +++ b/changes.txt @@ -45,7 +45,8 @@ - added: `U.streamer2()` same functionality as `U.streamer()` but it returns `Buffer` instead of `String` - added: `Number.round([precision])` - added: `UID([type])` supports custom types, e.g. `UID('users')` or `UID('orders')` -- added: `REQUEST()` alias to `U.request()` +- added: `REQUEST()` global method, it's alias to `U.request()` +- added: `NOW` global property, it's alias to `F.datetime` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index e7cf7e671..df066e2a7 100755 --- a/index.js +++ b/index.js @@ -97,6 +97,8 @@ Object.freeze(EMPTYREQUEST); global.EMPTYOBJECT = EMPTYOBJECT; global.EMPTYARRAY = EMPTYARRAY; +global.NOW = new Date(); + var PROTORES, PROTOREQ; var RANGE = { start: 0, end: 0 }; @@ -543,7 +545,7 @@ global.TRY = function(fn, err) { }; global.OBSOLETE = function(name, message) { - console.log(F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' :: OBSOLETE / IMPORTANT ---> "' + name + '"', message); + console.log(NOW.format('yyyy-MM-dd HH:mm:ss') + ' :: OBSOLETE / IMPORTANT ---> "' + name + '"', message); if (global.F) F.stats.other.obsolete++; }; @@ -772,7 +774,6 @@ function Framework() { this.directory = HEADERS.workers.cwd = directory; this.isLE = Os.endianness ? Os.endianness() === 'LE' : true; this.isHTTPS = false; - this.datetime = new Date(); // It's hidden // this.waits = {}; @@ -885,6 +886,12 @@ function Framework() { // ====================================================== Framework.prototype = { + get datetime() { + return global.NOW; + }, + set datetime(val) { + global.NOW = val; + }, get cluster() { return require('./cluster'); }, @@ -1348,7 +1355,7 @@ F.schedule = function(date, repeat, fn) { if (type === 'string') { date = date.parseDate(); - repeat && date < F.datetime && (date = F.datetime.add(repeat)); + repeat && date < NOW && (date = NOW.add(repeat)); } else if (type === 'number') date = new Date(date); @@ -3120,8 +3127,8 @@ F.error = function(err, name, uri) { return F; if (F.errors) { - F.datetime = new Date(); - F.errors.push({ error: err.stack, name: name, url: uri ? typeof(uri) === 'string' ? uri : Parser.format(uri) : undefined, date: F.datetime }); + NOW = new Date(); + F.errors.push({ error: err.stack, name: name, url: uri ? typeof(uri) === 'string' ? uri : Parser.format(uri) : undefined, date: NOW }); F.errors.length > 50 && F.errors.shift(); } @@ -3211,11 +3218,11 @@ F.trace = function(message, name, uri, ip) { else if (typeof(message) === 'object') message = JSON.stringify(message); - F.datetime = new Date(); - var obj = { message: message, name: name, url: uri ? typeof(uri) === 'string' ? uri : Parser.format(uri) : undefined, ip: ip, date: F.datetime }; + NOW = new Date(); + var obj = { message: message, name: name, url: uri ? typeof(uri) === 'string' ? uri : Parser.format(uri) : undefined, ip: ip, date: NOW }; F.logger('traces', obj.message, 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); - F.config['trace-console'] && console.log(F.datetime.format('yyyy-MM-dd HH:mm:ss'), '[trace]', message, '|', 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); + F.config['trace-console'] && console.log(NOW.format('yyyy-MM-dd HH:mm:ss'), '[trace]', message, '|', 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); if (F.traces) { F.traces.push(obj); @@ -3519,7 +3526,7 @@ F.uptodate = function(type, url, options, interval, callback, next) { options = null; } - var obj = { type: type, name: '', url: url, interval: interval, options: options, count: 0, updated: F.datetime, errors: [], callback: callback }; + var obj = { type: type, name: '', url: url, interval: interval, options: options, count: 0, updated: NOW, errors: [], callback: callback }; if (!F.uptodates) F.uptodates = []; @@ -3573,7 +3580,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe var content; var err; - F.datetime = new Date(); + NOW = new Date(); if (t === 'object') { t = typeof(options); @@ -3686,9 +3693,9 @@ F.install = function(type, name, declaration, options, callback, internal, useRe key = type + '.' + name; if (F.dependencies[key]) { - F.dependencies[key].updated = F.datetime; + F.dependencies[key].updated = NOW; } else { - F.dependencies[key] = { name: name, type: type, installed: F.datetime, updated: null, count: 0 }; + F.dependencies[key] = { name: name, type: type, installed: NOW, updated: null, count: 0 }; if (internal) F.dependencies[key].url = internal; } @@ -3698,7 +3705,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -3711,7 +3718,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe delete F.temporary['mail-settings']; F.emit(type + '#' + name, F.config); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -3726,7 +3733,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -3741,7 +3748,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; F.consoledebug('install', type + '#' + name); }, 500); @@ -3756,7 +3763,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; F.consoledebug('install', type + '#' + name); }, 500); @@ -3794,7 +3801,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.components.has = true; var link = F.config['static-url-components']; - F.components.version = F.datetime.getTime(); + F.components.version = NOW.getTime(); F.components.links = (F.components.js ? ''.format(link, F.components.version) : '') + (F.components.css ? ''.format(link, F.components.version) : ''); if (content.install) { @@ -3853,14 +3860,14 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp.css = true; } - tmp.version = F.datetime.getTime(); + tmp.version = NOW.getTime(); tmp.links = (tmp.js ? ''.format(link, tmp.version, key) : '') + (tmp.css ? ''.format(link, tmp.version, key) : ''); } !skipEmit && setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -3890,8 +3897,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.emit(type + '#' + name); F.emit('install', 'module', name); F.emit('install', type, name); - F.temporary.ready['package#' + name] = F.datetime; - F.temporary.ready['module#' + name] = F.datetime; + F.temporary.ready['package#' + name] = NOW; + F.temporary.ready['module#' + name] = NOW; }, 500); F.consoledebug('install', 'package#' + name); callback && callback(err, name); @@ -3914,7 +3921,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe !skipEmit && setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -3940,8 +3947,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.emit(type + '#' + name); F.emit('install', type, name); F.emit('install', 'module', name); - F.temporary.ready['package#' + name] = F.datetime; - F.temporary.ready['module#' + name] = F.datetime; + F.temporary.ready['package#' + name] = NOW; + F.temporary.ready['module#' + name] = NOW; }, 500); F.consoledebug('install', 'package#' + name); callback && callback(err, name); @@ -3970,7 +3977,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -4020,7 +4027,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name); F.emit('install', type, name); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); return F; @@ -4089,7 +4096,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe setTimeout(function() { F.emit(type + '#' + name, obj); F.emit('install', type, name, obj); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); return F; @@ -4166,10 +4173,10 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (tmp) { F.dependencies[key] = tmp; - F.dependencies[key].updated = F.datetime; + F.dependencies[key].updated = NOW; } else { - F.dependencies[key] = { name: name, type: type, installed: F.datetime, updated: null, count: 0 }; + F.dependencies[key] = { name: name, type: type, installed: NOW, updated: null, count: 0 }; if (internal) F.dependencies[key].url = internal; } @@ -4191,7 +4198,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe !skipEmit && setTimeout(function() { F.emit(type + '#' + name, obj); F.emit('install', type, name, obj); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); F.consoledebug('install', type + '#' + name); @@ -4300,10 +4307,10 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (tmp) { F.dependencies[key] = tmp; - F.dependencies[key].updated = F.datetime; + F.dependencies[key].updated = NOW; } else { - F.dependencies[key] = { name: name, type: type, installed: F.datetime, updated: null, count: 0, _id: _ID }; + F.dependencies[key] = { name: name, type: type, installed: NOW, updated: null, count: 0, _id: _ID }; if (internal) F.dependencies[key].url = internal; } @@ -4587,7 +4594,7 @@ F.install_make = function(key, name, obj, options, callback, skipEmit, type) { setTimeout(function() { F.emit(type + '#' + name, obj); F.emit('install', type, name, obj); - F.temporary.ready[type + '#' + name] = F.datetime; + F.temporary.ready[type + '#' + name] = NOW; }, 500); } @@ -4844,12 +4851,12 @@ F.uninstall = function(type, name, options, skipEmit, packageName) { } } - tmp.version = F.datetime.getTime(); + tmp.version = NOW.getTime(); } } if (is) - F.components.version = F.datetime.getTime(); + F.components.version = NOW.getTime(); F.consoledebug('uninstall', type + '#' + name); } @@ -4981,8 +4988,8 @@ F.eval = function(script) { * @return {Framework} */ F.onError = function(err, name, uri) { - F.datetime = new Date(); - console.log('======= ' + (F.datetime.format('yyyy-MM-dd HH:mm:ss')) + ': ' + (name ? name + ' ---> ' : '') + err.toString() + (uri ? ' (' + Parser.format(uri) + ')' : ''), err.stack); + NOW = new Date(); + console.log('======= ' + (NOW.format('yyyy-MM-dd HH:mm:ss')) + ': ' + (name ? name + ' ---> ' : '') + err.toString() + (uri ? ' (' + Parser.format(uri) + ')' : ''), err.stack); return F; }; @@ -5315,9 +5322,9 @@ F.onMeta = function() { // @arguments {Object params} global.LOG = F.log = function() { - F.datetime = new Date(); - var filename = F.datetime.getFullYear() + '-' + (F.datetime.getMonth() + 1).toString().padLeft(2, '0') + '-' + F.datetime.getDate().toString().padLeft(2, '0'); - var time = F.datetime.getHours().toString().padLeft(2, '0') + ':' + F.datetime.getMinutes().toString().padLeft(2, '0') + ':' + F.datetime.getSeconds().toString().padLeft(2, '0'); + NOW = new Date(); + var filename = NOW.getFullYear() + '-' + (NOW.getMonth() + 1).toString().padLeft(2, '0') + '-' + NOW.getDate().toString().padLeft(2, '0'); + var time = NOW.getHours().toString().padLeft(2, '0') + ':' + NOW.getMinutes().toString().padLeft(2, '0') + ':' + NOW.getSeconds().toString().padLeft(2, '0'); var str = ''; var length = arguments.length; @@ -5338,8 +5345,8 @@ global.LOG = F.log = function() { }; global.LOGGER = F.logger = function() { - F.datetime = new Date(); - var dt = F.datetime.getFullYear() + '-' + (F.datetime.getMonth() + 1).toString().padLeft(2, '0') + '-' + F.datetime.getDate().toString().padLeft(2, '0') + ' ' + F.datetime.getHours().toString().padLeft(2, '0') + ':' + F.datetime.getMinutes().toString().padLeft(2, '0') + ':' + F.datetime.getSeconds().toString().padLeft(2, '0'); + NOW = new Date(); + var dt = NOW.getFullYear() + '-' + (NOW.getMonth() + 1).toString().padLeft(2, '0') + '-' + NOW.getDate().toString().padLeft(2, '0') + ' ' + NOW.getHours().toString().padLeft(2, '0') + ':' + NOW.getMinutes().toString().padLeft(2, '0') + ':' + NOW.getSeconds().toString().padLeft(2, '0'); var str = ''; var length = arguments.length; @@ -5398,7 +5405,7 @@ F.usage = function(detailed) { output.framework = { id: F.id, - datetime: F.datetime, + datetime: NOW, pid: process.pid, node: process.version, version: 'v' + F.version_header, @@ -6861,7 +6868,7 @@ F.console = function() { console.log('Name : ' + F.config.name); console.log('Version : ' + F.config.version); console.log('Author : ' + F.config.author); - console.log('Date : ' + F.datetime.format('yyyy-MM-dd HH:mm:ss')); + console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); console.log('Mode : ' + (F.config.debug ? 'debug' : 'release')); console.log('====================================================\n'); console.log('{2}://{0}:{1}/'.format(F.ip, F.port, F.isHTTPS ? 'https' : 'http')); @@ -6906,7 +6913,7 @@ F.reconnect = function() { */ F.service = function(count) { - UIDGENERATOR.date = F.datetime.format('yyMMddHHmm'); + UIDGENERATOR.date = NOW.format('yyMMddHHmm'); UIDGENERATOR.index = 1; var keys; @@ -6934,7 +6941,7 @@ F.service = function(count) { // Clears command cache Image.clear(); - var dt = F.datetime.add('-5 minutes'); + var dt = NOW.add('-5 minutes'); for (var key in F.databases) F.databases[key] && F.databases[key].inmemorylastusage < dt && F.databases[key].release(); @@ -6974,10 +6981,10 @@ F.service = function(count) { var hasUpdate = false; F.uptodates.wait(function(item, next) { - if (item.updated.add(item.interval) > F.datetime) + if (item.updated.add(item.interval) > NOW) return next(); - item.updated = F.datetime; + item.updated = NOW; item.count++; setTimeout(function() { @@ -7025,7 +7032,7 @@ F.service = function(count) { } // Update expires date - count % 1000 === 0 && (DATE_EXPIRES = F.datetime.add('y', 1).toUTCString()); + count % 1000 === 0 && (DATE_EXPIRES = NOW.add('y', 1).toUTCString()); if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { keys = Object.keys(F.databasescleaner); @@ -7050,7 +7057,7 @@ F.service = function(count) { if (!F.schedules.length) return F; - var expire = F.datetime.getTime(); + var expire = NOW.getTime(); var index = 0; while (true) { @@ -7063,7 +7070,7 @@ F.service = function(count) { index--; if (schedule.repeat) - schedule.expire = F.datetime.add(schedule.repeat); + schedule.expire = NOW.add(schedule.repeat); else F.schedules.splice(index, 1); @@ -9463,7 +9470,7 @@ FrameworkCache.prototype.loadpersistent = function(callback) { for (var i = 0, length = keys.length; i < length; i++) { var key = keys[i]; var item = data[key]; - if (item.expire >= F.datetime) + if (item.expire >= NOW) self.items[key] = item; } } catch (e) {} @@ -9489,7 +9496,8 @@ FrameworkCache.prototype.recycle = function() { var items = this.items; var isPersist = false; - F.datetime = new Date(); + + NOW = new Date(); this.count++; @@ -9497,7 +9505,7 @@ FrameworkCache.prototype.recycle = function() { var value = items[o]; if (!value) delete items[o]; - else if (value.expire < F.datetime) { + else if (value.expire < NOW) { if (value.persist) isPersist = true; F.emit('cache-expire', o, value.value); @@ -9530,7 +9538,7 @@ FrameworkCache.prototype.set = FrameworkCache.prototype.add = function(name, val expire = expire.parseDateExpiration(); break; case 'undefined': - expire = F.datetime.add('m', 5); + expire = NOW.add('m', 5); break; } @@ -9552,9 +9560,9 @@ FrameworkCache.prototype.read = FrameworkCache.prototype.get = function(key, def if (!value) return def; - F.datetime = new Date(); + NOW = new Date(); - if (value.expire < F.datetime) { + if (value.expire < NOW) { this.items[key] = undefined; F.$events['cache-expire'] && F.emit('cache-expire', key, value.value); return def; @@ -9569,7 +9577,7 @@ FrameworkCache.prototype.read2 = FrameworkCache.prototype.get2 = function(key, d if (!value) return def; - if (value.expire < F.datetime) { + if (value.expire < NOW) { this.items[key] = undefined; F.$events['cache-expire'] && F.emit('cache-expire', key, value.value); return def; diff --git a/mail.js b/mail.js index f318eb311..b08c936da 100755 --- a/mail.js +++ b/mail.js @@ -483,7 +483,7 @@ Mailer.prototype.send = function(smtp, options, messages, callback) { obj.count = 0; obj.socket; obj.tls = false; - obj.date = global.F ? global.F.datetime : new Date(); + obj.date = global.NOW ? global.NOW : new Date(); smtp = smtp || null; diff --git a/nosql.js b/nosql.js index 3bbd666f9..1dcf3c56b 100755 --- a/nosql.js +++ b/nosql.js @@ -1185,7 +1185,7 @@ Database.prototype.$inmemory = function(view, callback) { self.readonly && self.throwReadonly(); // Last usage - self.inmemorylastusage = global.F ? global.F.datetime : undefined; + self.inmemorylastusage = global.F ? global.NOW : undefined; if (self.inmemory[view]) return callback(); @@ -2694,8 +2694,8 @@ function DatabaseBuilder2(db) { DatabaseBuilder2.prototype.log = function(msg, user) { var self = this; if (msg) { - F.datetime = new Date(); - self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + NOW = new Date(); + self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; @@ -2751,8 +2751,8 @@ DatabaseBuilder.prototype.insert = function(fn) { DatabaseBuilder.prototype.log = function(msg, user) { var self = this; if (msg) { - F.datetime = new Date(); - self.$options.log = (self.$options.log ? self.$options.log : '') + F.datetime.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + NOW = new Date(); + self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; @@ -2998,7 +2998,7 @@ DatabaseBuilder.prototype.backup = function(user) { }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, F.datetime.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, NOW.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); return this; }; @@ -3467,7 +3467,7 @@ Counter.prototype.min = function(id, count) { return self; } - var key = 'mma' + F.datetime.getFullYear() + '' + id; + var key = 'mma' + NOW.getFullYear() + '' + id; if (!self.cache || !self.cache[key]) self.empty(key, count); @@ -3494,7 +3494,7 @@ Counter.prototype.max = function(id, count) { return self; } - var key = 'mma' + F.datetime.getFullYear() + '' + id; + var key = 'mma' + NOW.getFullYear() + '' + id; if (!self.cache || !self.cache[key]) self.empty(key, count); else { @@ -3520,7 +3520,7 @@ Counter.prototype.inc = Counter.prototype.hit = function(id, count) { return self; } - var key = 'sum' + F.datetime.getFullYear() + '' + id; + var key = 'sum' + NOW.getFullYear() + '' + id; if (!self.cache || !self.cache[key]) self.empty(key, count || 1); else @@ -4316,7 +4316,7 @@ Counter.prototype.save = function() { var filename = self.db.filename + EXTENSION_COUNTER; var reader = Fs.createReadStream(filename); var writer = Fs.createWriteStream(filename + '-tmp'); - var dt = F.datetime.format('MMdd') + '='; + var dt = NOW.format('MMdd') + '='; var cache = self.cache; var counter = 0; @@ -4453,7 +4453,7 @@ function Binary(db, directory) { this.directory = directory; this.$events = {}; this.metafile = directory + 'meta.json'; - this.meta = { $version: 1, updated: F.datetime }; + this.meta = { $version: 1, updated: NOW }; this.cachekey = 'nobin_' + db.name + '_'; this.$refresh(); } @@ -4469,7 +4469,7 @@ Binary.prototype.$refresh = function() { this.meta.index = config.index; this.meta.count = config.count; this.meta.free = config.free || []; - this.meta.updated = config.updated || F.datetime; + this.meta.updated = config.updated || NOW; } } catch(e) {} }; @@ -4477,7 +4477,7 @@ Binary.prototype.$refresh = function() { Binary.prototype.$save = function() { var self = this; self.check(); - self.meta.updated = F.datetime; + self.meta.updated = NOW; Fs.writeFile(self.metafile, JSON.stringify(self.meta), F.error()); return self; }; @@ -4603,7 +4603,7 @@ Binary.prototype.insert = function(name, buffer, callback) { if (!dimension) dimension = { width: 0, height: 0 }; - var time = F.datetime.format('yyyyMMdd'); + var time = NOW.format('yyyyMMdd'); var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, date: time }; var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); @@ -4641,7 +4641,7 @@ Binary.prototype.insert = function(name, buffer, callback) { Binary.prototype.insertstream = function(id, name, type, stream, callback) { var self = this; - var time = F.datetime.format('yyyyMMdd'); + var time = NOW.format('yyyyMMdd'); var h = { name: name, size: 0, type: type, width: 0, height: 0, date: time }; var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); @@ -4761,7 +4761,7 @@ Binary.prototype.update = function(id, name, buffer, callback) { return self.insertstream(id, name, type, buffer, callback); var isnew = false; - var time = F.datetime.format('yyyyMMdd'); + var time = NOW.format('yyyyMMdd'); var size = buffer.length; var ext = framework_utils.getExtension(name); var dimension; @@ -5097,7 +5097,7 @@ Indexes.prototype.create = function(name, properties, type) { meta.key = key; } } else { - self.meta[name] = { key: key, documents: 0, changes: 0, cleaned: F.datetime.getTime() }; + self.meta[name] = { key: key, documents: 0, changes: 0, cleaned: NOW.getTime() }; reindex = true; } @@ -5319,7 +5319,7 @@ Indexes.prototype.reindex = function(callback) { F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" re-indexing (beg)'.format(self.db.name)); var keys = Object.keys(self.meta); - var ticks = F.datetime.getTime(); + var ticks = NOW.getTime(); for (var i = 0; i < self.indexes.length; i++) { var item = self.meta[self.indexes[i].name]; @@ -5535,7 +5535,7 @@ Indexes.prototype.flush = function() { }; var arr = self.changes.splice(0, 50); - var ticks = F.datetime.getTime() - CLEANDBTICKS; + var ticks = NOW.getTime() - CLEANDBTICKS; for (var i = 0; i < arr.length; i++) { @@ -5607,7 +5607,7 @@ Indexes.prototype.$free = function() { db.clean(function() { db.PENDING--; var a = self.meta[db.CLEANDB]; - a.cleaned = F.datetime.getTime(); + a.cleaned = NOW.getTime(); a.changes = 0; db.CLEANDB = null; self.$meta(); @@ -5662,7 +5662,7 @@ Storage.prototype.insert = function(doc) { if (doc == null) { if (self.pending.length) { - var dt = F.datetime.format('yyyyMMdd'); + var dt = NOW.format('yyyyMMdd'); self.locked_reader = true; self.check(); Fs.appendFile(self.db.filenameStorage.format(dt), self.pending.join(NEWLINE) + NEWLINE, function(err) { @@ -5691,7 +5691,7 @@ Storage.prototype.insert = function(doc) { self.check(); self.locked_writer = true; - Fs.appendFile(self.db.filenameStorage.format(F.datetime.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function(err) { + Fs.appendFile(self.db.filenameStorage.format(NOW.format('yyyyMMdd')), JSON.stringify(doc) + NEWLINE, function(err) { self.locked_writer = false; self.locked_reader = false; self.pending.length && self.insert(); @@ -5844,7 +5844,7 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { stats.processed = 0; stats.canceled = false; - var today = +F.datetime.format('yyyyMMdd'); + var today = +NOW.format('yyyyMMdd'); var process = function(item, next, index) { if (self.locked_read) { diff --git a/utils.js b/utils.js index ac50bf549..1d961b5fe 100755 --- a/utils.js +++ b/utils.js @@ -3105,7 +3105,7 @@ String.prototype.parseDate = function() { } } - return new Date(parsed[0], parsed[1] - 1, parsed[2], parsed[3], parsed[4] - F.datetime.getTimezoneOffset(), parsed[5]); + return new Date(parsed[0], parsed[1] - 1, parsed[2], parsed[3], parsed[4] - NOW.getTimezoneOffset(), parsed[5]); }; String.prototype.parseDateExpiration = function() { From 02cc6c4a57287ba899cd40a7bcca8c140b9a1f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 21 Apr 2018 09:35:33 +0200 Subject: [PATCH 0236/1669] Added a support for `async` --> `async` schema operations. --- builders.js | 304 +++++++++++++++++++++++++++++++++------------------- utils.js | 1 - 2 files changed, 194 insertions(+), 111 deletions(-) diff --git a/builders.js b/builders.js index 9fba9a3e9..fcdf93610 100755 --- a/builders.js +++ b/builders.js @@ -28,7 +28,7 @@ const REQUIRED = 'The field "@" is invalid.'; const DEFAULT_SCHEMA = 'default'; -const SKIP = { $$schema: true, $$result: true, $$callback: true, $$async: true, $$index: true, $$repository: true, $$can: true, $$controller: true }; +const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true }; const REGEXP_CLEAN_EMAIL = /\s/g; const REGEXP_CLEAN_PHONE = /\s|\.|-|\(|\)/g; const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; @@ -1955,7 +1955,7 @@ SchemaBuilderEntity.prototype.$process_hook = function(model, type, name, builde var self = this; var has = builder.hasError(); has && self.onError && self.onError(builder, model, type, name); - callback(has ? builder : null, model); + callback(has ? builder : null, model, result); return self; }; @@ -2019,7 +2019,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk var hook = self.hooks ? self.hooks[name] : undefined; if (!hook || !hook.length) { - callback(null, model); + callback(null, model, EMPTYARRAY); return self; } @@ -2037,7 +2037,6 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk var output = []; async_wait(hook, function(item, next) { - if (item.fn.$newversion) item.fn.call(self, new SchemaOptions(builder, model, options, function(result) { output.push(result == undefined ? model : result); @@ -2069,18 +2068,18 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk self.resourcePrefix && builder.setPrefix(self.resourcePrefix); async_wait(hook, function(item, next, index) { - if (!isGenerator(self, 'hook.' + name + '.' + index, item.fn)) { - if (item.fn.$newversion) + if (item.fn.$newversion) { item.fn.call(self, new SchemaOptions(builder, model, options, function(res) { output.push(res === undefined ? model : res); next(); }, controller)); - else + } else { item.fn.call(self, builder, model, options, function(res) { output.push(res === undefined ? model : res); next(); }, controller, skip !== true); + } return; } @@ -2448,25 +2447,49 @@ SchemaInstance.prototype.$$schema = null; SchemaInstance.prototype.$async = function(callback, index) { var self = this; !callback && (callback = function(){}); - self.$$async = []; - self.$$result = []; - self.$$index = index; - self.$$callback = callback; - self.$$can = true; - setImmediate(async_continue, self); + + var a = self.$$async = {}; + + a.callback = callback; + a.index = index; + a.indexer = 0; + a.response = []; + a.fn = []; + a.pending = 0; + + a.next = function() { + var fn = a.fn ? a.fn.shift() : null; + if (fn) { + a.pending++; + fn.fn(a.done, a.indexer++); + fn.async && a.next(); + } + }; + + a.done = function() { + a.pending--; + if (a.fn.length) + setImmediate(a.next); + else if (!a.pending && a.callback) + a.callback(null, a.index != null ? a.response[a.index] : a.response); + }; + + setImmediate(a.next); return self; }; -function async_continue(self) { - self.$$can = false; - async_queue(self.$$async, function() { - self.$$callback(null, self.$$index !== undefined ? self.$$result[self.$$index] : self.$$result); - self.$$callback = null; - }); +function async_wait(arr, onItem, onCallback, index) { + var item = arr[index]; + if (item) + onItem(item, () => async_wait(arr, onItem, onCallback, index + 1), index); + else + onCallback(); } SchemaInstance.prototype.$response = function(index) { - return index == undefined ? this.$$result : this.$$result[index === 'prev' ? (this.$$result.length - 1) : index]; + var a = this.$$async; + if (a) + return index == undefined ? a.response : a.response[index === 'prev' ? (a.response.length - 1) : index]; }; SchemaInstance.prototype.$repository = function(name, value) { @@ -2486,90 +2509,97 @@ SchemaInstance.prototype.$repository = function(name, value) { }; SchemaInstance.prototype.$index = function(index) { - if (typeof(index) === 'string') - this.$$index = (this.$$index || 0).add(index); - this.$$index = index; + var a = this.$$async; + if (a) { + if (typeof(index) === 'string') + a.index = (a.index || 0).add(index); + a.index = index; + } return this; }; SchemaInstance.prototype.$callback = function(callback) { - this.$$callback = callback; + var a = this.$$async; + if (a) + a.callback = callback; return this; }; SchemaInstance.prototype.$output = function() { - this.$$index = this.$$result.length; + var a = this.$$async; + if (a) + a.index = true; return this; }; SchemaInstance.prototype.$stop = function() { - this.$$async.length = 0; + this.async.length = 0; return this; }; -SchemaInstance.prototype.$push = function(type, name, helper, first) { +SchemaInstance.prototype.$push = function(type, name, helper, first, async, callback) { var self = this; var fn; if (type === 'save' || type === 'insert' || type === 'update') { - - helper = name; - name = undefined; - - fn = function(next) { + fn = function(next, indexer) { self.$$schema[type](self, helper, function(err, result) { - self.$$result && self.$$result.push(err ? null : copy(result)); + var a = self.$$async; + a.response && (a.response[indexer] = err ? null : copy(result)); + if (a.index === true) + a.index = indexer; + callback && callback(err, a.response[indexer]); if (!err) return next(); next = null; - self.$$async = null; - self.$$callback(err, self.$$result); - self.$$callback = null; + a.callback(err, a.response); }, self.$$controller); }; } else if (type === 'query' || type === 'get' || type === 'read' || type === 'remove') { - - helper = name; - name = undefined; - - fn = function(next) { + fn = function(next, indexer) { self.$$schema[type](helper, function(err, result) { - self.$$result && self.$$result.push(err ? null : copy(result)); + var a = self.$$async; + a.response && (a.response[indexer] = err ? null : copy(result)); + if (a.index === true) + a.index = indexer; + callback && callback(err, a.response[indexer]); if (!err) return next(); next = null; - self.$$async = null; - self.$$callback(err, self.$$result); - self.$$callback = null; + a.callback(err, a.response); }, self.$$controller); }; - } else { - fn = function(next) { + fn = function(next, indexer) { self.$$schema[type](name, self, helper, function(err, result) { - self.$$result && self.$$result.push(err ? null : copy(result)); + var a = self.$$async; + a.response && (a.response[indexer] = err ? null : copy(result)); + if (a.index === true) + a.index = indexer; + callback && callback(err, a.response[indexer]); if (!err) return next(); next = null; - self.$$async = null; - self.$$callback(err, self.$$result); - self.$$callback = null; + a.callback(err, a.response); }, self.$$controller); }; } + var a = self.$$async; + var obj = { fn: fn, async: async, index: a.length }; + if (first) - self.$$async.unshift(fn); + a.fn.unshift(obj); else - self.$$async.push(fn); + a.fn.push(obj); return self; }; -SchemaInstance.prototype.$next = function(type, name, helper) { - return this.$push(type, name, helper, true); +SchemaInstance.prototype.$next = function(type, name, helper, async) { + return this.$push(type, name, helper, true, async); }; SchemaInstance.prototype.$exec = function(name, helper, callback) { @@ -2596,53 +2626,99 @@ SchemaInstance.prototype.$controller = function(controller) { return this; }; -SchemaInstance.prototype.$save = function(helper, callback) { - if (this.$$can && this.$$async) - this.$push('save', helper); - else +SchemaInstance.prototype.$save = function(helper, callback, async) { + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('save', null, helper, null, async, callback); + + } else this.$$schema.save(this, helper, callback, this.$$controller); return this; }; -SchemaInstance.prototype.$insert = function(helper, callback) { - if (this.$$can && this.$$async) - this.$push('insert', helper); - else +SchemaInstance.prototype.$insert = function(helper, callback, async) { + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('insert', null, helper, null, async, callback); + + } else this.$$schema.insert(this, helper, callback, this.$$controller); return this; }; -SchemaInstance.prototype.$update = function(helper, callback) { - if (this.$$can && this.$$async) - this.$push('update', helper); - else +SchemaInstance.prototype.$update = function(helper, callback, async) { + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('update', null, helper, null, async, callback); + + } else this.$$schema.update(this, helper, callback, this.$$controller); return this; }; -SchemaInstance.prototype.$query = function(helper, callback) { - if (this.$$can && this.$$async) - this.$push('query', helper); - else +SchemaInstance.prototype.$query = function(helper, callback, async) { + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('query', null, helper, null, async, callback); + } else this.$$schema.query(this, helper, callback, this.$$controller); return this; }; -SchemaInstance.prototype.$read = SchemaInstance.prototype.$get = function(helper, callback) { +SchemaInstance.prototype.$read = SchemaInstance.prototype.$get = function(helper, callback, async) { - if (this.$$can && this.$$async) - this.$push('get', helper); - else + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('get', null, helper, null, async, callback); + } else this.$$schema.get(this, helper, callback, this.$$controller); return this; }; -SchemaInstance.prototype.$delete = SchemaInstance.prototype.$remove = function(helper, callback) { +SchemaInstance.prototype.$delete = SchemaInstance.prototype.$remove = function(helper, callback, async) { - if (this.$$can && this.$$async) - this.$push('remove', helper); - else + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('remove', null, helper, null, async, callback); + + } else this.$$schema.remove(helper, callback, this.$$controller); return this; @@ -2656,40 +2732,64 @@ SchemaInstance.prototype.$destroy = function() { return this.$$schema.destroy(); }; -SchemaInstance.prototype.$transform = function(name, helper, callback) { +SchemaInstance.prototype.$transform = function(name, helper, callback, async) { - if (this.$$can && this.$$async) - this.$push('transform', name, helper); - else + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('transform', name, helper, null, async, callback); + + } else this.$$schema.transform(name, this, helper, callback, undefined, this.$$controller); return this; }; -SchemaInstance.prototype.$workflow = function(name, helper, callback) { +SchemaInstance.prototype.$workflow = function(name, helper, callback, async) { - if (this.$$can && this.$$async) - this.$push('workflow', name, helper); - else + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('workflow', name, helper, null, async, callback); + + } else this.$$schema.workflow(name, this, helper, callback, undefined, this.$$controller); return this; }; -SchemaInstance.prototype.$hook = function(name, helper, callback) { +SchemaInstance.prototype.$hook = function(name, helper, callback, async) { - if (this.$$can && this.$$async) - this.$push('hook', name, helper); - else + if (this.$$async) { + + if (callback === true) { + var a = async; + async = true; + callback = a; + } + + this.$push('hook', name, helper, null, async, callback); + + } else this.$$schema.hook(name, this, helper, callback, undefined, this.$$controller); return this; }; -SchemaInstance.prototype.$operation = function(name, helper, callback) { +SchemaInstance.prototype.$operation = function(name, helper, callback, async) { - if (this.$$can && this.$$async) - this.$push('operation', name, helper); + if (this.$$async) + this.$push('operation', name, helper, null, async, callback); else this.$$schema.operation(name, this, helper, callback, undefined, this.$$controller); @@ -3807,22 +3907,6 @@ TransformBuilder.setDefaultTransform = function(name) { delete transforms['transformbuilder_default']; }; -function async_queue(arr, callback) { - var item = arr.shift(); - if (item) - item(() => async_queue(arr, callback)); - else - callback(); -} - -function async_wait(arr, onItem, onCallback, index) { - var item = arr[index]; - if (item) - onItem(item, () => async_wait(arr, onItem, onCallback, index + 1), index); - else - onCallback(); -} - function RESTBuilder(url) { this.$url = url; diff --git a/utils.js b/utils.js index 1d961b5fe..005a5e241 100755 --- a/utils.js +++ b/utils.js @@ -4109,7 +4109,6 @@ Number.prototype.VAT = function(percentage, decimals, includedVAT) { if (!percentage || !num) return num; - return includedVAT ? (num / ((percentage / 100) + 1)).floor(decimals) : (num * ((percentage / 100) + 1)).floor(decimals); }; From 228fbb2ca770854e07e4aef58ab9387c448757b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 21 Apr 2018 12:42:01 +0200 Subject: [PATCH 0237/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c774ad484..bb9a0f89c 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-42", + "version": "3.0.0-43", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From b8dd189a3369f0b216cb195249acfd15e3326b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 21 Apr 2018 20:29:16 +0200 Subject: [PATCH 0238/1669] Improved `String.parseConfig()`. --- index.js | 4 ++-- utils.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index df066e2a7..0899bc7b7 100755 --- a/index.js +++ b/index.js @@ -8688,9 +8688,9 @@ F.$configure_configs = function(arr, rewrite) { if (subtype === 'string') obj[name] = value; else if (subtype === 'number' || subtype === 'currency' || subtype === 'float' || subtype === 'double') - obj[name] = value.isNumber(true) ? value.parseFloat() : value.parseInt(); + obj[name] = value.isNumber(true) ? value.parseFloat2() : value.parseInt2(); else if (subtype === 'boolean' || subtype === 'bool') - obj[name] = value.parseBoolean(); + obj[name] = (/true|on|1|enabled/i).test(value); else if (subtype === 'eval' || subtype === 'object' || subtype === 'array') { try { obj[name] = new Function('return ' + value)(); diff --git a/utils.js b/utils.js index 005a5e241..5bf4fb9b6 100755 --- a/utils.js +++ b/utils.js @@ -3206,11 +3206,11 @@ String.prototype.parseConfig = function(def, onerr) { case 'float': case 'double': case 'currency': - obj[name] = value.isNumber(true) ? value.parseFloat() : value.parseInt(); + obj[name] = value.isNumber(true) ? value.parseFloat2() : value.parseInt2(); break; case 'boolean': case 'bool': - obj[name] = value.parseBoolean(); + obj[name] = (/true|on|1|enabled/i).test(value); break; case 'config': obj[name] = F.config[value]; From 002178e8005723e057b3b980ed93b2e71def56b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Apr 2018 10:37:31 +0200 Subject: [PATCH 0239/1669] Fixed compressing files in virtual directory. --- index.js | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 0899bc7b7..1a134afd5 100755 --- a/index.js +++ b/index.js @@ -45,6 +45,7 @@ const CT_TEXT = 'text/plain'; const CT_HTML = 'text/html'; const CT_JSON = 'application/json'; const COMPRESSION = { 'text/plain': true, 'text/javascript': true, 'text/css': true, 'text/jsx': true, 'application/javascript': true, 'application/x-javascript': true, 'application/json': true, 'text/xml': true, 'image/svg+xml': true, 'text/x-markdown': true, 'text/html': true }; +const COMPRESSIONSPECIAL = { 'js': 1, 'css': 1 }; const REG_TEMPORARY = /\//g; const REG_MOBILE = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|Tablet/i; const REG_ROBOT = /search|agent|bot|crawler|spider/i; @@ -5727,11 +5728,18 @@ F.compile_virtual = function(res) { return; } - if (!res.noCompress && (req.extension === 'js' || req.extension === 'css') && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) { + if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) { res.options.filename = tmpname; - compile_file(res); + return compile_file(res); + } + + var tmp = F.temporary.path[req.$key] = [tmpname, size, stats.mtime.toUTCString()]; + if (F.config['allow-gzip'] && COMPRESSION[U.getContentType(req.extension)]) { + compile_gzip(tmp, function() { + delete F.temporary.processing[req.$key]; + res.$file(); + }); } else { - F.temporary.path[req.$key] = [tmpname, size, stats.mtime.toUTCString()]; delete F.temporary.processing[req.$key]; res.$file(); } @@ -5754,7 +5762,7 @@ function compile_check(res) { if (e) { - if (!res.noCompress && (req.extension === 'js' || req.extension === 'css') && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) + if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) return compile_file(res); var tmp = F.temporary.path[req.$key] = [res.options.filename, size, stats.mtime.toUTCString()]; @@ -16073,7 +16081,7 @@ function fsFileExists(filename, callback, a, b, c) { }); } -function fsStreamRead(filename, options, callback, req, res) { +function fsStreamRead(filename, options, callback, res) { if (!callback) { callback = options; @@ -16097,7 +16105,7 @@ function fsStreamRead(filename, options, callback, req, res) { U.queue('F.files', F.config['default-maximum-file-descriptors'], function(next) { var stream = Fs.createReadStream(filename, opt); stream.on('error', NOOP); - callback(stream, next, req, res); + callback(stream, next, res); }, filename); } From 5ab83cf7784cc071d5b4345e20f86f70630a20f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Apr 2018 10:37:48 +0200 Subject: [PATCH 0240/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bb9a0f89c..46126829b 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-43", + "version": "3.0.0-44", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9c30d4631277921333feb73b8ed80d0f9f29caac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Apr 2018 20:00:37 +0200 Subject: [PATCH 0241/1669] Fixed modifying documents. --- nosql.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nosql.js b/nosql.js index 1dcf3c56b..945bb7b54 100755 --- a/nosql.js +++ b/nosql.js @@ -1370,8 +1370,7 @@ Database.prototype.$update = function() { output[key] = val(output[key], output); else output[key] = val; - } else - output[key] = undefined; + } } } else output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; @@ -1495,12 +1494,12 @@ Database.prototype.$update_inmemory = function() { if (item.keys) { for (var j = 0, jl = item.keys.length; j < jl; j++) { var val = item.doc[item.keys[j]]; - if (val === undefined) - continue; - if (typeof(val) === 'function') - doc[item.keys[j]] = val(doc[item.keys[j]], doc); - else - doc[item.keys[j]] = val; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[item.keys[j]] = val(doc[item.keys[j]], doc); + else + doc[item.keys[j]] = val; + } } } else doc = typeof(item.doc) === 'function' ? item.doc(doc) : item.doc; From ebdaa64a76267924111ad9f6f2689b4e1e845013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Apr 2018 20:00:49 +0200 Subject: [PATCH 0242/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 46126829b..c5aabdc3c 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-44", + "version": "3.0.0-45", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From b0b9992c7823af88865f75085af94511cb1afbce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 10:36:21 +0200 Subject: [PATCH 0243/1669] Added `nosql.ready(fn)`. --- changes.txt | 1 + nosql.js | 24 ++++++++++++++++++++++++ nosqlworker.js | 1 + 3 files changed, 26 insertions(+) diff --git a/changes.txt b/changes.txt index 14b74d534..5813094f1 100755 --- a/changes.txt +++ b/changes.txt @@ -26,6 +26,7 @@ - added: `DatabaseBuilder.query(code)` can contain a raw JS condition in the form e.g. `doc.age > 18 && doc.age < 33` - added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) - added: `Database.lock(callback(next))` locks all internal DB operations +- added: `Database.ready(callback)` executes a callback when DB is ready to use (only for special cases if you use indexes) - added: new directory `schemas` with a new configuration item `directory-schemas' - added: new directory `operations` with a new configuration item `directory-operations' - added: `String.crc32([unsigned])` diff --git a/nosql.js b/nosql.js index 945bb7b54..51d2206cd 100755 --- a/nosql.js +++ b/nosql.js @@ -342,6 +342,11 @@ exports.worker = function() { return this; }; + Database.prototype.ready = function(callback) { + send(this, 'ready').callback = callback; + return this; + }; + Database.prototype.remove = function(filename) { return send(this, 'remove', filename).builder = new DatabaseBuilder(this); }; @@ -654,6 +659,19 @@ function next_operation(self, type) { self.next(type); } +Database.prototype.ready = function(fn) { + var self = this; + + if (!self.indexes) + fn.call(self); + else if (self.indexes.isreindex || self.indexes.reindexing) + setTimeout((self, fn) => self.ready(fn), 500, self, fn); + else + fn.call(self); + + return self; +}; + Database.prototype.insert = function(doc, unique) { var self = this; @@ -5067,6 +5085,7 @@ function Indexes(db, directory) { this.flushing = false; this.instances = {}; this.reindexing = false; + this.isreindex = false; this.meta = { $version: 1 }; try { this.meta = Fs.readFileSync(this.db.filename + EXTENSION_INDEXES).toString('utf8').parseJSON(true) || {}; @@ -5100,12 +5119,16 @@ Indexes.prototype.create = function(name, properties, type) { reindex = true; } + if (!self.isreindex && reindex) + self.isreindex = reindex; + reindex && setTimeout2(self.db.name + '_reindex', () => self.reindex(), 1000); return self; }; Indexes.prototype.noreindex = function() { var self = this; + self.isreindex = false; clearTimeout2(self.db.name + '_reindex'); return self; }; @@ -5345,6 +5368,7 @@ Indexes.prototype.reindex = function(callback) { self.insert(docs[i], true); self.$reindexingnext = next; }, function() { + self.isreindex = false; self.$meta(); self.db.$events['indexing-end'] && self.db.emit('indexing-end'); self.reindexing = false; diff --git a/nosqlworker.js b/nosqlworker.js index 28b0acfcf..530fd3e52 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -273,6 +273,7 @@ process.on('message', function(msg) { break; case 'clean': case 'clear': + case 'ready': db[msg.TYPE](function(err) { RESCALLBACK.id = msg.id; RESCALLBACK.err = err; From 8fc307de10be5769f78a4687af06f6166c5ede8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 12:01:40 +0200 Subject: [PATCH 0244/1669] Fixed schema validation. --- builders.js | 4 ++++ utils.js | 50 +++++++++++++++++++++----------------------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/builders.js b/builders.js index fcdf93610..27bc0df11 100755 --- a/builders.js +++ b/builders.js @@ -1734,6 +1734,10 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { // object case 6: item[property] = self.$onprepare(property, model[property], undefined, model); + + if (item[property] === undefined) + item[property] = null; + break; // enum diff --git a/utils.js b/utils.js index 5bf4fb9b6..745b90b6a 100755 --- a/utils.js +++ b/utils.js @@ -2038,6 +2038,7 @@ exports.validate_builder = function(model, error, schema, collection, path, inde var prepare = entity.onValidate || F.onValidate || NOOP; var current = path === undefined ? '' : path + '.'; var properties = entity.properties; + var result; if (!pluspath) pluspath = ''; @@ -2052,7 +2053,6 @@ exports.validate_builder = function(model, error, schema, collection, path, inde continue; var TYPE = collection[schema].schema[name]; - if (TYPE.can && !TYPE.can(model)) continue; @@ -2068,36 +2068,28 @@ exports.validate_builder = function(model, error, schema, collection, path, inde if (TYPE.isArray) { - if (!(value instanceof Array) || !value.length) { - error.push(pluspath + name, '@', current + name, index, prefix); - continue; - } - - for (var j = 0, jl = value.length; j < jl; j++) { - if (TYPE.type === 7) { - // Another schema + if (TYPE.type === 7 && value instanceof Array && value.length) { + for (var j = 0, jl = value.length; j < jl; j++) exports.validate_builder(value[j], error, TYPE.raw, collection, current + name + '[' + j + ']', j, undefined, pluspath); - } else { - - // Basic types - var result = TYPE.validate ? TYPE.validate(value[j], model) : prepare(name, value, current + name + '[' + j + ']', model, schema, TYPE); - if (result == null) { - result = validate_builder_default(name, value[j], TYPE); - if (result == null || result === true) - continue; - } + } else { - type = typeof(result); - if (type === 'string') { - if (result[0] === '@') - error.push(pluspath + name, '@', current + name + '[' + j + ']', j, entity.resourcePrefix + result.substring(1)); - else - error.push(pluspath + name, result, current + name + '[' + j + ']', j, prefix); - } else if (type === 'boolean') { - !result && error.push(pluspath + name, '@', current + name + '[' + j + ']', j, prefix); - } else if (result.isValid === false) - error.push(pluspath + name, result.error, current + name + '[' + j + ']', j, prefix); + result = TYPE.validate ? TYPE.validate(value, model) : prepare(name, value, current + name, model, schema, TYPE); + if (result == null) { + result = validate_builder_default(name, value[j], TYPE); + if (result == null || result === true) + continue; } + + type = typeof(result); + if (type === 'string') { + if (result[0] === '@') + error.push(pluspath + name, '@', current + name, index, entity.resourcePrefix + result.substring(1)); + else + error.push(pluspath + name, result, current + name, index, prefix); + } else if (type === 'boolean') { + !result && error.push(pluspath + name, '@', current + name, index, prefix); + } else if (result.isValid === false) + error.push(pluspath + name, result.error, current + name, index, prefix); } continue; } @@ -2108,7 +2100,7 @@ exports.validate_builder = function(model, error, schema, collection, path, inde continue; } - var result = TYPE.validate ? TYPE.validate(value, model) : prepare(name, value, current + name, model, schema, TYPE); + result = TYPE.validate ? TYPE.validate(value, model) : prepare(name, value, current + name, model, schema, TYPE); if (result == null) { result = validate_builder_default(name, value, TYPE); if (result == null || result === true) From 766a5a1740e5e0b15e3987049cd2e60d013aab07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 15:34:56 +0200 Subject: [PATCH 0245/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c5aabdc3c..d8eceb5b4 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-45", + "version": "3.0.0-46", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 10383713ee5ea865241c81c698f38526bd2a6024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 19:45:13 +0200 Subject: [PATCH 0246/1669] Added promises. --- nosql.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nosql.js b/nosql.js index 51d2206cd..5617e03c0 100755 --- a/nosql.js +++ b/nosql.js @@ -2708,6 +2708,8 @@ function DatabaseBuilder2(db) { this.$options = {}; } +DatabaseBuilder2.prototype.promise = promise; + DatabaseBuilder2.prototype.log = function(msg, user) { var self = this; if (msg) { @@ -2755,6 +2757,19 @@ function DatabaseBuilder(db) { this.$counter = 0; } +function promise(fn) { + return new Promise(function(resolve, reject) { + this.callback(function(err, result) { + if (err) + reject(err); + else + resolve(fn == null ? result : fn(result)); + }); + }); +} + +DatabaseBuilder.prototype.promise = promise; + DatabaseBuilder.prototype.id = function(id) { this.$options.id = id; return this; @@ -3400,6 +3415,7 @@ function Counter(db) { self.$events = {}; } +Counter.prototype.promise = promise; Counter.prototype.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { @@ -4475,6 +4491,7 @@ function Binary(db, directory) { this.$refresh(); } +Binary.prototype.promise = promise; Binary.prototype.$refresh = function() { this.meta.index = 0; this.meta.count = 0; @@ -5092,6 +5109,7 @@ function Indexes(db, directory) { } catch (e) {} } +Indexes.prototype.promise = promise; Indexes.prototype.create = function(name, properties, type) { var self = this; @@ -5657,6 +5675,7 @@ function Storage(db, directory) { } } +Storage.prototype.promise = promise; Storage.prototype.refresh = function() { try { this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); From 533e1ee58a35a3b921baa8be8656914201b2ba18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 19:46:37 +0200 Subject: [PATCH 0247/1669] A small update of function. --- nosql.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nosql.js b/nosql.js index 5617e03c0..7f06b703f 100755 --- a/nosql.js +++ b/nosql.js @@ -77,6 +77,17 @@ var JSONBUFFER = process.argv.findIndex(n => n.endsWith('nosqlworker.js')) === - var FORK; var FORKCALLBACKS; +function promise(fn) { + return new Promise(function(resolve, reject) { + this.callback(function(err, result) { + if (err) + reject(err); + else + resolve(fn == null ? result : fn(result)); + }); + }); +} + Object.freeze(EMPTYARRAY); exports.kill = function(signal) { @@ -2757,17 +2768,6 @@ function DatabaseBuilder(db) { this.$counter = 0; } -function promise(fn) { - return new Promise(function(resolve, reject) { - this.callback(function(err, result) { - if (err) - reject(err); - else - resolve(fn == null ? result : fn(result)); - }); - }); -} - DatabaseBuilder.prototype.promise = promise; DatabaseBuilder.prototype.id = function(id) { From 35510c850f5b9643aaa1db1efdea002b8fc89bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 20:36:18 +0200 Subject: [PATCH 0248/1669] Fixed NoSQL `binary.all()`. --- nosql.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 7f06b703f..6988f2eb6 100755 --- a/nosql.js +++ b/nosql.js @@ -5013,8 +5013,9 @@ Binary.prototype.browse = function(directory, callback) { stream.on('data', function(buffer) { var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, '').parseJSON(true); if (json) { - json.id = item.substring(0, item.length - le); - json.id = +json.id.substring(json.id.length - DIRECTORYLENGTH); + var id = item.substring(0, item.length - le); + json.id = 'B' + json.date + 'T' + id; + json.index = +id.substring(id.length - DIRECTORYLENGTH); json.ctime = stat.ctime; json.mtime = stat.mtime; output.push(json); From 35dcebbfd66b90b7ec4c1b50e8d37eaf916206e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 20:36:42 +0200 Subject: [PATCH 0249/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d8eceb5b4..d37d17540 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-46", + "version": "3.0.0-47", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From b871bdc06e2fc72a56379a41f7ea885e9102e32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Apr 2018 20:44:44 +0200 Subject: [PATCH 0250/1669] Fixed promises. --- nosql.js | 3 ++- package.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 6988f2eb6..afa59b5af 100755 --- a/nosql.js +++ b/nosql.js @@ -78,8 +78,9 @@ var FORK; var FORKCALLBACKS; function promise(fn) { + var self = this; return new Promise(function(resolve, reject) { - this.callback(function(err, result) { + self.callback(function(err, result) { if (err) reject(err); else diff --git a/package.json b/package.json index d37d17540..d70a254bf 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-47", + "version": "3.0.0-48", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 59cd0c1135413898784cfca63579b74378745ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 25 Apr 2018 09:31:57 +0200 Subject: [PATCH 0251/1669] Updated promises. --- builders.js | 12 ++++++++++++ changes.txt | 2 ++ nosql.js | 4 ---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/builders.js b/builders.js index 27bc0df11..0aa8a98c3 100755 --- a/builders.js +++ b/builders.js @@ -3956,6 +3956,18 @@ RESTBuilder.setDefaultTransform = function(name) { delete transforms['restbuilder_default']; }; +RESTBuilder.prototype.promise = function(fn) { + var self = this; + return new Promise(function(resolve, reject) { + self.exec(function(err, result) { + if (err) + reject(err); + else + resolve(fn == null ? result : fn(result)); + }); + }); +}; + RESTBuilder.prototype.setTransform = function(name) { this.$transform = name; return this; diff --git a/changes.txt b/changes.txt index 5813094f1..7a6ed33b1 100755 --- a/changes.txt +++ b/changes.txt @@ -48,6 +48,8 @@ - added: `UID([type])` supports custom types, e.g. `UID('users')` or `UID('orders')` - added: `REQUEST()` global method, it's alias to `U.request()` - added: `NOW` global property, it's alias to `F.datetime` +- added: `DatabaseBuilder.promise()` +- added: `RESTBuilder.promise()` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/nosql.js b/nosql.js index afa59b5af..7b9329ed6 100755 --- a/nosql.js +++ b/nosql.js @@ -3416,7 +3416,6 @@ function Counter(db) { self.$events = {}; } -Counter.prototype.promise = promise; Counter.prototype.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { @@ -4492,7 +4491,6 @@ function Binary(db, directory) { this.$refresh(); } -Binary.prototype.promise = promise; Binary.prototype.$refresh = function() { this.meta.index = 0; this.meta.count = 0; @@ -5111,7 +5109,6 @@ function Indexes(db, directory) { } catch (e) {} } -Indexes.prototype.promise = promise; Indexes.prototype.create = function(name, properties, type) { var self = this; @@ -5677,7 +5674,6 @@ function Storage(db, directory) { } } -Storage.prototype.promise = promise; Storage.prototype.refresh = function() { try { this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); From 5b4cc2d40ca199b1da98ea61c5422ac6c9fbda5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 25 Apr 2018 10:19:18 +0200 Subject: [PATCH 0252/1669] Fixed parsing of response in `RESTBuilder`. --- builders.js | 54 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/builders.js b/builders.js index 0aa8a98c3..9529a0563 100755 --- a/builders.js +++ b/builders.js @@ -4180,6 +4180,11 @@ RESTBuilder.prototype.raw = function(value) { return this; }; +RESTBuilder.prototype.custom = function() { + this.$custom = true; + return this; +}; + RESTBuilder.prototype.cook = function(value) { this.$flags = null; this.$persistentcookies = value !== false; @@ -4293,7 +4298,7 @@ RESTBuilder.prototype.exec = function(callback) { var data = F.cache.read2(key); if (data) { var evt = new framework_utils.EventEmitter2(); - process.nextTick(exec_removelisteners, evt); + setImmediate(exec_removelisteners, evt); callback(null, self.maketransform(this.$schema ? this.$schema.make(data.value) : data.value, data), data); return evt; } @@ -4304,19 +4309,31 @@ RESTBuilder.prototype.exec = function(callback) { var type = err ? '' : headers['content-type'] || ''; var output = new RESTBuilderResponse(); - switch (type.toLowerCase()) { - case 'text/xml': - output.value = response.parseXML(); - break; - case 'application/x-www-form-urlencoded': - output.value = F.onParseQuery(response); - break; - case 'application/json': - output.value = response.parseJSON(true); - break; - default: - output.value = response.isJSON() ? response.parseJSON(true) : null; - break; + if (type) { + var index = type.lastIndexOf(';'); + if (index !== 1) + type = type.substring(0, index).trim(); + } + + if (self.$custom) { + output.value = response; + } else { + switch (type.toLowerCase()) { + case 'text/xml': + case 'application/xml': + output.value = response.parseXML(); + break; + case 'application/x-www-form-urlencoded': + output.value = F.onParseQuery(response); + break; + case 'application/json': + case 'text/json': + output.value = response.parseJSON(true); + break; + default: + output.value = response.isJSON() ? response.parseJSON(true) : null; + break; + } } if (output.value == null) @@ -4340,13 +4357,12 @@ RESTBuilder.prototype.exec = function(callback) { output.cache = true; }); - return; + } else { + !err && key && F.cache.add(key, output, self.$cache_expire); + callback(err, self.maketransform(output.value, output), output); + output.cache = true; } - !err && key && F.cache.add(key, output, self.$cache_expire); - callback(err, self.maketransform(output.value, output), output); - output.cache = true; - }, self.$cookies, self.$headers, undefined, self.$timeout, self.$files); }; From 9ac184c8b55ad8629488ef5966a8096b3ab6eef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 25 Apr 2018 10:19:51 +0200 Subject: [PATCH 0253/1669] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 7a6ed33b1..e486b143a 100755 --- a/changes.txt +++ b/changes.txt @@ -50,6 +50,7 @@ - added: `NOW` global property, it's alias to `F.datetime` - added: `DatabaseBuilder.promise()` - added: `RESTBuilder.promise()` +- added: `RESTBuilder.custom()` it returns a raw string from the response body - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` From 775d7ea81673ff9c0da5d643be8ee79af7e4b922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 25 Apr 2018 10:25:48 +0200 Subject: [PATCH 0254/1669] Renamed method `.custom()` to `.plain()`. --- builders.js | 6 +++--- changes.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/builders.js b/builders.js index 9529a0563..3a5e6d031 100755 --- a/builders.js +++ b/builders.js @@ -4180,8 +4180,8 @@ RESTBuilder.prototype.raw = function(value) { return this; }; -RESTBuilder.prototype.custom = function() { - this.$custom = true; +RESTBuilder.prototype.plain = function() { + this.$plain = true; return this; }; @@ -4315,7 +4315,7 @@ RESTBuilder.prototype.exec = function(callback) { type = type.substring(0, index).trim(); } - if (self.$custom) { + if (self.$plain) { output.value = response; } else { switch (type.toLowerCase()) { diff --git a/changes.txt b/changes.txt index e486b143a..639c900cd 100755 --- a/changes.txt +++ b/changes.txt @@ -50,7 +50,7 @@ - added: `NOW` global property, it's alias to `F.datetime` - added: `DatabaseBuilder.promise()` - added: `RESTBuilder.promise()` -- added: `RESTBuilder.custom()` it returns a raw string from the response body +- added: `RESTBuilder.plain()` it returns a raw string from the response body - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` From 837805ed6e231b79e479c95044d334e7363786b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Apr 2018 20:45:02 +0200 Subject: [PATCH 0255/1669] Fixed bundles. --- bundles.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bundles.js b/bundles.js index cfb9e276c..f390944cf 100755 --- a/bundles.js +++ b/bundles.js @@ -159,7 +159,13 @@ function cleanFiles(callback) { blacklist[F.config['directory-private']] = 1; blacklist[F.config['directory-databases']] = 1; - var meta = U.parseJSON(Fs.readFileSync(Path.join(path, 'bundle.json')).toString('utf8'), true) || {}; + var meta; + + try { + meta = U.parseJSON(Fs.readFileSync(Path.join(path, 'bundle.json')).toString('utf8'), true) || {}; + } catch (e) { + meta = {}; + } if (meta.files && meta.files.length) { for (var i = 0, length = meta.files.length; i < length; i++) { From 1837cd8a63beaf7bf7c3d7d7fc8c6f6c0c78ae0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Apr 2018 20:45:19 +0200 Subject: [PATCH 0256/1669] Updated version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d70a254bf..8e9d45b86 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-48", + "version": "3.0.0-49", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0d00af504bd96192504c52b557f7473af9253be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Apr 2018 20:47:12 +0200 Subject: [PATCH 0257/1669] Added better handling of errors in NoSQL binary files. --- nosql.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 7b9329ed6..76eaa5e1a 100755 --- a/nosql.js +++ b/nosql.js @@ -4878,9 +4878,18 @@ Binary.prototype.read = function(id, callback) { var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); stream.on('data', function(buffer) { + var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, ''); + var meta; + stream = Fs.createReadStream(filename, BINARYREADDATA); - callback(null, stream, JSON.parse(json, jsonparser)); + try { + meta = JSON.parse(json, jsonparser); + callback(null, stream, meta); + } catch (e) { + F.error(e, 'nosql.binary.read', filename); + callback(e); + } CLEANUP(stream); }); From c7f8c954edc0ab56263cc9729c515626a7f16b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 11:17:09 +0200 Subject: [PATCH 0258/1669] Improved regexp for `NEWOPERATIONS`. --- builders.js | 2 +- index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index 3a5e6d031..c9da50f89 100755 --- a/builders.js +++ b/builders.js @@ -31,7 +31,7 @@ const DEFAULT_SCHEMA = 'default'; const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true }; const REGEXP_CLEAN_EMAIL = /\s/g; const REGEXP_CLEAN_PHONE = /\s|\.|-|\(|\)/g; -const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; +const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); diff --git a/index.js b/index.js index 1a134afd5..4d940f43f 100755 --- a/index.js +++ b/index.js @@ -52,7 +52,7 @@ const REG_ROBOT = /search|agent|bot|crawler|spider/i; const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.]+\.(jpg|js|css|png|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; -const REG_NEWIMPL = /^(async\s)?function(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; +const REG_NEWIMPL = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; const REG_RANGE = /bytes=/; const REG_EMPTY = /\s/g; const REG_ACCEPTCLEANER = /\s|\./g; From 9795bd4af170909437de3bfe0e44e9081a9c824a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 11:17:18 +0200 Subject: [PATCH 0259/1669] Fixed validation for schema arrays. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 745b90b6a..87acd7acd 100755 --- a/utils.js +++ b/utils.js @@ -2075,7 +2075,7 @@ exports.validate_builder = function(model, error, schema, collection, path, inde result = TYPE.validate ? TYPE.validate(value, model) : prepare(name, value, current + name, model, schema, TYPE); if (result == null) { - result = validate_builder_default(name, value[j], TYPE); + result = value[j] instanceof Array ? value[j].length > 0 : false; if (result == null || result === true) continue; } From 8dc09a4e446bc14addb727b64d437331b7a8d8e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 11:17:22 +0200 Subject: [PATCH 0260/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8e9d45b86..1c48b6d80 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-49", + "version": "3.0.0-50", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From bddbde0089a8a2e339e04e3279072339a0ade438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 12:14:28 +0200 Subject: [PATCH 0261/1669] Fixed array validation again. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 87acd7acd..d98816c9d 100755 --- a/utils.js +++ b/utils.js @@ -2075,7 +2075,7 @@ exports.validate_builder = function(model, error, schema, collection, path, inde result = TYPE.validate ? TYPE.validate(value, model) : prepare(name, value, current + name, model, schema, TYPE); if (result == null) { - result = value[j] instanceof Array ? value[j].length > 0 : false; + result = value instanceof Array ? value.length > 0 : false; if (result == null || result === true) continue; } From ca3ba2a418b6afd79ed4389cf231442939e06fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 12:14:41 +0200 Subject: [PATCH 0262/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1c48b6d80..cff096661 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-50", + "version": "3.0.0-51", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4f4531ac52720d3df3f58c188b4631526845cd1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 20:18:02 +0200 Subject: [PATCH 0263/1669] New updates. --- changes.txt | 2 ++ index.js | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index 639c900cd..1eac6bee3 100755 --- a/changes.txt +++ b/changes.txt @@ -51,6 +51,7 @@ - added: `DatabaseBuilder.promise()` - added: `RESTBuilder.promise()` - added: `RESTBuilder.plain()` it returns a raw string from the response body +- added: `versions` file supports `auto` value for generating auto-checksum of files - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` @@ -66,6 +67,7 @@ - updated: Chunker supports `autoremove` processed files in `each()` or `read()` method, default `true` - updated: `String.parseConfig(def, [onError])` can handle errors better - updated: `middleware`, now Total.js supports new declaration `F.middleware(function($) {})` +- updated: `F.wait()` HTML template - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index 4d940f43f..4f02ddf1d 100755 --- a/index.js +++ b/index.js @@ -2336,7 +2336,7 @@ global.MERGE = F.merge = function(url) { url = '/' + url; var filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'merged_' + createTemporaryKey(url)); - F.routes.merge[url] = { filename: filename.replace(/\.(js|css)$/g, ext => '.min' + ext), files: arr }; + F.routes.merge[url] = { filename: filename.replace(/\.(js|css)$/g, ext => '.min' + ext), files: arr, stamp: 0 }; Fs.unlink(F.routes.merge[url].filename, NOOP); F.owners.push({ type: 'merge', owner: _owner, id: url }); return F; @@ -6419,10 +6419,10 @@ F.response501 = function(req, res, problem) { F.response503 = function(req, res) { var keys = ''; for (var m in F.waits) - keys += (res.options.body ? ', ' : '') + '' + m + ''; + keys += '' + m + '⧗'; res.options.code = 503; res.options.headers = HEADERS.response503; - res.options.body = '
Please wait (10) for ' + (F.config.name + ' v' + F.config.version) + ' application.
The application is waiting for: ' + keys + '.'; + res.options.body = 'Please wait
10
PLEASE WAIT
Application is starting …
' + keys + '
'; res.$throw(); return F; }; @@ -8511,13 +8511,54 @@ F.$configure_versions = function(arr, clean) { var len = ismap ? 3 : 2; var key = str.substring(0, index).trim(); var filename = str.substring(index + len).trim(); - F.versions[key] = filename; - ismap && F.map(filename, F.path.public(key)); + + if (filename === 'auto') { + + if (ismap) + throw new Error('/versions: "auto" value can\'t be used with mapping'); + + (function(key, filename) { + ON('ready', function() { + F.consoledebug('"versions" is getting checksum of ' + key); + makehash(key, function(hash) { + F.consoledebug('"versions" is getting checksum of ' + key + ' (done)'); + if (hash) { + var index = key.lastIndexOf('.'); + filename = key.substring(0, index) + '-' + hash + key.substring(index); + F.versions[key] = filename; + } + }); + }); + })(key, filename); + } else { + F.versions[key] = filename; + ismap && F.map(filename, F.path.public(key)); + } } return F; }; +function makehash(url, callback) { + url = 'http://' + (F.ip === 'auto' ? '0.0.0.0' : F.ip) + ':' + F.port + url; + U.download(url, ['get'], function(err, stream, status) { + + if (status !== 200) { + callback(''); + return; + } + + var hash = Crypto.createHash('md5'); + hash.setEncoding('hex'); + stream.pipe(hash); + stream.on('end', function() { + hash.end(); + callback(hash.read().crc32(true)); + }); + stream.on('error', () => callback('')); + }); +} + F.$configure_configs = function(arr, rewrite) { var type = typeof(arr); From 4c8bef1ddfbc86c9205a8129ea4ef0f596b41783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Apr 2018 20:18:58 +0200 Subject: [PATCH 0264/1669] Removed useless field. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 4f02ddf1d..cb93a5683 100755 --- a/index.js +++ b/index.js @@ -2336,7 +2336,7 @@ global.MERGE = F.merge = function(url) { url = '/' + url; var filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'merged_' + createTemporaryKey(url)); - F.routes.merge[url] = { filename: filename.replace(/\.(js|css)$/g, ext => '.min' + ext), files: arr, stamp: 0 }; + F.routes.merge[url] = { filename: filename.replace(/\.(js|css)$/g, ext => '.min' + ext), files: arr }; Fs.unlink(F.routes.merge[url].filename, NOOP); F.owners.push({ type: 'merge', owner: _owner, id: url }); return F; From 6e764eafe57dc2f2bf414ac470063edf20a8ad06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 28 Apr 2018 21:11:59 +0200 Subject: [PATCH 0265/1669] Added testing for `LOAD()`. --- index.js | 73 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/index.js b/index.js index cb93a5683..df891f664 100755 --- a/index.js +++ b/index.js @@ -1258,6 +1258,11 @@ global.NOSQL = F.nosql = function(name) { F.stop = F.kill = function(signal) { + if (F.isKilled) + return F; + + F.isKilled = true; + if (!signal) signal = 'SIGTERM'; @@ -4463,7 +4468,6 @@ F.$restart = function() { F.controllers = {}; F.dependencies = {}; F.isomorphic = {}; - F.tests = []; F.errors = []; F.problems = []; F.changes = []; @@ -6453,10 +6457,37 @@ global.LOAD = F.load = function(debug, types, pwd) { else if (pwd) F.directory = directory = U.$normalize(pwd); - if (debug === 'release') - debug = false; - else if (debug === 'debug') - debug = true; + if (typeof(debug) === 'string') { + switch (debug.toLowerCase().replace(/\.|\s/g, '-')) { + case 'release': + case 'production': + break; + + case 'debug': + case 'develop': + case 'development': + debug = true; + break; + + case 'test-debug': + case 'debug-test': + case 'testing-debug': + debug = true; + F.isTest = true; + break; + + case 'test': + case 'testing': + case 'test-release': + case 'release-test': + case 'testing-release': + case 'test-production': + case 'testing-production': + debug = false; + F.isTest = true; + break; + } + } F.isWorker = true; F.config.debug = debug; @@ -6503,11 +6534,17 @@ global.LOAD = F.load = function(debug, types, pwd) { F.removeAllListeners('load'); F.removeAllListeners('ready'); - // clear unnecessary items - delete F.tests; - delete F.test; - delete F.testing; - delete F.assert; + if (F.isTest) { + F.console(); + F.test(); + return F; + } + + setTimeout(function() { + if (!F.isTest) + delete F.test; + }, 5000); + }, 500); if (F.config['allow-debug']) { @@ -6660,17 +6697,13 @@ F.initialize = function(http, debug, options, restart) { if (F.isTest) { var sleep = options.sleep || options.delay || 1000; - setTimeout(() => F.test(true, options.tests || options.test), sleep); + setTimeout(F.test, sleep); return F; } setTimeout(function() { - if (F.isTest) - return; - delete F.tests; - delete F.test; - delete F.testing; - delete F.assert; + if (!F.isTest) + delete F.test; }, 5000); }); }, true); @@ -6879,8 +6912,10 @@ F.console = function() { console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); console.log('Mode : ' + (F.config.debug ? 'debug' : 'release')); console.log('====================================================\n'); - console.log('{2}://{0}:{1}/'.format(F.ip, F.port, F.isHTTPS ? 'https' : 'http')); - console.log(''); + if (!F.isWorker) { + console.log('{2}://{0}:{1}/'.format(F.ip, F.port, F.isHTTPS ? 'https' : 'http')); + console.log(''); + } }; F.usagesnapshot = function(filename) { From 2fdb11fb9284bf086107045ad6ee6729832ca547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 28 Apr 2018 21:12:36 +0200 Subject: [PATCH 0266/1669] Added a new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 1eac6bee3..82a4ea400 100755 --- a/changes.txt +++ b/changes.txt @@ -52,6 +52,7 @@ - added: `RESTBuilder.promise()` - added: `RESTBuilder.plain()` it returns a raw string from the response body - added: `versions` file supports `auto` value for generating auto-checksum of files +- added: `F.load()` supports `test` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` From ddc6f593e8ca15fe8d56cb65f94ff6526ed6d20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 09:15:39 +0200 Subject: [PATCH 0267/1669] Added `custom` small data attributes to NoSQL binary. --- changes.txt | 1 + nosql.js | 87 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/changes.txt b/changes.txt index 82a4ea400..f65733fa2 100755 --- a/changes.txt +++ b/changes.txt @@ -53,6 +53,7 @@ - added: `RESTBuilder.plain()` it returns a raw string from the response body - added: `versions` file supports `auto` value for generating auto-checksum of files - added: `F.load()` supports `test` +- added: NoSQL binary supports `custom` small data attributes - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/nosql.js b/nosql.js index 76eaa5e1a..7575409f7 100755 --- a/nosql.js +++ b/nosql.js @@ -4590,22 +4590,29 @@ Binary.prototype.removeAllListeners = function(name) { return this; }; -Binary.prototype.insert = function(name, buffer, callback) { +Binary.prototype.insert = function(name, buffer, custom, callback) { var self = this; var type = framework_utils.getContentType(framework_utils.getExtension(name)); - var isfn = typeof(buffer) === 'function'; - - if (isfn || !buffer) { - if (isfn) { + if (buffer && !(buffer instanceof Buffer)) { + if (typeof(buffer) === 'function') { callback = buffer; - buffer = undefined; + buffer = custom = null; + } else { + callback = custom; + custom = buffer; + buffer = null; } + } else if (typeof(custom) === 'function') { + callback = custom; + custom = null; + } + if (!buffer) { var reader = Fs.createReadStream(name); CLEANUP(reader); - return self.insertstream(null, framework_utils.getName(name), type, reader, callback); + return self.insertstream(null, framework_utils.getName(name), type, reader, callback, custom); } if (typeof(buffer) === 'string') @@ -4633,11 +4640,20 @@ Binary.prototype.insert = function(name, buffer, callback) { break; } - if (!dimension) - dimension = { width: 0, height: 0 }; - var time = NOW.format('yyyyMMdd'); - var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, date: time }; + var h = { name: name, size: size, type: type, date: time }; + + if (custom) + h.custom = custom; + + if (dimension) { + if (dimension.width) + h.width = dimension.width; + + if (dimension.height) + h.height = dimension.height; + } + var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); header.write(JSON.stringify(h)); @@ -4671,11 +4687,15 @@ Binary.prototype.insert = function(name, buffer, callback) { return id; }; -Binary.prototype.insertstream = function(id, name, type, stream, callback) { +Binary.prototype.insertstream = function(id, name, type, stream, callback, custom) { var self = this; var time = NOW.format('yyyyMMdd'); - var h = { name: name, size: 0, type: type, width: 0, height: 0, date: time }; + var h = { name: name, size: 0, type: type, date: time }; + + if (custom) + h.custom = custom; + var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); @@ -4747,8 +4767,10 @@ Binary.prototype.insertstream = function(id, name, type, stream, callback) { CLEANUP(writer, function() { if (dimension) { - h.width = dimension.width; - h.height = dimension.height; + if (dimension.width) + h.width = dimension.width; + if (dimension.height) + h.height = dimension.height; } h.size = writer.bytesWritten; @@ -4769,13 +4791,27 @@ Binary.prototype.insertstream = function(id, name, type, stream, callback) { return cacheid || id; }; -Binary.prototype.update = function(id, name, buffer, callback) { +Binary.prototype.update = function(id, name, buffer, custom, callback) { var type = framework_utils.getContentType(framework_utils.getExtension(name)); var self = this; var isfn = typeof(buffer) === 'function'; - if (isfn || !buffer) { + if (buffer && !(buffer instanceof Buffer)) { + if (typeof(buffer) === 'function') { + callback = buffer; + buffer = custom = null; + } else { + callback = custom; + custom = buffer; + buffer = null; + } + } else if (typeof(custom) === 'function') { + callback = custom; + custom = null; + } + + if (!buffer) { if (isfn) { callback = buffer; @@ -4784,7 +4820,7 @@ Binary.prototype.update = function(id, name, buffer, callback) { var reader = Fs.createReadStream(name); CLEANUP(reader); - return self.insertstream(id, framework_utils.getName(name), type, reader, callback); + return self.insertstream(id, framework_utils.getName(name), type, reader, callback, custom); } if (typeof(buffer) === 'string') @@ -4836,10 +4872,19 @@ Binary.prototype.update = function(id, name, buffer, callback) { break; } - if (!dimension) - dimension = { width: 0, height: 0 }; + var h = { name: name, size: size, type: type, date: time }; + + if (custom) + h.custom = custom; + + if (dimension) { + if (dimension.width) + h.width = dimension.width; + + if (dimension.height) + h.height = dimension.height; + } - var h = { name: name, size: size, type: type, width: dimension.width, height: dimension.height, date: time }; var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); From a1fbedbb0a9ef2b67fa0f9ae41ab703ceb049649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 09:15:58 +0200 Subject: [PATCH 0268/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cff096661..14ae57f71 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-51", + "version": "3.0.0-52", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9239d723d0e379cc344ef95a687bb3260ce05dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 17:56:03 +0200 Subject: [PATCH 0269/1669] Fixed reading of last document in DB. --- nosqlstream.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/nosqlstream.js b/nosqlstream.js index d0d542b40..014600230 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -1,4 +1,4 @@ -// Copyright 2012-2018 (c) Peter Širka +// Copyright 2018 (c) Peter Širka // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the @@ -326,7 +326,6 @@ NoSQLStream.prototype.writehelpers = function() { beg = 0; self.buffer = Buffer.concat(self.cache); - } else self.buffer = chunk; @@ -566,6 +565,12 @@ NoSQLStream.prototype.read = function() { if (!self.fd || size <= 0 || self.canceled) { + if (self.buffer.length && !self.canceled) { + self.cb_readbuffer(null, 1, NEWLINEBUFFER); + self.buffer = framework_utils.createBufferSize(0); + return; + } + if (self.docscount) { self.ondocuments(); self.docscount = 0; @@ -590,13 +595,23 @@ NoSQLStream.prototype.readreverse = function() { NoSQLStream.prototype.readreverse2 = function() { var self = this; + if (!self.fd || self.position <= 0 || self.canceled) { + + if (self.buffer.length && !self.canceled) { + self.cb_readreversebuffer(null, 1, NEWLINEBUFFER); + self.buffer = framework_utils.createBufferSize(0); + return; + } + if (self.docscount) { self.ondocuments(); self.docs = ''; self.docscount = 0; } + self.close(); + } else { var size = self.stats.size - self.bytesread; size = size < BUFFERSIZE ? size : BUFFERSIZE; @@ -607,10 +622,19 @@ NoSQLStream.prototype.readreverse2 = function() { }; NoSQLStream.prototype.readupdate = function() { + var self = this; var size = self.stats.size - self.position; + if (!self.fd || size <= 0 || self.canceled) { + if (self.buffer.length && !self.canceled) { + self.positionappend++; + self.cb_readwritebuffer(null, 1, NEWLINEBUFFER); + self.buffer = framework_utils.createBufferSize(0); + return; + } + if (self.docsbuffer.length) { self.ondocuments(); self.docsbuffer = []; From 122a2ad77e1090800000ac59a5e69fe9485ee89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 18:22:48 +0200 Subject: [PATCH 0270/1669] Fixed NoSQL streamer. --- nosqlstream.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosqlstream.js b/nosqlstream.js index 014600230..750f4f102 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -565,7 +565,7 @@ NoSQLStream.prototype.read = function() { if (!self.fd || size <= 0 || self.canceled) { - if (self.buffer.length && !self.canceled) { + if (!self.canceled && self.buffer && self.buffer.length) { self.cb_readbuffer(null, 1, NEWLINEBUFFER); self.buffer = framework_utils.createBufferSize(0); return; @@ -598,7 +598,7 @@ NoSQLStream.prototype.readreverse2 = function() { if (!self.fd || self.position <= 0 || self.canceled) { - if (self.buffer.length && !self.canceled) { + if (!self.canceled && self.buffer && self.buffer.length) { self.cb_readreversebuffer(null, 1, NEWLINEBUFFER); self.buffer = framework_utils.createBufferSize(0); return; @@ -628,7 +628,7 @@ NoSQLStream.prototype.readupdate = function() { if (!self.fd || size <= 0 || self.canceled) { - if (self.buffer.length && !self.canceled) { + if (!self.canceled && self.buffer && self.buffer.length) { self.positionappend++; self.cb_readwritebuffer(null, 1, NEWLINEBUFFER); self.buffer = framework_utils.createBufferSize(0); From a969d2ac3eba3a39a5b1bbf4d1394e5305702f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 20:04:22 +0200 Subject: [PATCH 0271/1669] Fixed animations. --- index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index df891f664..b8481b9cf 100755 --- a/index.js +++ b/index.js @@ -6423,10 +6423,11 @@ F.response501 = function(req, res, problem) { F.response503 = function(req, res) { var keys = ''; for (var m in F.waits) - keys += '' + m + '⧗'; + keys += '' + m + '⧗'; + res.options.code = 503; res.options.headers = HEADERS.response503; - res.options.body = 'Please wait
10
PLEASE WAIT
Application is starting …
' + keys + '
'; + res.options.body = 'Please wait
10
PLEASE WAIT
Application is starting …
' + keys + '
'; res.$throw(); return F; }; From 21387d140d43923cfb1dd81df7f11b406e4ffdec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 20:04:27 +0200 Subject: [PATCH 0272/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 14ae57f71..8c90e2f74 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-52", + "version": "3.0.0-53", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0bf7bcb20bc5123e2b9f6aa6b95e492ec9e4e58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Apr 2018 21:15:23 +0200 Subject: [PATCH 0273/1669] Fixed NoSQL logging. --- nosql.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 7575409f7..0bf6c1591 100755 --- a/nosql.js +++ b/nosql.js @@ -288,8 +288,11 @@ exports.worker = function() { builder.callback(function(err, d) { if (d) callback && callback(null, 0); - else - self.insert(doc).callback(callback); + else { + var tmp = self.insert(doc); + tmp.callback(callback); + tmp.$options.log = builder.$options.log; + } }); builder.callback = function(fn) { @@ -1461,7 +1464,9 @@ Database.prototype.$update = function() { var item = filter[i]; if (item.insert && !item.count) { item.builder.$insertcallback && item.builder.$insertcallback(item.insert); - self.insert(item.insert).$callback = item.builder.$callback; + var tmp = self.insert(item.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; } else { item.builder.$options.log && item.builder.log(); item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); @@ -1553,7 +1558,9 @@ Database.prototype.$update_inmemory = function() { var item = filter[i]; if (item.insert && !item.count) { item.builder.$insertcallback && item.builder.$insertcallback(item.insert); - self.insert(item.insert).$callback = item.builder.$callback; + var tmp = self.insert(item.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; } else { var e = item.keys ? 'modify' : 'update'; item.count && self.$events[e] && self.emit(e, item.doc); From 3b929d53b5b6115b36f1508c516c22d2b7abd55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Apr 2018 08:00:02 +0200 Subject: [PATCH 0274/1669] Added `@{root}` markup. --- changes.txt | 1 + internal.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index f65733fa2..ebe56dc14 100755 --- a/changes.txt +++ b/changes.txt @@ -54,6 +54,7 @@ - added: `versions` file supports `auto` value for generating auto-checksum of files - added: `F.load()` supports `test` - added: NoSQL binary supports `custom` small data attributes +- added: `@{root}` for getting sub-root path - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/internal.js b/internal.js index 8d5e75797..d6a5061d1 100755 --- a/internal.js +++ b/internal.js @@ -1917,7 +1917,7 @@ function view_parse(content, minify, filename, controller) { } else { counter--; - builder += '}return $output;})()'; + builder += '}return $output})()'; newCommand = ''; } @@ -2081,6 +2081,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { case '!isomorphic': return '$STRING(' + command + ')'; + case 'root': + var r = F.config['default-root']; + return '\'' + (r ? r.substring(0, r.length - 1) : r) + '\''; + case 'M': case 'R': case 'G': From 8a582ab02acd3f477b992edc79770b08f09aca95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Apr 2018 08:45:32 +0200 Subject: [PATCH 0275/1669] Added: CSS + JS a simple view engine markup. --- changes.txt | 1 + index.js | 2 +- internal.js | 75 +++++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/changes.txt b/changes.txt index ebe56dc14..f3f9e24b2 100755 --- a/changes.txt +++ b/changes.txt @@ -55,6 +55,7 @@ - added: `F.load()` supports `test` - added: NoSQL binary supports `custom` small data attributes - added: `@{root}` for getting sub-root path +- added: CSS and JS supports a simple View Engine markup (config + resources + F.global) - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index b8481b9cf..d018c5ae7 100755 --- a/index.js +++ b/index.js @@ -3077,7 +3077,7 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { if (err) return res.throw404(); - content = F.translator(req.$language, framework_internal.modificators(content.toString(ENCODING), filename, 'static')); + content = framework_internal.markup(F.translator(req.$language, framework_internal.modificators(content.toString(ENCODING), filename, 'static'))); Fs.lstat(filename, function(err, stats) { diff --git a/internal.js b/internal.js index d6a5061d1..23188165a 100755 --- a/internal.js +++ b/internal.js @@ -1245,6 +1245,9 @@ function minify_javascript(data) { exports.compile_css = function(value, filename) { + // Internal markup + value = markup(value); + if (global.F) { value = modificators(value, filename, 'style'); if (F.onCompileStyle) @@ -1255,10 +1258,7 @@ exports.compile_css = function(value, filename) { var isVariable = false; - value = nested(value, '', function() { - isVariable = true; - }); - + value = nested(value, '', () => isVariable = true); value = compile_autovendor(value); if (isVariable) @@ -1266,13 +1266,16 @@ exports.compile_css = function(value, filename) { return value; } catch (ex) { - F.error(new Error('CSS compiler exception: ' + ex.message)); + F.error(new Error('CSS compiler error: ' + ex.message)); return ''; } }; exports.compile_javascript = function(source, filename) { + // Internal markup + source = markup(source); + if (global.F) { source = modificators(source, filename, 'script'); if (F.onCompileScript) @@ -1973,7 +1976,7 @@ function view_parse(content, minify, filename, controller) { } } catch (e) { - console.log('VIEW EXCEPTION --->', filename, e, tmp); + console.log('A view compilation error --->', filename, e, tmp); F.errors.push({ error: e.stack, name: filename, url: null, date: new Date() }); if (view_parse_plus(builder)) @@ -2301,7 +2304,7 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'hidden': case 'textarea': case 'password': - return 'self.$' + exports.appendModel(command); + return 'self.$' + appendModel(command); default: return F.helpers[name] ? ('helpers.' + view_insert_call(command)) : ('$STRING(' + (functions.indexOf(name) === -1 ? command[0] === '!' ? command.substring(1) + ')' : command + ').encode()' : command + ')')); @@ -2387,7 +2390,7 @@ function view_is_assign(value) { return false; } -function view_find_command(content, index) { +function view_find_command(content, index, entire) { index = content.indexOf('@{', index); if (index === -1) @@ -2417,12 +2420,12 @@ function view_find_command(content, index) { if (command[0] === '{') return view_find_command(content, index + 1); - return { - beg: index, - end: i, - line: view_line_counter(content.substr(0, index)), - command: command - }; + var obj = { beg: index, end: i, line: view_line_counter(content.substr(0, index)), command: command }; + + if (entire) + obj.phrase = content.substring(index, i + 1); + + return obj; } return null; @@ -3022,13 +3025,13 @@ function viewengine_dynamic(content, language, controller, cachekey) { return generator; } -exports.appendModel = function(str) { +function appendModel(str) { var index = str.indexOf('('); if (index === -1) return str; var end = str.substring(index + 1); return str.substring(0, index) + '(model' + (end[0] === ')' ? end : ',' + end); -}; +} function cleanURL(url, index) { var o = url.substring(0, index); @@ -3287,6 +3290,43 @@ exports.restart = function() { INDEXFILE = 0; }; +function markup(body) { + var command = view_find_command(body, 0, true); + if (!command) + return body; + + var G = F.global; + var config = F.config; + var resource = F.resource; + var r = []; + + while (command) { + var cmd = command.command; + + switch (cmd) { + case 'author': + cmd = 'F.config.author'; + break; + case 'root': + cmd = 'F.config[\'default-root\']'; + break; + } + + try { + r.push({ cmd: command.phrase, value: eval('(' + cmd + ')') }); + } catch (e) { + console.log('A markup compilation error -->', cmd, e); + } + + command = view_find_command(body, command.end, true); + } + + for (var i = 0; i < r.length; i++) + body = body.replace(r[i].cmd, r[i].value); + + return body; +} + global.HttpFile = HttpFile; exports.HttpFile = HttpFile; exports.viewEngineCompile = viewengine_dynamic; @@ -3295,4 +3335,5 @@ exports.parseLocalization = view_parse_localization; exports.findLocalization = view_find_localization; exports.destroyStream = destroyStream; exports.onFinished = onFinished; -exports.modificators = modificators; \ No newline at end of file +exports.modificators = modificators; +exports.markup = markup; \ No newline at end of file From daa7ae7be7056aec7c62bc9ad9e0e6d74f16a82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Apr 2018 08:59:15 +0200 Subject: [PATCH 0276/1669] Fixed markup. --- internal.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/internal.js b/internal.js index 23188165a..e491b46c3 100755 --- a/internal.js +++ b/internal.js @@ -1243,10 +1243,11 @@ function minify_javascript(data) { return output.join('').trim(); } -exports.compile_css = function(value, filename) { +exports.compile_css = function(value, filename, nomarkup) { // Internal markup - value = markup(value); + if (!nomarkup) + value = markup(value); if (global.F) { value = modificators(value, filename, 'style'); @@ -1271,10 +1272,11 @@ exports.compile_css = function(value, filename) { } }; -exports.compile_javascript = function(source, filename) { +exports.compile_javascript = function(source, filename, nomarkup) { // Internal markup - source = markup(source); + if (!nomarkup) + source = markup(source); if (global.F) { source = modificators(source, filename, 'script'); @@ -1722,10 +1724,10 @@ function view_parse(content, minify, filename, controller) { }).trim(); if (!nocompressJS) - content = compressJS(content, 0, filename); + content = compressJS(content, 0, filename, true); if (!nocompressCSS) - content = compressCSS(content, 0, filename); + content = compressCSS(content, 0, filename, true); content = F.$versionprepare(content); @@ -2524,7 +2526,7 @@ function compressView(html, minify) { * @param {Number} index Last index. * @return {String} */ -function compressJS(html, index, filename) { +function compressJS(html, index, filename, nomarkup) { if (!F.config['allow-compile-script']) return html; @@ -2550,12 +2552,12 @@ function compressJS(html, index, filename) { return html; var val = js.substring(strFrom.length, js.length - strTo.length).trim(); - var compiled = exports.compile_javascript(val, filename); + var compiled = exports.compile_javascript(val, filename, nomarkup); html = html.replacer(js, strFrom + compiled.trim() + strTo.trim()); - return compressJS(html, indexBeg + compiled.length + 9, filename); + return compressJS(html, indexBeg + compiled.length + 9, filename, nomarkup); } -function compressCSS(html, index, filename) { +function compressCSS(html, index, filename, nomarkup) { if (!F.config['allow-compile-style']) return html; @@ -2577,9 +2579,9 @@ function compressCSS(html, index, filename) { var css = html.substring(indexBeg, indexEnd + strTo.length); var val = css.substring(strFrom.length, css.length - strTo.length).trim(); - var compiled = exports.compile_css(val, filename); + var compiled = exports.compile_css(val, filename, nomarkup); html = html.replacer(css, (strFrom + compiled.trim() + strTo).trim()); - return compressCSS(html, indexBeg + compiled.length + 8, filename); + return compressCSS(html, indexBeg + compiled.length + 8, filename, nomarkup); } function variablesCSS(content) { From bf2e1f89013974403aa03ae583c8f26b3aa797e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Apr 2018 10:32:36 +0200 Subject: [PATCH 0277/1669] Improved NoSQL boolean types. --- nosql.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 0bf6c1591..a1dfd35c8 100755 --- a/nosql.js +++ b/nosql.js @@ -60,6 +60,8 @@ const FLAGS_READ = ['get']; const COUNTER_MMA = [0, 0]; const REGNUMBER = /^\d+$/; const REGINDEXCHAR = /[a-z]{1,2}/; +const REGBOOL = /":true/; // for updates of boolean types +const JSONBOOL = '":true '; const DIRECTORYLENGTH = 9; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; @@ -715,7 +717,7 @@ Database.prototype.insert = function(doc, unique) { builder = new DatabaseBuilder2(self); var json = framework_builders.isSchema(doc) ? doc.$clean() : doc; - self.pending_append.push({ doc: JSON.stringify(json), raw: doc, builder: builder }); + self.pending_append.push({ doc: JSON.stringify(json).replace(REGBOOL, JSONBOOL), raw: doc, builder: builder }); setImmediate(next_operation, self, 1); self.$events.insert && self.emit('insert', json); return builder; @@ -1201,7 +1203,7 @@ Database.prototype.$save = function(view) { var data = self.inmemory[view] || EMPTYARRAY; var builder = []; for (var i = 0, length = data.length; i < length; i++) - builder.push(JSON.stringify(data[i])); + builder.push(JSON.stringify(data[i]).replace(REGBOOL, JSONBOOL)); var filename = self.filename; if (view !== '#') @@ -1429,7 +1431,7 @@ Database.prototype.$update = function() { } } - var upd = JSON.stringify(doc); + var upd = JSON.stringify(doc).replace(REGBOOL, JSONBOOL); if (upd === rec.doc) continue; @@ -2331,7 +2333,7 @@ Database.prototype.$views = function() { item.response.limit(20, function(items, next) { var builder = []; for (var i = 0, length = items.length; i < length; i++) - builder.push(JSON.stringify(items[i])); + builder.push(JSON.stringify(items[i]).replace(REGBOOL, JSONBOOL)); Fs.appendFile(filename, builder.join(NEWLINE) + NEWLINE, next); }, function() { // clears in-memory From 12fec6c696e688ee46c9b883dc00beaf8a3baa3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Apr 2018 11:59:28 +0200 Subject: [PATCH 0278/1669] Added `controller.split`. --- changes.txt | 1 + index.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/changes.txt b/changes.txt index f3f9e24b2..fdac5e4bb 100755 --- a/changes.txt +++ b/changes.txt @@ -56,6 +56,7 @@ - added: NoSQL binary supports `custom` small data attributes - added: `@{root}` for getting sub-root path - added: CSS and JS supports a simple View Engine markup (config + resources + F.global) +- added: `controller.split` alias to `controller.req.split` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index d018c5ae7..f322ada97 100755 --- a/index.js +++ b/index.js @@ -9833,6 +9833,10 @@ Controller.prototype = { return this.route.options; }, + get split() { + return this.req.split; + }, + get flags() { return this.route.flags; }, From 413b565541ff99e34a4cd17658ea3fd021fc7b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Apr 2018 12:10:57 +0200 Subject: [PATCH 0279/1669] Improved joins. --- nosql.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index a1dfd35c8..6fd3a6c7c 100755 --- a/nosql.js +++ b/nosql.js @@ -2850,14 +2850,18 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { })(unique[i]); } - } else { - join.builder.$options.fields && join.builder.$options.fields.push(join.a); - join.builder.$callback = function(err, docs) { - join.items = docs; + if (unique.length) { + join.builder.$options.fields && join.builder.$options.fields.push(join.a); + join.builder.$callback = function(err, docs) { + join.items = docs; + next(); + }; + db.find(join.view, join.builder).in(join.a, unique); + } else { + join.items = join.builder.$options.first ? null : EMPTYARRAY; next(); - }; - db.find(join.view, join.builder).in(join.a, unique); + } } }, callback); From 38191aa5bba1f6bcd1e9cbd9f36d2a00f7a09739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 1 May 2018 20:53:10 +0200 Subject: [PATCH 0280/1669] Improved NoSQL builder `.fields()`. --- nosql.js | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/nosql.js b/nosql.js index 6fd3a6c7c..6187e8b3c 100755 --- a/nosql.js +++ b/nosql.js @@ -2761,16 +2761,9 @@ function DatabaseBuilder(db) { this.$take = 0; this.$skip = 0; this.$filter = []; - // this.$sort; this.$first = false; this.$scope = 0; - // this.$fields; - // this.$join; this.$callback = NOOP; - // this.$scalar; - // this.$scalarfield; - // this.$done; --> internal for indexes - this.$code = []; this.$params = {}; this.$options = {}; @@ -3296,7 +3289,7 @@ DatabaseBuilder.prototype.repository = function(key, value) { DatabaseBuilder.prototype.compile = function() { var self = this; var raw = self.$code.join(''); - var code = 'var repository=$F.repository;var options=$F.options;var arg=$F.arg;var fn=$F.fn;var $is=false;var $tmp;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + 'if(!options.fields)return doc;var $doc={};for(var $i=0;$i Date: Wed, 2 May 2018 21:48:38 +0200 Subject: [PATCH 0281/1669] Improved JavaScript minificator. --- internal.js | 50 ++++++++++++++++++++++++++++++++++------- test/test-javascript.js | 2 +- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/internal.js b/internal.js index e491b46c3..01d6e9afd 100755 --- a/internal.js +++ b/internal.js @@ -1146,11 +1146,12 @@ function minify_javascript(data) { var alpha = /[0-9a-z$]/i; var white = /\W/; var skip = { '$': true, '_': true }; + var newlines = { '\n': 1, '\r': 1 }; var regexp = false; - var scope; - var prev; - var next; - var last; + var scope, prev, next, last; + var vtmp = false; + var regvar = /^(\s)*var /; + var vindex = 0; while (true) { @@ -1181,7 +1182,7 @@ function minify_javascript(data) { if (c === '/' && next === '/') { isCI = true; continue; - } else if (isCI && (c === '\n' || c === '\r')) { + } else if (isCI && newlines[c]) { isCI = false; alpha.test(last) && output.push(' '); last = ''; @@ -1192,7 +1193,7 @@ function minify_javascript(data) { continue; } - if (c === '\t' || c === '\n' || c === '\r') { + if (c === '\t' || newlines[c]) { if (!last || !alpha.test(last)) continue; output.push(' '); @@ -1201,8 +1202,11 @@ function minify_javascript(data) { } if (!regexp && (c === ' ' && (white.test(prev) || white.test(next)))) { - if (!skip[prev] && !skip[next]) - continue; + // if (!skip[prev] && !skip[next]) + if (!skip[prev]) { + if (!skip[next] || !alpha.test(prev)) + continue; + } } if (regexp) { @@ -1233,6 +1237,36 @@ function minify_javascript(data) { scope = c; } + // var + if (!scope && c === 'v' && next === 'a') { + var v = c + data[index] + data[index + 1] + data[index + 2]; + if (v === 'var ') { + if (vtmp && output[output.length - 1] === ';') { + output.pop(); + output.push(','); + } else + output.push('var '); + index += 3; + vtmp = true; + continue; + } + } else { + if (vtmp) { + + + vindex = index + 1; + + while (true) { + if (!data[vindex] || !white.test(data[vindex])) + break; + vindex++; + } + + if (c === '(' || c === ')' || (c === ';' && !regvar.test(data.substring(vindex, vindex + 20)))) + vtmp = false; + } + } + if ((c === '}' && last === ';') || ((c === '}' || c === ']') && output[output.length - 1] === ' ' && alpha.test(output[output.length - 2]))) output.pop(); diff --git a/test/test-javascript.js b/test/test-javascript.js index e3c516052..8daa874b8 100755 --- a/test/test-javascript.js +++ b/test/test-javascript.js @@ -12,7 +12,7 @@ buffer.push('console.log("OK");'); buffer.push(''); -var result1 = ''; +var result1 = ''; assert.ok(javascript.compile_javascript(buffer.join('\n')) === result1, 'javascript'); assert.ok(Buffer.from(javascript.compile_javascript(fs.readFileSync('javascript.js').toString('utf8'))).toString('base64') === 'cmV0dXJuJ1xcJysyO3ZhciBhdHRyaWJ1dGVzPSJcXFsiK2ErIiooIitiKyIpKD86IitjKyIqKFsqXiR8IX5dPz0pIitkKyIqKD86JygoPzpcXFxcLnxbXlxcXFwnXSkqKSd8XCIoKD86XFxcXC58W15cXFxcXCJdKSopXCJ8KCIrZSsiKSl8KSIrZisiKlxcXSI7dmFyIGE9MjAwOw==', 'Problem 1'); From c029dc7ad8fa6475251c2a7cdec08da967082a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 2 May 2018 21:55:05 +0200 Subject: [PATCH 0282/1669] Fixed unit-test. --- test/controllers/default.js | 4 ++-- test/test-framework-debug.js | 11 +++++------ test/test-framework-release.js | 29 +++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/test/controllers/default.js b/test/controllers/default.js index 4252d3aa5..847a63522 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -549,8 +549,8 @@ function viewViews() { assert.ok(output.contains('
4
4
FOREACH
'), name + 'foreach'); assert.ok(output.contains('
3
3
C:10
C:11
C:12
'), name + 'foreach - nested'); assert.ok(output.contains('5'), name + 'Inline assign value'); - assert.ok(output.contains('var d="$\'"'), name + 'JS script special chars 1'); - assert.ok(output.contains("var e='$\\'';"), name + "JS script special chars 2"); + assert.ok(output.contains(',d="$\'"'), name + 'JS script special chars 1'); + assert.ok(output.contains(",e='$\\'',"), name + "JS script special chars 2"); assert.ok(output.contains('') || output.contains(''), name + ' minify html'); assert.ok(output.contains('#tag-encode<b>A</b>#'), name + 'encode value'); assert.ok(output.contains('#tag-rawA#'), name + 'raw value'); diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 0ee9861ee..92acd48e5 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -79,7 +79,6 @@ function test_view_functions(next) { if (error) assert.ok(false, 'test_view_functions: ' + error.toString()); - assert.ok(data === '{"r":true}', 'json'); next(); }); @@ -754,7 +753,7 @@ function test_routing(next) { utils.request(url + 'merge-blocks-a.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var a=true;') !== -1, 'merge blocks - A'); + assert(data.indexOf('var common=true,a=true;') !== -1, 'merge blocks - A'); complete(); }); }); @@ -763,7 +762,7 @@ function test_routing(next) { utils.request(url + 'merge-blocks-b.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var b=true;') !== -1, 'merge blocks - B'); + assert(data.indexOf('var common=true,b=true;') !== -1, 'merge blocks - B'); complete(); }); }); @@ -772,7 +771,7 @@ function test_routing(next) { utils.request(url + 'blocks-a.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var a=true;') !== -1, 'mapping blocks - A'); + assert(data.indexOf('var common=true,a=true;') !== -1, 'mapping blocks - A'); complete(); }); }); @@ -781,7 +780,7 @@ function test_routing(next) { utils.request(url + 'blocks-b.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var b=true;') !== -1, 'mapping blocks - B'); + assert(data.indexOf('var common=true,b=true;') !== -1, 'mapping blocks - B'); complete(); }); }); @@ -790,7 +789,7 @@ function test_routing(next) { utils.request(url + 'blocks-c.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var a=true;var b=true;') !== -1, 'mapping blocks - C'); + assert(data.indexOf('var common=true,a=true,b=true;') !== -1, 'mapping blocks - C'); complete(); }); }); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 0ec9e3c8e..38fd04ea8 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -79,7 +79,6 @@ function test_view_functions(next) { if (error) assert.ok(false, 'test_view_functions: ' + error.toString()); - assert.ok(data === '{"r":true}', 'json'); next(); }); @@ -397,6 +396,24 @@ function test_routing(next) { }); }); + async.await('prefix -- 1', function(complete) { + utils.request(url + 'prefix1/test/', ['get'], function(error, data) { + if (error) + throw error; + assert.ok(data === 'PREFIX1TEST', 'Group + Prefix 1'); + complete(); + }); + }); + + async.await('prefix -- 2', function(complete) { + utils.request(url + 'prefix2/test/', ['get'], function(error, data) { + if (error) + throw error; + assert.ok(data === 'PREFIX2TEST', 'Group + Prefix 2'); + complete(); + }); + }); + async.await('package/', function(complete) { utils.request(url + 'package/', ['get'], function(error, data, code, headers) { if (error) @@ -736,7 +753,7 @@ function test_routing(next) { utils.request(url + 'merge-blocks-a.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var a=true;') !== -1, 'merge blocks - A'); + assert(data.indexOf('var common=true,a=true;') !== -1, 'merge blocks - A'); complete(); }); }); @@ -745,7 +762,7 @@ function test_routing(next) { utils.request(url + 'merge-blocks-b.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var b=true;') !== -1, 'merge blocks - B'); + assert(data.indexOf('var common=true,b=true;') !== -1, 'merge blocks - B'); complete(); }); }); @@ -754,7 +771,7 @@ function test_routing(next) { utils.request(url + 'blocks-a.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var a=true;') !== -1, 'mapping blocks - A'); + assert(data.indexOf('var common=true,a=true;') !== -1, 'mapping blocks - A'); complete(); }); }); @@ -763,7 +780,7 @@ function test_routing(next) { utils.request(url + 'blocks-b.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var b=true;') !== -1, 'mapping blocks - B'); + assert(data.indexOf('var common=true,b=true;') !== -1, 'mapping blocks - B'); complete(); }); }); @@ -772,7 +789,7 @@ function test_routing(next) { utils.request(url + 'blocks-c.js', [], function(error, data, code, headers) { if (error) throw error; - assert(data.indexOf('var common=true;var a=true;var b=true;') !== -1, 'mapping blocks - C'); + assert(data.indexOf('var common=true,a=true,b=true;') !== -1, 'mapping blocks - C'); complete(); }); }); From c90a8e1458534baa3b09229226b35a63f5a2f806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 2 May 2018 21:56:31 +0200 Subject: [PATCH 0283/1669] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index fdac5e4bb..a751d9255 100755 --- a/changes.txt +++ b/changes.txt @@ -73,6 +73,7 @@ - updated: `String.parseConfig(def, [onError])` can handle errors better - updated: `middleware`, now Total.js supports new declaration `F.middleware(function($) {})` - updated: `F.wait()` HTML template +- updated: JavaScript compressor, now optimizes multiple `var` declarations - fixed: mail attachments - fixed: mail `message.manually()` From b1abd4e0c15dd1f35f8e986de5289e8a50800c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 2 May 2018 22:09:37 +0200 Subject: [PATCH 0284/1669] Fixed auto-parsing types from `config`. --- index.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index f322ada97..0496f04af 100755 --- a/index.js +++ b/index.js @@ -8788,8 +8788,14 @@ F.$configure_configs = function(arr, rewrite) { obj[name] = value.parseDate(); else if (subtype === 'env' || subtype === 'environment') obj[name] = process.env[value]; - else - obj[name] = value.isNumber() ? U.parseInt(value) : value.isNumber(true) ? U.parseFloat(value) : value.isBoolean() ? value.toLowerCase() === 'true' : value; + else { + if (value.isNumber()) { + obj[name] = value[0] !== '0' ? U.parseInt(value) : value; + } else if (value.isNumber(true)) + obj[name] = value.indexOf(',') === -1 && !(/^0{2,}/).test(value) ? U.parseFloat(value) : value; + else + obj[name] = value.isBoolean() ? value.toLowerCase() === 'true' : value; + } break; } } From 3dccad582daa4017648e1d92cd3fcf39b9144fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 3 May 2018 09:24:01 +0200 Subject: [PATCH 0285/1669] Added `unhandledRejection` event. --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 0496f04af..dd4804bbd 100755 --- a/index.js +++ b/index.js @@ -16135,6 +16135,10 @@ function $decodeURIComponent(value) { global.Controller = Controller; +process.on('unhandledRejection', function(e) { + F.error(e, '', null); +}); + process.on('uncaughtException', function(e) { var err = e.toString(); From 8b77ca662041b1ba711da1aa464bd3b059c8e693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 3 May 2018 10:40:55 +0200 Subject: [PATCH 0286/1669] Fixed schema validation. --- utils.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/utils.js b/utils.js index d98816c9d..768dafbb6 100755 --- a/utils.js +++ b/utils.js @@ -2096,7 +2096,21 @@ exports.validate_builder = function(model, error, schema, collection, path, inde if (TYPE.type === 7) { // Another schema - exports.validate_builder(value, error, TYPE.raw, collection, current + name, undefined, undefined, pluspath); + result = TYPE.validate ? TYPE.validate(value, model) : null; + if (result == null) + exports.validate_builder(value, error, TYPE.raw, collection, current + name, undefined, undefined, pluspath); + else { + type = typeof(result); + if (type === 'string') { + if (result[0] === '@') + error.push(pluspath + name, '@', current + name, index, entity.resourcePrefix + result.substring(1)); + else + error.push(pluspath + name, result, current + name, index, prefix); + } else if (type === 'boolean') { + !result && error.push(pluspath + name, '@', current + name, index, prefix); + } else if (result.isValid === false) + error.push(pluspath + name, result.error, current + name, index, prefix); + } continue; } From 4367c3ec4acc331d60a1ee46508b63f01ca760db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 5 May 2018 20:40:17 +0200 Subject: [PATCH 0287/1669] Added nicer error response messages. --- changes.txt | 1 + error.html | 41 +++++++++++++++++++++++++++++++++++++++++ index.js | 24 +++++++++++++++++++++--- 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 error.html diff --git a/changes.txt b/changes.txt index a751d9255..30a5f058f 100755 --- a/changes.txt +++ b/changes.txt @@ -57,6 +57,7 @@ - added: `@{root}` for getting sub-root path - added: CSS and JS supports a simple View Engine markup (config + resources + F.global) - added: `controller.split` alias to `controller.req.split` +- added: nicer error response messages - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/error.html b/error.html new file mode 100644 index 000000000..ff5e166a5 --- /dev/null +++ b/error.html @@ -0,0 +1,41 @@ + + + + @{model.code}: @{model.status} + + + + + + + +
+
+
+ @{model.code} +
@{model.status}
+ @{if model.error}
@{model.error}
@{fi} +
+
+
+ + \ No newline at end of file diff --git a/index.js b/index.js index dd4804bbd..71f25b5f2 100755 --- a/index.js +++ b/index.js @@ -91,6 +91,7 @@ const CLUSTER_CACHE_REMOVEALL = { TYPE: 'cache-remove-all' }; const CLUSTER_CACHE_CLEAR = { TYPE: 'cache-clear' }; const GZIPFILE = { memLevel: 9 }; const GZIPSTREAM = { memLevel: 1 }; +const MODELERROR = {}; Object.freeze(EMPTYOBJECT); Object.freeze(EMPTYARRAY); @@ -850,9 +851,11 @@ function Framework() { error403: 0, error404: 0, error408: 0, + error409: 0, error431: 0, error500: 0, - error501: 0 + error501: 0, + error503: 0 } }; @@ -12318,6 +12321,15 @@ Controller.prototype.throw501 = Controller.prototype.view501 = function(problem) return controller_error_status(this, 501, problem); }; +/** + * Throw 501 - Not implemented + * @param {String} problem Description of the problem (optional) + * @return {Controller} + */ +Controller.prototype.throw503 = Controller.prototype.view501 = function(problem) { + return controller_error_status(this, 503, problem); +}; + /** * Creates a redirect * @param {String} url @@ -14522,8 +14534,14 @@ function extend_request(PROTO) { res.options.type = this.$total_exception.contentType; res.$text(); } else { - res.options.body = U.httpStatus(status) + prepare_error(this.$total_exception); - res.options.type = CT_TEXT; + + MODELERROR.code = status; + MODELERROR.status = U.httpStatus(status, false); + MODELERROR.error = this.$total_exception ? prepare_error(this.$total_exception) : null; + + var path = require.resolve('./index'); + res.options.body = F.view('.' + path.substring(0, path.length - 8) + 'error', MODELERROR); + res.options.type = CT_HTML; res.options.code = status || 404; res.$text(); } From 7c8dc92dc8f183ee5a3091e8715f4c67066ef603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 5 May 2018 20:55:16 +0200 Subject: [PATCH 0288/1669] Improved `error` messages. --- error.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/error.html b/error.html index ff5e166a5..53decb30f 100644 --- a/error.html +++ b/error.html @@ -14,6 +14,7 @@ .cell { display: table-cell; vertical-align: middle; text-align: center; width: 100%; height: 100%; font-size: 40px; } .cell b { font-size: 60px; background-color: black; color: white; position: relative; display: inline-block; padding: 5px 10px; border-radius: 2px; margin-bottom: 5px; animation: anim 0.5s forwards 2s; } .error { font-size: 11px; color: gray; width: 90%; max-width: 800px; margin: 20px auto; background-color: #F0F0F0; padding: 10px; border-radius: 2px; text-align: left; font-family: monospace; overflow: auto; } + #url { font-size: 12px; color: gray; } @keyframes anim { 0% { transform: rotate(0deg); } @@ -33,9 +34,15 @@
@{model.code}
@{model.status}
+
@{if model.error}
@{model.error}
@{fi}
+ + + \ No newline at end of file From c729b562f44b2b91b4022225a9b3eaf081774086 Mon Sep 17 00:00:00 2001 From: Martin Smola Date: Sun, 6 May 2018 08:57:50 +0200 Subject: [PATCH 0289/1669] Fixed typo --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 71f25b5f2..63cd49361 100755 --- a/index.js +++ b/index.js @@ -12322,11 +12322,11 @@ Controller.prototype.throw501 = Controller.prototype.view501 = function(problem) }; /** - * Throw 501 - Not implemented + * Throw 503 - Service unavailable * @param {String} problem Description of the problem (optional) * @return {Controller} */ -Controller.prototype.throw503 = Controller.prototype.view501 = function(problem) { +Controller.prototype.throw503 = Controller.prototype.view503 = function(problem) { return controller_error_status(this, 503, problem); }; From 2f897eae8ad09570f9f957146f24b6929d6add7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 6 May 2018 10:36:54 +0200 Subject: [PATCH 0290/1669] Improved `F.wait()` response message. --- 503.html | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ index.js | 12 +++++------ 2 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 503.html diff --git a/503.html b/503.html new file mode 100644 index 000000000..45ffc1e30 --- /dev/null +++ b/503.html @@ -0,0 +1,66 @@ + + + + @(Please wait) + + + + + + + + +
+
+
10
+
@(Please wait)
+
+ @(Application is starting …) +
+ + @{foreach m in model} + + + + + @{end} +
@{m.key}
+
+
+ + + + + \ No newline at end of file diff --git a/index.js b/index.js index 71f25b5f2..410378876 100755 --- a/index.js +++ b/index.js @@ -93,6 +93,9 @@ const GZIPFILE = { memLevel: 9 }; const GZIPSTREAM = { memLevel: 1 }; const MODELERROR = {}; +var PATHMODULES = require.resolve('./index'); +PATHMODULES = PATHMODULES.substring(0, PATHMODULES.length - 8); + Object.freeze(EMPTYOBJECT); Object.freeze(EMPTYARRAY); Object.freeze(EMPTYREQUEST); @@ -6424,13 +6427,9 @@ F.response501 = function(req, res, problem) { }; F.response503 = function(req, res) { - var keys = ''; - for (var m in F.waits) - keys += '' + m + '⧗'; - res.options.code = 503; res.options.headers = HEADERS.response503; - res.options.body = 'Please wait
10
PLEASE WAIT
Application is starting …
' + keys + '
'; + res.options.body = F.view('.' + PATHMODULES + '503', F.waits); res.$throw(); return F; }; @@ -14539,8 +14538,7 @@ function extend_request(PROTO) { MODELERROR.status = U.httpStatus(status, false); MODELERROR.error = this.$total_exception ? prepare_error(this.$total_exception) : null; - var path = require.resolve('./index'); - res.options.body = F.view('.' + path.substring(0, path.length - 8) + 'error', MODELERROR); + res.options.body = F.view('.' + PATHMODULES + 'error', MODELERROR); res.options.type = CT_HTML; res.options.code = status || 404; res.$text(); From 125d7963811ca1cbefaa97218fe911000ef4203f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 6 May 2018 10:36:58 +0200 Subject: [PATCH 0291/1669] Improved code --- internal.js | 1 + 1 file changed, 1 insertion(+) diff --git a/internal.js b/internal.js index 01d6e9afd..64b2f7c5f 100755 --- a/internal.js +++ b/internal.js @@ -3035,6 +3035,7 @@ function viewengine_load(name, filename, controller) { filename += '.html'; var key = 'view#' + filename + (controller.language || ''); + var generator = F.temporary.views[key]; if (generator) return generator; From 93ef7f3cfcd6f0358125458a12cc6b59785f1e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 6 May 2018 21:03:22 +0200 Subject: [PATCH 0292/1669] Improved `NoSQL.one()`. --- nosql.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 6187e8b3c..fb5e977af 100755 --- a/nosql.js +++ b/nosql.js @@ -1665,7 +1665,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { for (var i = 0; i < length; i++) { var fil = filter[i]; - if (!fil.builder.$options.first) + if (!fil.builder.$options.first || fil.builder.$options.sort) first = false; fil.scalarcount = 0; fil.compare = fil.builder.compile(); @@ -2557,6 +2557,7 @@ Database.prototype.$clean = function() { var length = filter.length; var now = Date.now(); + F.databasescleaner[self.name] = undefined; F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" cleaning (beg)'.format(self.name)); var fs = new NoSQLStream(self.filename); @@ -4944,7 +4945,8 @@ Binary.prototype.read = function(id, callback) { meta = JSON.parse(json, jsonparser); callback(null, stream, meta); } catch (e) { - F.error(e, 'nosql.binary.read', filename); + F.error(e, 'nosql.binary.read', filename + ' --> ' + json); + console.log(e, filename, json, BINARYREADDATA); callback(e); } CLEANUP(stream); From 9912844e5df84c01426dbcebee22955b01000be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 7 May 2018 11:48:21 +0200 Subject: [PATCH 0293/1669] Improved schema validation. --- builders.js | 2 +- package.json | 2 +- utils.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builders.js b/builders.js index c9da50f89..45790fd35 100755 --- a/builders.js +++ b/builders.js @@ -1398,7 +1398,7 @@ SchemaBuilderEntity.prototype.validate = function(model, resourcePrefix, resourc else path = ''; - framework_utils.validate_builder.call(self, model, builder, self.name, self.parent.collection, self.name, index, filter, path); + framework_utils.validate_builder.call(self, model, builder, self.name, self.parent.collection, '', index, filter, path); return builder; }; diff --git a/package.json b/package.json index 8c90e2f74..c4690394a 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-53", + "version": "3.0.0-54", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/utils.js b/utils.js index 768dafbb6..76d78af98 100755 --- a/utils.js +++ b/utils.js @@ -2036,7 +2036,7 @@ exports.validate_builder = function(model, error, schema, collection, path, inde var entity = collection[schema]; var prepare = entity.onValidate || F.onValidate || NOOP; - var current = path === undefined ? '' : path + '.'; + var current = path ? path + '.' : ''; var properties = entity.properties; var result; From 6eef352fd01193f371447e8bdd6d39838a087039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 7 May 2018 17:55:04 +0200 Subject: [PATCH 0294/1669] Fixed `.readreverse()`. --- nosqlstream.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/nosqlstream.js b/nosqlstream.js index 750f4f102..d44562219 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -175,19 +175,6 @@ NoSQLStream.prototype.readhelpers = function() { break; } - if (self.position === 0 && self.buffer.length) { - var tmp = self.buffer.toString('utf8'); - if (tmp[0] !== '"') { - self.docs += (self.docs ? self.divider : '') + tmp; - self.docscount++; - if (self.docscount >= BUFFERDOCS) { - self.ondocuments(); - self.docs = ''; - self.docscount = 0; - } - } - } - self.ticks++; if (self.ticks % 5 === 0) From b12dd1b02f546a274c0d0c5a6258469b9eb27ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 9 May 2018 17:13:33 +0200 Subject: [PATCH 0295/1669] Fixed `500` error messages. --- error.html | 2 +- index.js | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/error.html b/error.html index 53decb30f..71b8f4a95 100644 --- a/error.html +++ b/error.html @@ -35,7 +35,7 @@ @{model.code}
@{model.status}
- @{if model.error}
@{model.error}
@{fi} + @{if model.error}
@{model.error.replace(/\n/g, '
')}
@{fi} diff --git a/index.js b/index.js index ed0b08690..9c61bc4cd 100755 --- a/index.js +++ b/index.js @@ -16313,10 +16313,9 @@ function prepare_error(e) { if (!e) return ''; else if (e instanceof ErrorBuilder) - return ' :: ' + e.plain(); - else if (e.stack) - return RELEASE ? '' : e.stack; - return RELEASE ? '' : ' :: ' + e.toString(); + return e.plain(); + else if (DEBUG) + return e.stack ? e.stack : e.toString(); } function prepare_filename(name) { From 778411251906134004889685c411717896fa33c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 9 May 2018 19:37:29 +0200 Subject: [PATCH 0296/1669] Updated tests. --- test/test-builders.js | 6 +++--- test/test-framework-debug.js | 4 ++-- test/test-framework-release.js | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test-builders.js b/test/test-builders.js index e0d1413cc..a32b6e253 100755 --- a/test/test-builders.js +++ b/test/test-builders.js @@ -446,9 +446,9 @@ function test_Schema() { }); schema.make({ enum_int: '5', 'keyvalue': 'luciaa', enum_string: 'Širkaa' }, function(err) { - assert.ok(err.items[0].path === 'EnumKeyValue.enum_int', 'Schema enums (int) 2'); - assert.ok(err.items[1].path === 'EnumKeyValue.enum_string', 'Schema enums (string) 2'); - assert.ok(err.items[2].path === 'EnumKeyValue.keyvalue', 'Schema keyvalue 2'); + assert.ok(err.items[0].path === 'enum_int', 'Schema enums (int) 2'); + assert.ok(err.items[1].path === 'enum_string', 'Schema enums (string) 2'); + assert.ok(err.items[2].path === 'keyvalue', 'Schema keyvalue 2'); }); }); diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 92acd48e5..a7b40761c 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -496,7 +496,7 @@ function test_routing(next) { utils.request(url + 'schema-filter/', ['post'], 'EMPTY', function(error, data, code, headers) { if (error) throw error; - assert(data === '[{"name":"age","error":"The field \\"age\\" is invalid.","path":"filter.age","prefix":"age"}]', 'schema filter'); + assert(data === '[{"name":"age","error":"The field \\"age\\" is invalid.","path":"age","prefix":"age"}]', 'schema filter'); complete(); }); }); @@ -514,7 +514,7 @@ function test_routing(next) { utils.request(url + 'post/schema/', ['post'], 'age=Peter123456789012345678901234567890#', function(error, data, code, headers) { if (error) throw error; - assert(data === '[{"name":"name","error":"default","path":"User.name","prefix":"name"}]', 'post-schema 2'); + assert(data === '[{"name":"name","error":"default","path":"name","prefix":"name"}]', 'post-schema 2'); complete(); }); }); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 38fd04ea8..c0c9ec8d8 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -496,7 +496,7 @@ function test_routing(next) { utils.request(url + 'schema-filter/', ['post'], 'EMPTY', function(error, data, code, headers) { if (error) throw error; - assert(data === '[{"name":"age","error":"The field \\"age\\" is invalid.","path":"filter.age","prefix":"age"}]', 'schema filter'); + assert(data === '[{"name":"age","error":"The field \\"age\\" is invalid.","path":"age","prefix":"age"}]', 'schema filter'); complete(); }); }); @@ -514,7 +514,7 @@ function test_routing(next) { utils.request(url + 'post/schema/', ['post'], 'age=Peter123456789012345678901234567890#', function(error, data, code, headers) { if (error) throw error; - assert(data === '[{"name":"name","error":"default","path":"User.name","prefix":"name"}]', 'post-schema 2'); + assert(data === '[{"name":"name","error":"default","path":"name","prefix":"name"}]', 'post-schema 2'); complete(); }); }); From c8b41efa5760e9e25f0b59f4bf8858a57864a0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 10 May 2018 10:43:44 +0200 Subject: [PATCH 0297/1669] Added better mail error handling. --- mail.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mail.js b/mail.js index b08c936da..0130d625a 100755 --- a/mail.js +++ b/mail.js @@ -510,15 +510,18 @@ Mailer.prototype.send = function(smtp, options, messages, callback) { obj.host = smtp.substring(smtp.lastIndexOf('.', smtp.lastIndexOf('.') - 1) + 1); obj.socket.on('error', function(err) { mailer.destroy(obj); + var is = obj.callback ? true : false; obj.callback && obj.callback(err); obj.callback = null; if (obj.try || err.stack.indexOf('ECONNRESET') !== -1) return; + !obj.try && !is && F.error(err, 'mail-smtp', smtp); mailer.$events.error && mailer.emit('error', err, obj); }); obj.socket.on('clientError', function(err) { mailer.destroy(obj); + !obj.try && !obj.callback && F.error(err, 'mail-smtp', smtp); obj.callback && obj.callback(err); obj.callback = null; mailer.$events.error && !obj.try && mailer.emit('error', err, obj); @@ -527,6 +530,7 @@ Mailer.prototype.send = function(smtp, options, messages, callback) { obj.socket.setTimeout(options.timeout || 8000, function() { var err = new Error(framework_utils.httpStatus(408)); mailer.destroy(obj); + !obj.try && !obj.callback && F.error(err, 'mail-smtp', smtp); obj.callback && obj.callback(err); obj.callback = null; mailer.$events.error && !obj.try && mailer.emit('error', err, obj); From e5e55e6180419bd66eea597e56ced296cab0749b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 10 May 2018 15:25:53 +0200 Subject: [PATCH 0298/1669] Added `proxy` for `U.request()` and `RESTBuilder`. --- builders.js | 6 +++++ changes.txt | 2 ++ utils.js | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index 45790fd35..ee4ba73a0 100755 --- a/builders.js +++ b/builders.js @@ -3968,6 +3968,11 @@ RESTBuilder.prototype.promise = function(fn) { }); }; +RESTBuilder.prototype.proxy = function(value) { + this.$proxy = value; + return this; +}; + RESTBuilder.prototype.setTransform = function(name) { this.$transform = name; return this; @@ -4276,6 +4281,7 @@ RESTBuilder.prototype.exec = function(callback) { self.$persistentcookies && flags.push('cookies'); self.$length && flags.push('<' + self.$length); self.$redirect === false && flags.push('noredirect'); + self.$proxy && flags.push('proxy ' + self.$proxy); if (self.$files) { flags.push('upload'); diff --git a/changes.txt b/changes.txt index 30a5f058f..0a449e1e9 100755 --- a/changes.txt +++ b/changes.txt @@ -58,6 +58,8 @@ - added: CSS and JS supports a simple View Engine markup (config + resources + F.global) - added: `controller.split` alias to `controller.req.split` - added: nicer error response messages +- added: `RESTBuilder.proxy(proxy)` for HTTP proxy +- added: `U.request()` supports a new flag `proxy`, for example `proxy 127.0.0.1:8080` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/utils.js b/utils.js index 76d78af98..5bc6f5927 100755 --- a/utils.js +++ b/utils.js @@ -443,6 +443,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, var type = 0; var isCookies = false; var def; + var proxy; if (headers) { headers = exports.extend({}, headers); @@ -464,6 +465,38 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, continue; } + if (flags[i][0] === 'p' && flags[i][4] === 'y') { + // proxy + proxy = {}; + + var p = flags[i].substring(6); + var index = p.indexOf('@'); + if (index !== -1) { + // auth + var t = p.substring(0, index).split(':'); + p.username = t[0]; + p.password = t[1]; + p = p.substring(index + 1); + } + + proxy.protocol = p.substring(0, 7); + if (proxy.protocol === 'http://') { + proxy.protocol = 'http:'; + p = p.substring(7); + } else if (proxy.protocol === 'https:') { + proxy.protocol = 'https:'; + p = p.substring(8); + } else + proxy.protocol = 'http:'; + + index = p.lastIndexOf(':'); + + proxy.hostname = p.substring(0, index); + proxy.port = p.substring(index + 1); + proxy.method = 'CONNECT'; + continue; + } + switch (flags[i].toLowerCase()) { case 'utf8': case 'ascii': @@ -573,10 +606,14 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, var uri = Url.parse(url); uri.method = method; - uri.agent = false; + // uri.agent = false; uri.headers = headers; + options.uri = uri; - if (options.resolve) { + if (proxy) { + options.proxy = proxy; + request_proxy(options); + } else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); request_call(uri, options); @@ -587,6 +624,34 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, return options.evt; }; +function request_proxy(options) { + + var proxy = options.proxy; + proxy.path = options.uri.hostname; + proxy.headers = { host: options.uri.hostname }; + + var req = proxy.protocol === 'http:' ? Http.request(proxy) : Https.request(proxy); + + req.on('error', function(e) { + options.callback(new Error('Proxy error: ' + e.toString()), '', 0, EMPTYOBJECT, proxy.hostname, EMPTYOBJECT); + options.callback = null; + }); + + req.on('connect', function(res, socket) { + if (res.statusCode === 200) { + options.uri.agent = options.uri.protocol === 'http:' ? new Http.Agent() : new Https.Agent(); + options.uri.agent.reuseSocket(socket, req); + options.socket = socket; + request_call(options.uri, options); + } else { + options.callback(new Error((res.statusMessage || 'Proxy error') + ': ' + res.statusCode), '', res.statusCode, res.headers, proxy.hostname, EMPTYOBJECT); + options.callback = null; + } + }); + + req.end(); +} + function request_call(uri, options) { var connection = uri.protocol === 'https:' ? Https : Http; @@ -775,6 +840,12 @@ function request_response(res, uri, options) { }); res.on('end', function() { + + if (options.socket) { + options.uri.agent.destroy(); + options.socket.destroy(); + } + var self = this; var str = self._buffer ? self._buffer.toString(options.encoding) : ''; self._buffer = undefined; From 6870c187f02de5d8250f43ab94eb6f5150805204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 10 May 2018 15:26:19 +0200 Subject: [PATCH 0299/1669] Added new test file. --- test/test-tmp.js | 51 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/test/test-tmp.js b/test/test-tmp.js index cf3890e06..0b0d4be9d 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,6 +1,49 @@ require('../index'); +const Http = require('http'); +const Https = require('https'); +const Url = require('url'); -F.backup(F.path.root('semtu.package'), ['config-debug', 'my-config.txt', '/workers/'], function(err, filename) { - console.log(filename); - F.restore(filename, F.path.root('tmp')); -}); \ No newline at end of file +function rrr() { + var options = { port: 8080, hostname: '127.0.0.1', method: 'CONNECT', headers: { host: 'www.totaljs.com:443' }}; + var req = Http.request(options); + + req.on('connect', function(res, socket) { + + console.log(res.statusCode); + options = Url.parse('https://www.totaljs.com'); + + var agent = new Https.Agent(); + agent.reuseSocket(socket, req); + options.agent = agent; + + var r = Http.request(options); + + r.on('response', function(res) { + + res.on('data', function(data) { + console.log(data.toString('utf8')); + }); + + res.on('end', function() { + agent.destroy(); + agent = null; + socket.destroy(); + }); + }); + + r.end(); + + }); + + req.end(); +} + +//U.request('http://www.vyvojari.sk', ['get', 'proxy http://127.0.0.1:8080'], console.log); + +RESTBuilder.make(function(builder) { + builder.url('https://www.spektrum-bb.sk'); + builder.proxy('127.0.0.1:8080'); + builder.exec(console.log); +}); + +//rrr(); \ No newline at end of file From 25e20525424cfdde7fa7f3573fae11d9c1044ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 10 May 2018 20:19:58 +0200 Subject: [PATCH 0300/1669] Improved handling errors and added `security.txt`. --- changes.txt | 1 + index.js | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 0a449e1e9..b50c538a2 100755 --- a/changes.txt +++ b/changes.txt @@ -35,6 +35,7 @@ - added: config `nosql-inmemory' can contain name of databases e.g. (`users, products`) or String Array - added: config `nosql-cleaner` for cleaning databases from removed documents (default: `1440` === 24 hours) - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning +- added: config `security.txt` for auto-generating security.txt content (more in docs) - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` diff --git a/index.js b/index.js index 9c61bc4cd..57f819af9 100755 --- a/index.js +++ b/index.js @@ -609,6 +609,7 @@ function Framework() { author: '', secret: Os.hostname() + '-' + Os.platform() + '-' + Os.arch(), + 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'default-xpoweredby': 'Total.js', 'etag-version': '', 'directory-src': '/.src/', @@ -8770,6 +8771,10 @@ F.$configure_configs = function(arr, rewrite) { obj[name] = value; break; + case 'security.txt': + obj[name] = value ? value.split(',').trim().join('\n') : ''; + break; + default: if (subtype === 'string') @@ -15290,6 +15295,8 @@ function extend_response(PROTO) { return res.$text(); }; + const SECURITYTXT = { '/security.txt': 1, '/.well-known/security.txt': 1 }; + PROTO.continue = function(callback) { var res = this; @@ -15305,6 +15312,11 @@ function extend_response(PROTO) { return res; } + if (SECURITYTXT[req.url] && F.config['security.txt']) { + res.send(200, F.config['security.txt'], 'text/plain'); + return; + } + req.$key = createTemporaryKey(req); if (F.temporary.notfound[req.$key]) { @@ -15887,18 +15899,22 @@ function extend_response(PROTO) { return res; var req = res.req; + var key = 'error' + res.options.code; + res.options.problem && F.problem(res.options.problem, 'response' + res.options.code + '()', req.uri, req.ip); - res.writeHead(res.options.code || 501, res.options.headers || HEADERS.responseCode); - if (req.method === 'HEAD') + if (req.method === 'HEAD') { + res.writeHead(res.options.code || 501, res.options.headers || HEADERS.responseCode); res.end(); - else - res.end(res.options.body || U.httpStatus(res.options.code) + prepare_error(res.options && res.options.problem)); + F.stats.response[key]++; + response_end(res); + } else { + req.$total_route = F.lookup(req, '#' + res.options.code, EMPTYARRAY, 0); + req.$total_exception = res.options.problem; + req.$total_execute(res.options.code, true); + } - var key = 'error' + res.options.code; F.$events[key] && F.emit(key, req, res, res.options.problem); - F.stats.response[key]++; - response_end(res); return res; }; } From 30ced0170b7d62321f952b52bffe19c3a4a944cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 10 May 2018 20:32:02 +0200 Subject: [PATCH 0301/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c4690394a..85c65023c 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-54", + "version": "3.0.0-55", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 04293c10be35bda49147a1e8e848018bb5a402cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 10 May 2018 21:23:51 +0200 Subject: [PATCH 0302/1669] Fixed proxy auth. --- utils.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/utils.js b/utils.js index 5bc6f5927..b58587331 100755 --- a/utils.js +++ b/utils.js @@ -467,18 +467,8 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (flags[i][0] === 'p' && flags[i][4] === 'y') { // proxy - proxy = {}; - var p = flags[i].substring(6); - var index = p.indexOf('@'); - if (index !== -1) { - // auth - var t = p.substring(0, index).split(':'); - p.username = t[0]; - p.password = t[1]; - p = p.substring(index + 1); - } - + proxy = {}; proxy.protocol = p.substring(0, 7); if (proxy.protocol === 'http://') { proxy.protocol = 'http:'; @@ -489,8 +479,16 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, } else proxy.protocol = 'http:'; - index = p.lastIndexOf(':'); + var index = p.indexOf('@'); + if (index !== -1) { + // auth + var t = p.substring(0, index).split(':'); + proxy.username = t[0]; + proxy.password = t[1]; + p = p.substring(index + 1); + } + index = p.lastIndexOf(':'); proxy.hostname = p.substring(0, index); proxy.port = p.substring(index + 1); proxy.method = 'CONNECT'; @@ -630,6 +628,9 @@ function request_proxy(options) { proxy.path = options.uri.hostname; proxy.headers = { host: options.uri.hostname }; + if (proxy.username && proxy.password) + proxy.headers.authorization = 'Basic ' + U.createBuffer(proxy.username + ':' + proxy.password).toString('base64'); + var req = proxy.protocol === 'http:' ? Http.request(proxy) : Https.request(proxy); req.on('error', function(e) { From a9755e2bef96df64b27c041d4e2dc2f81bc4c84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 11 May 2018 09:16:33 +0200 Subject: [PATCH 0303/1669] Fixed `join` for `NoSQL.one()` method. --- nosql.js | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/nosql.js b/nosql.js index fb5e977af..607707035 100755 --- a/nosql.js +++ b/nosql.js @@ -2798,18 +2798,23 @@ DatabaseBuilder.prototype.log = function(msg, user) { DatabaseBuilder.prototype.$callbackjoin = function(callback) { var self = this; - Object.keys(self.$join).wait(function(key, next) { + var join = self.$join[key]; var response = self.$response; var unique = []; - for (var i = 0; i < response.length; i++) { - var item = response[i]; - var val = item[join.b]; - if (val !== undefined) - if (unique.indexOf(val) === -1) + if (response instanceof Array && response.length) { + for (var i = 0; i < response.length; i++) { + var item = response[i]; + var val = item[join.b]; + if (val !== undefined && unique.indexOf(val) === -1) unique.push(val); + } + } else if (response) { + var val = response[join.b]; + if (val !== undefined && unique.indexOf(val) === -1) + unique.push(val); } var db = NOSQL(join.name); @@ -2845,6 +2850,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { })(unique[i]); } } else { + if (unique.length) { join.builder.$options.fields && join.builder.$options.fields.push(join.a); join.builder.$callback = function(err, docs) { @@ -2858,7 +2864,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { } } - }, callback); + }, callback, 2); return self; }; From 1a68ec719bfb4676bb8422f83e2a9627e0069091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 11 May 2018 20:39:40 +0200 Subject: [PATCH 0304/1669] Fixed bugs when DB reading files. --- changes.txt | 1 + nosql.js | 65 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/changes.txt b/changes.txt index b50c538a2..67cb5a109 100755 --- a/changes.txt +++ b/changes.txt @@ -61,6 +61,7 @@ - added: nicer error response messages - added: `RESTBuilder.proxy(proxy)` for HTTP proxy - added: `U.request()` supports a new flag `proxy`, for example `proxy 127.0.0.1:8080` +- added: NoSQL database a new event `change`, more in docs - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/nosql.js b/nosql.js index 607707035..058a0079b 100755 --- a/nosql.js +++ b/nosql.js @@ -777,6 +777,7 @@ Database.prototype.restore = function(filename, callback) { self.storage && self.storage.refresh(); } } + self.$events.change && self.emit('change', 'restore'); callback && callback(err, response); }); @@ -1311,6 +1312,7 @@ Database.prototype.$append = function() { function next_append(self) { self.$writting = false; self.next(0); + self.$events.change && self.emit('change', 'insert'); self.views && setImmediate(views_refresh, self); } @@ -1355,6 +1357,7 @@ Database.prototype.$update = function() { var length = filter.length; var backup = false; var filters = 0; + var change = false; for (var i = 0; i < length; i++) { var fil = filter[i]; @@ -1437,6 +1440,9 @@ Database.prototype.$update = function() { var was = true; + if (!change) + change = true; + if (rec.doc.length === upd.length) { var b = Buffer.byteLength(upd); if (rec.length === b) { @@ -1478,7 +1484,11 @@ Database.prototype.$update = function() { fs = null; self.$writting = false; self.next(0); - self.views && setImmediate(views_refresh, self); + + if (change) { + self.$events.change && self.emit('change', 'update'); + self.views && setImmediate(views_refresh, self); + } }; fs.openupdate(); @@ -1573,7 +1583,12 @@ Database.prototype.$update_inmemory = function() { setImmediate(function() { self.next(0); - change && self.views && setImmediate(views_refresh, self); + + if (change) { + self.$events.change && self.emit('change', 'update'); + self.views && setImmediate(views_refresh, self); + } + }); }); }; @@ -2484,6 +2499,9 @@ Database.prototype.$remove = function() { } } + if (!change) + change = true; + item.count++; self.$events.remove && self.emit('remove', doc); @@ -2509,7 +2527,11 @@ Database.prototype.$remove = function() { fs = null; self.$writting = false; self.next(0); - change && self.views && setImmediate(views_refresh, self); + + if (change) { + self.views && setImmediate(views_refresh, self); + self.$events.change && self.emit('change', 'remove'); + } }; fs.openupdate(); @@ -2538,6 +2560,7 @@ Database.prototype.$clear = function() { for (var i = 0; i < filter.length; i++) filter[i](); self.views && setImmediate(views_refresh, self); + self.$events.change && self.emit('change', 'clear'); self.next(0); } }); @@ -2667,7 +2690,10 @@ Database.prototype.$remove_inmemory = function() { } self.next(0); - change && self.views && setImmediate(views_refresh, self); + if (change) { + self.views && setImmediate(views_refresh, self); + self.$events.change && self.emit('change', 'remove'); + } }); }; @@ -2717,6 +2743,7 @@ Database.prototype.$drop = function() { remove.wait((filename, next) => Fs.unlink(filename, next), function() { self.next(0); self.free(true); + self.$events.change && self.emit('change', 'drop'); }, 5); Object.keys(self.inmemory).forEach(function(key) { @@ -4918,9 +4945,15 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { return cacheid; }; -Binary.prototype.read = function(id, callback) { +Binary.prototype.read = function(id, callback, count) { var self = this; + + if (count > 3) { + callback(new Error('File not found.')); + return self; + } + var isnew = false; if (id > 0) @@ -4943,24 +4976,26 @@ Binary.prototype.read = function(id, callback) { stream.on('error', err => callback(err)); stream.on('data', function(buffer) { - var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, ''); - var meta; + var json = buffer.toString('utf8').replace(REG_CLEAN, ''); - stream = Fs.createReadStream(filename, BINARYREADDATA); - try { - meta = JSON.parse(json, jsonparser); - callback(null, stream, meta); - } catch (e) { - F.error(e, 'nosql.binary.read', filename + ' --> ' + json); - console.log(e, filename, json, BINARYREADDATA); - callback(e); + if (!json) { + setTimeout(readfileattempt, 100, self, id, callback, count || 1); + return; } + + var meta = JSON.parse(json, jsonparser); + stream = Fs.createReadStream(filename, BINARYREADDATA); + callback(null, stream, meta); CLEANUP(stream); }); return self; }; +function readfileattempt(self, id, callback, count) { + self.read(id, callback, count + 1); +} + Binary.prototype.remove = function(id, callback) { var self = this; From 159ad9a285e3a8b95ac6560b992efcdfa351844e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 11 May 2018 22:17:30 +0200 Subject: [PATCH 0305/1669] Improved code. --- nosql.js | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/nosql.js b/nosql.js index 058a0079b..a4ea71e08 100755 --- a/nosql.js +++ b/nosql.js @@ -4967,26 +4967,21 @@ Binary.prototype.read = function(id, callback, count) { var filename; if (isnew) { - var path = self.$directory(id); - filename = Path.join(path, id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); } else filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); stream.on('data', function(buffer) { - var json = buffer.toString('utf8').replace(REG_CLEAN, ''); - - if (!json) { + if (json) { + var meta = JSON.parse(json, jsonparser); + stream = Fs.createReadStream(filename, BINARYREADDATA); + callback(null, stream, meta); + CLEANUP(stream); + } else setTimeout(readfileattempt, 100, self, id, callback, count || 1); - return; - } - - var meta = JSON.parse(json, jsonparser); - stream = Fs.createReadStream(filename, BINARYREADDATA); - callback(null, stream, meta); - CLEANUP(stream); }); return self; From f988923ed09517e8fcc0e2321849b2f0ff0989f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 12 May 2018 09:18:46 +0200 Subject: [PATCH 0306/1669] Fixed `builder.in()` filter in NoSQL embedded. --- nosql.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index a4ea71e08..33fc35bd4 100755 --- a/nosql.js +++ b/nosql.js @@ -3335,7 +3335,7 @@ DatabaseBuilder.prototype.in = function(name, value) { var self = this; var key = 'in' + (self.$counter++); self.$params[key] = value instanceof Array ? value : [value]; - var code = '$tmp=doc.{0};if($tmp instanceof Array){for(var $i=0;$i<$tmp.length;$i++){if(arg.{1}.indexOf($tmp[$i])!==-1){$is=true;break}}}else{if(arg.{1}.indexOf($tmp)!==-1)$is=true}'.format(name, key); + var code = 'if($is)$is=false;$tmp=doc.{0};if($tmp instanceof Array){for(var $i=0;$i<$tmp.length;$i++){if(arg.{1}.indexOf($tmp[$i])!==-1){$is=true;break}}}else{if(arg.{1}.indexOf($tmp)!==-1)$is=true}'.format(name, key); if (self.$scope) code = 'if(!$is){' + code + '}'; self.$code.push(code); diff --git a/package.json b/package.json index 85c65023c..0d1e4d2f1 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-55", + "version": "3.0.0-56", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 7e2583e549a6d6715070b4d90bcb11eb49813f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 12 May 2018 10:12:01 +0200 Subject: [PATCH 0307/1669] Improved error handling. --- websocketclient.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/websocketclient.js b/websocketclient.js index 00715f668..c0d17fb60 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -493,7 +493,7 @@ WebSocketClient.prototype.$onerror = function(err) { if (this.isClosed) return; - if (REG_WEBSOCKET_ERROR.test(err.stack)) { + if (REG_WEBSOCKET_ERROR.test(err.stack) || err.code === 'HPE_INVALID_CONSTANT') { this.isClosed = true; this.$onclose(); } else From a917c7a18a86578f830cf39252e08a8fce6dd5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 12 May 2018 10:20:30 +0200 Subject: [PATCH 0308/1669] Fixed error handling. --- websocketclient.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/websocketclient.js b/websocketclient.js index c0d17fb60..a11d13dcf 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -33,7 +33,6 @@ const Http = require('http'); const Url = require('url'); const Zlib = require('zlib'); const ENCODING = 'utf8'; -const REG_WEBSOCKET_ERROR = /ECONNRESET|EHOSTUNREACH|EPIPE|is closed/i; const WEBSOCKET_COMPRESS = U.createBuffer([0x00, 0x00, 0xFF, 0xFF]); const WEBSOCKET_COMPRESS_OPTIONS = { windowBits: Zlib.Z_DEFAULT_WINDOWBITS }; const CONCAT = [null, null]; @@ -489,15 +488,11 @@ WebSocketClient.prototype.parseInflate = function() { }; WebSocketClient.prototype.$onerror = function(err) { - - if (this.isClosed) - return; - - if (REG_WEBSOCKET_ERROR.test(err.stack) || err.code === 'HPE_INVALID_CONSTANT') { + this.$events.error && this.emit('error', err); + if (!this.isClosed) { this.isClosed = true; this.$onclose(); - } else - this.$events.error && this.emit('error', err); + } }; WebSocketClient.prototype.$onclose = function() { From 5de39623edd3db30128f7f86da9754b1c49cf101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 12 May 2018 21:03:49 +0200 Subject: [PATCH 0309/1669] Improved `proxy`. --- changes.txt | 1 + index.js | 1 + utils.js | 91 ++++++++++++++++++++++++++++++++++------------------- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/changes.txt b/changes.txt index 67cb5a109..39113335c 100755 --- a/changes.txt +++ b/changes.txt @@ -36,6 +36,7 @@ - added: config `nosql-cleaner` for cleaning databases from removed documents (default: `1440` === 24 hours) - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: config `security.txt` for auto-generating security.txt content (more in docs) +- added: config `default-proxy` for default web proxy server - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` diff --git a/index.js b/index.js index 57f819af9..90c9b4e63 100755 --- a/index.js +++ b/index.js @@ -653,6 +653,7 @@ function Framework() { 'default-layout': 'layout', 'default-theme': '', + 'default-proxy': '', // default maximum request size / length // default 10 kB diff --git a/utils.js b/utils.js index b58587331..188693e6f 100755 --- a/utils.js +++ b/utils.js @@ -80,6 +80,7 @@ const STREAMPIPE = { end: false }; const CT = 'Content-Type'; const CRC32TABLE = '00000000,77073096,EE0E612C,990951BA,076DC419,706AF48F,E963A535,9E6495A3,0EDB8832,79DCB8A4,E0D5E91E,97D2D988,09B64C2B,7EB17CBD,E7B82D07,90BF1D91,1DB71064,6AB020F2,F3B97148,84BE41DE,1ADAD47D,6DDDE4EB,F4D4B551,83D385C7,136C9856,646BA8C0,FD62F97A,8A65C9EC,14015C4F,63066CD9,FA0F3D63,8D080DF5,3B6E20C8,4C69105E,D56041E4,A2677172,3C03E4D1,4B04D447,D20D85FD,A50AB56B,35B5A8FA,42B2986C,DBBBC9D6,ACBCF940,32D86CE3,45DF5C75,DCD60DCF,ABD13D59,26D930AC,51DE003A,C8D75180,BFD06116,21B4F4B5,56B3C423,CFBA9599,B8BDA50F,2802B89E,5F058808,C60CD9B2,B10BE924,2F6F7C87,58684C11,C1611DAB,B6662D3D,76DC4190,01DB7106,98D220BC,EFD5102A,71B18589,06B6B51F,9FBFE4A5,E8B8D433,7807C9A2,0F00F934,9609A88E,E10E9818,7F6A0DBB,086D3D2D,91646C97,E6635C01,6B6B51F4,1C6C6162,856530D8,F262004E,6C0695ED,1B01A57B,8208F4C1,F50FC457,65B0D9C6,12B7E950,8BBEB8EA,FCB9887C,62DD1DDF,15DA2D49,8CD37CF3,FBD44C65,4DB26158,3AB551CE,A3BC0074,D4BB30E2,4ADFA541,3DD895D7,A4D1C46D,D3D6F4FB,4369E96A,346ED9FC,AD678846,DA60B8D0,44042D73,33031DE5,AA0A4C5F,DD0D7CC9,5005713C,270241AA,BE0B1010,C90C2086,5768B525,206F85B3,B966D409,CE61E49F,5EDEF90E,29D9C998,B0D09822,C7D7A8B4,59B33D17,2EB40D81,B7BD5C3B,C0BA6CAD,EDB88320,9ABFB3B6,03B6E20C,74B1D29A,EAD54739,9DD277AF,04DB2615,73DC1683,E3630B12,94643B84,0D6D6A3E,7A6A5AA8,E40ECF0B,9309FF9D,0A00AE27,7D079EB1,F00F9344,8708A3D2,1E01F268,6906C2FE,F762575D,806567CB,196C3671,6E6B06E7,FED41B76,89D32BE0,10DA7A5A,67DD4ACC,F9B9DF6F,8EBEEFF9,17B7BE43,60B08ED5,D6D6A3E8,A1D1937E,38D8C2C4,4FDFF252,D1BB67F1,A6BC5767,3FB506DD,48B2364B,D80D2BDA,AF0A1B4C,36034AF6,41047A60,DF60EFC3,A867DF55,316E8EEF,4669BE79,CB61B38C,BC66831A,256FD2A0,5268E236,CC0C7795,BB0B4703,220216B9,5505262F,C5BA3BBE,B2BD0B28,2BB45A92,5CB36A04,C2D7FFA7,B5D0CF31,2CD99E8B,5BDEAE1D,9B64C2B0,EC63F226,756AA39C,026D930A,9C0906A9,EB0E363F,72076785,05005713,95BF4A82,E2B87A14,7BB12BAE,0CB61B38,92D28E9B,E5D5BE0D,7CDCEFB7,0BDBDF21,86D3D2D4,F1D4E242,68DDB3F8,1FDA836E,81BE16CD,F6B9265B,6FB077E1,18B74777,88085AE6,FF0F6A70,66063BCA,11010B5C,8F659EFF,F862AE69,616BFFD3,166CCF45,A00AE278,D70DD2EE,4E048354,3903B3C2,A7672661,D06016F7,4969474D,3E6E77DB,AED16A4A,D9D65ADC,40DF0B66,37D83BF0,A9BCAE53,DEBB9EC5,47B2CF7F,30B5FFE9,BDBDF21C,CABAC28A,53B39330,24B4A3A6,BAD03605,CDD70693,54DE5729,23D967BF,B3667A2E,C4614AB8,5D681B02,2A6F2B94,B40BBE37,C30C8EA1,5A05DF1B,2D02EF8D'.split(',').map(s => parseInt(s, 16)); const REGISARR = /\[\d+\]$/; +const PROXYBLACKLIST = { 'localhost': 1, '127.0.0.1': 1, '0.0.0.0': 1 }; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -411,6 +412,43 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng return keys; }; +function parseProxy(p) { + + var key = 'proxy_' + p; + + if (F.temporary.other[key]) + return F.temporary.other[key]; + + var proxy = {}; + + proxy.protocol = p.substring(0, 7); + + if (proxy.protocol === 'http://') { + proxy.protocol = 'http:'; + p = p.substring(7); + } else if (proxy.protocol === 'https:') { + proxy.protocol = 'https:'; + p = p.substring(8); + } else + proxy.protocol = 'http:'; + + var index = p.indexOf('@'); + if (index !== -1) { + // auth + var t = p.substring(0, index).split(':'); + proxy.username = t[0]; + proxy.password = t[1]; + p = p.substring(index + 1); + } + + index = p.lastIndexOf(':'); + proxy.hostname = p.substring(0, index); + proxy.port = p.substring(index + 1); + proxy.method = 'CONNECT'; + + return F.temporary.other[key] = proxy; +} + /** * Create a request to a specific URL * @param {String} url URL address. @@ -466,32 +504,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, } if (flags[i][0] === 'p' && flags[i][4] === 'y') { - // proxy - var p = flags[i].substring(6); - proxy = {}; - proxy.protocol = p.substring(0, 7); - if (proxy.protocol === 'http://') { - proxy.protocol = 'http:'; - p = p.substring(7); - } else if (proxy.protocol === 'https:') { - proxy.protocol = 'https:'; - p = p.substring(8); - } else - proxy.protocol = 'http:'; - - var index = p.indexOf('@'); - if (index !== -1) { - // auth - var t = p.substring(0, index).split(':'); - proxy.username = t[0]; - proxy.password = t[1]; - p = p.substring(index + 1); - } - - index = p.lastIndexOf(':'); - proxy.hostname = p.substring(0, index); - proxy.port = p.substring(index + 1); - proxy.method = 'CONNECT'; + proxy = parseProxy(flags[i].substring(6)); continue; } @@ -608,9 +621,12 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, uri.headers = headers; options.uri = uri; + if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) + proxy = parseProxy(F.config['default-proxy']); + if (proxy) { options.proxy = proxy; - request_proxy(options); + request_proxy(options, request_call); } else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); @@ -622,7 +638,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, return options.evt; }; -function request_proxy(options) { +function request_proxy(options, callback) { var proxy = options.proxy; proxy.path = options.uri.hostname; @@ -643,7 +659,7 @@ function request_proxy(options) { options.uri.agent = options.uri.protocol === 'http:' ? new Http.Agent() : new Https.Agent(); options.uri.agent.reuseSocket(socket, req); options.socket = socket; - request_call(options.uri, options); + callback(options.uri, options); } else { options.callback(new Error((res.statusMessage || 'Proxy error') + ': ' + res.statusCode), '', res.statusCode, res.headers, proxy.hostname, EMPTYOBJECT); options.callback = null; @@ -727,7 +743,6 @@ function request_writefile(req, options, file, next) { } } - function request_response(res, uri, options) { res._buffer = null; @@ -925,8 +940,8 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi if (typeof(encoding) !== 'string') encoding = ENCODING; + var proxy, type = 0; var method = 'GET'; - var type = 0; var options = { callback: callback, resolve: false, length: 0, evt: new EventEmitter2(), timeout: timeout || 60000, post: false, encoding: encoding }; if (headers) @@ -951,6 +966,11 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi continue; } + if (flags[i][0] === 'p' && flags[i][4] === 'y') { + proxy = parseProxy(flags[i].substring(6)); + continue; + } + switch (flags[i].toLowerCase()) { case 'utf8': @@ -1033,13 +1053,20 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi uri.method = method; uri.agent = false; uri.headers = headers; + options.uri = uri; if (data.length) { options.data = exports.createBuffer(data, ENCODING); headers['Content-Length'] = options.data.length; } - if (options.resolve) { + if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) + proxy = parseProxy(F.config['default-proxy']); + + if (proxy) { + options.proxy = proxy; + request_proxy(options, download_call); + } else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); download_call(uri, options); From 74f938c7b983944d0c86f788499943a326293415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 May 2018 13:01:04 +0200 Subject: [PATCH 0310/1669] Fixed closing of WebSocket. --- websocketclient.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/websocketclient.js b/websocketclient.js index a11d13dcf..506ae66e7 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -21,7 +21,7 @@ /** * @module WebSocketClient - * @version 2.9.0 + * @version 3.0.0 */ if (!global.framework_utils) @@ -290,7 +290,7 @@ WebSocketClient.prototype.$ondata = function(data) { case 0x08: // close - this.close(); + this.close(true); break; case 0x09: @@ -496,6 +496,7 @@ WebSocketClient.prototype.$onerror = function(err) { }; WebSocketClient.prototype.$onclose = function() { + if (this._isClosed) return; @@ -515,7 +516,7 @@ WebSocketClient.prototype.$onclose = function() { } this.$events.close && this.emit('close'); - this.socket.removeAllListeners(); + this.socket && this.socket.removeAllListeners(); }; /** @@ -605,6 +606,12 @@ function websocket_deflate(data) { * @return {WebSocketClient} */ WebSocketClient.prototype.close = function(message, code) { + + if (message !== true) { + this.options.reconnect = 0; + } else + message = undefined; + if (!this.isClosed) { this.isClosed = true; this.socket.end(U.getWebSocketFrame(code || 1000, message ? (this.options.encodedecode ? encodeURIComponent(message) : message) : '', 0x08)); From 4e1f5113095aec079cce01f4a2cc04910a2f1772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 May 2018 19:13:25 +0200 Subject: [PATCH 0311/1669] Fixed `proxy` for `U.request()` + `U.download()`. --- utils.js | 174 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 120 insertions(+), 54 deletions(-) diff --git a/utils.js b/utils.js index 188693e6f..ff89d5216 100755 --- a/utils.js +++ b/utils.js @@ -36,6 +36,7 @@ const Fs = require('fs'); const Events = require('events'); const Crypto = require('crypto'); const Zlib = require('zlib'); +const Tls = require('tls'); const CONCAT = [null, null]; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { @@ -81,6 +82,8 @@ const CT = 'Content-Type'; const CRC32TABLE = '00000000,77073096,EE0E612C,990951BA,076DC419,706AF48F,E963A535,9E6495A3,0EDB8832,79DCB8A4,E0D5E91E,97D2D988,09B64C2B,7EB17CBD,E7B82D07,90BF1D91,1DB71064,6AB020F2,F3B97148,84BE41DE,1ADAD47D,6DDDE4EB,F4D4B551,83D385C7,136C9856,646BA8C0,FD62F97A,8A65C9EC,14015C4F,63066CD9,FA0F3D63,8D080DF5,3B6E20C8,4C69105E,D56041E4,A2677172,3C03E4D1,4B04D447,D20D85FD,A50AB56B,35B5A8FA,42B2986C,DBBBC9D6,ACBCF940,32D86CE3,45DF5C75,DCD60DCF,ABD13D59,26D930AC,51DE003A,C8D75180,BFD06116,21B4F4B5,56B3C423,CFBA9599,B8BDA50F,2802B89E,5F058808,C60CD9B2,B10BE924,2F6F7C87,58684C11,C1611DAB,B6662D3D,76DC4190,01DB7106,98D220BC,EFD5102A,71B18589,06B6B51F,9FBFE4A5,E8B8D433,7807C9A2,0F00F934,9609A88E,E10E9818,7F6A0DBB,086D3D2D,91646C97,E6635C01,6B6B51F4,1C6C6162,856530D8,F262004E,6C0695ED,1B01A57B,8208F4C1,F50FC457,65B0D9C6,12B7E950,8BBEB8EA,FCB9887C,62DD1DDF,15DA2D49,8CD37CF3,FBD44C65,4DB26158,3AB551CE,A3BC0074,D4BB30E2,4ADFA541,3DD895D7,A4D1C46D,D3D6F4FB,4369E96A,346ED9FC,AD678846,DA60B8D0,44042D73,33031DE5,AA0A4C5F,DD0D7CC9,5005713C,270241AA,BE0B1010,C90C2086,5768B525,206F85B3,B966D409,CE61E49F,5EDEF90E,29D9C998,B0D09822,C7D7A8B4,59B33D17,2EB40D81,B7BD5C3B,C0BA6CAD,EDB88320,9ABFB3B6,03B6E20C,74B1D29A,EAD54739,9DD277AF,04DB2615,73DC1683,E3630B12,94643B84,0D6D6A3E,7A6A5AA8,E40ECF0B,9309FF9D,0A00AE27,7D079EB1,F00F9344,8708A3D2,1E01F268,6906C2FE,F762575D,806567CB,196C3671,6E6B06E7,FED41B76,89D32BE0,10DA7A5A,67DD4ACC,F9B9DF6F,8EBEEFF9,17B7BE43,60B08ED5,D6D6A3E8,A1D1937E,38D8C2C4,4FDFF252,D1BB67F1,A6BC5767,3FB506DD,48B2364B,D80D2BDA,AF0A1B4C,36034AF6,41047A60,DF60EFC3,A867DF55,316E8EEF,4669BE79,CB61B38C,BC66831A,256FD2A0,5268E236,CC0C7795,BB0B4703,220216B9,5505262F,C5BA3BBE,B2BD0B28,2BB45A92,5CB36A04,C2D7FFA7,B5D0CF31,2CD99E8B,5BDEAE1D,9B64C2B0,EC63F226,756AA39C,026D930A,9C0906A9,EB0E363F,72076785,05005713,95BF4A82,E2B87A14,7BB12BAE,0CB61B38,92D28E9B,E5D5BE0D,7CDCEFB7,0BDBDF21,86D3D2D4,F1D4E242,68DDB3F8,1FDA836E,81BE16CD,F6B9265B,6FB077E1,18B74777,88085AE6,FF0F6A70,66063BCA,11010B5C,8F659EFF,F862AE69,616BFFD3,166CCF45,A00AE278,D70DD2EE,4E048354,3903B3C2,A7672661,D06016F7,4969474D,3E6E77DB,AED16A4A,D9D65ADC,40DF0B66,37D83BF0,A9BCAE53,DEBB9EC5,47B2CF7F,30B5FFE9,BDBDF21C,CABAC28A,53B39330,24B4A3A6,BAD03605,CDD70693,54DE5729,23D967BF,B3667A2E,C4614AB8,5D681B02,2A6F2B94,B40BBE37,C30C8EA1,5A05DF1B,2D02EF8D'.split(',').map(s => parseInt(s, 16)); const REGISARR = /\[\d+\]$/; const PROXYBLACKLIST = { 'localhost': 1, '127.0.0.1': 1, '0.0.0.0': 1 }; +const PROXYOPTIONS = {}; +const PROXYHEADERS = {}; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -413,40 +416,19 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng }; function parseProxy(p) { - var key = 'proxy_' + p; - if (F.temporary.other[key]) return F.temporary.other[key]; - var proxy = {}; - - proxy.protocol = p.substring(0, 7); - - if (proxy.protocol === 'http://') { - proxy.protocol = 'http:'; - p = p.substring(7); - } else if (proxy.protocol === 'https:') { - proxy.protocol = 'https:'; - p = p.substring(8); - } else - proxy.protocol = 'http:'; + if (p.indexOf('://') === -1) + p = 'http://' + p; - var index = p.indexOf('@'); - if (index !== -1) { - // auth - var t = p.substring(0, index).split(':'); - proxy.username = t[0]; - proxy.password = t[1]; - p = p.substring(index + 1); - } + var obj = Url.parse(p); - index = p.lastIndexOf(':'); - proxy.hostname = p.substring(0, index); - proxy.port = p.substring(index + 1); - proxy.method = 'CONNECT'; + if (obj.auth) + obj._auth = 'Basic ' + U.createBuffer(obj.auth).toString('base64'); - return F.temporary.other[key] = proxy; + return F.temporary.other[key] = obj; } /** @@ -617,7 +599,6 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, var uri = Url.parse(url); uri.method = method; - // uri.agent = false; uri.headers = headers; options.uri = uri; @@ -626,7 +607,10 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (proxy) { options.proxy = proxy; - request_proxy(options, request_call); + if (uri.protocol === 'http:') + request_call(uri, options); + else + request_proxy(options, request_call); } else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); @@ -640,14 +624,17 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, function request_proxy(options, callback) { + PROXYHEADERS.host = options.uri.hostname; + var proxy = options.proxy; - proxy.path = options.uri.hostname; - proxy.headers = { host: options.uri.hostname }; + proxy.path = options.uri.hostname + ':443'; + proxy.headers = PROXYHEADERS; + proxy.method = 'CONNECT'; - if (proxy.username && proxy.password) - proxy.headers.authorization = 'Basic ' + U.createBuffer(proxy.username + ':' + proxy.password).toString('base64'); + if (proxy._auth) + proxy.headers['Proxy-Authorization'] = proxy._auth; - var req = proxy.protocol === 'http:' ? Http.request(proxy) : Https.request(proxy); + var req = Http.request(proxy); req.on('error', function(e) { options.callback(new Error('Proxy error: ' + e.toString()), '', 0, EMPTYOBJECT, proxy.hostname, EMPTYOBJECT); @@ -656,10 +643,17 @@ function request_proxy(options, callback) { req.on('connect', function(res, socket) { if (res.statusCode === 200) { - options.uri.agent = options.uri.protocol === 'http:' ? new Http.Agent() : new Https.Agent(); - options.uri.agent.reuseSocket(socket, req); - options.socket = socket; - callback(options.uri, options); + var tls = Tls.connect(0, { servername: options.uri.hostname, headers: options.uri.headers, socket: socket }); + + tls.on('secureConnect', function() { + options.uri.agent = options.uri.protocol === 'http:' ? new Http.Agent() : new Https.Agent(); + options.uri.agent.reuseSocket(tls, req); + //req.onSocket(tls); + options.socket = tls; + options.proxy.tls = tls; + callback(options.uri, options); + }); + } else { options.callback(new Error((res.statusMessage || 'Proxy error') + ': ' + res.statusCode), '', res.statusCode, res.headers, proxy.hostname, EMPTYOBJECT); options.callback = null; @@ -672,7 +666,23 @@ function request_proxy(options, callback) { function request_call(uri, options) { var connection = uri.protocol === 'https:' ? Https : Http; - var req = options.post ? connection.request(uri, (res) => request_response(res, uri, options)) : connection.get(uri, (res) => request_response(res, uri, options)); + var opt; + + if (options.proxy && !options.proxy.tls) { + opt = PROXYOPTIONS; + opt.port = options.proxy.port; + opt.host = options.proxy.hostname; + opt.path = uri.href; + opt.headers = uri.headers; + opt.method = uri.method; + + if (options.proxy._auth) + opt.headers['Proxy-Authorization'] = options.proxy._auth; + + } else + opt = uri; + + var req = options.post ? connection.request(opt, (res) => request_response(res, uri, options)) : connection.get(opt, (res) => request_response(res, uri, options)); if (!options.callback) { req.on('error', NOOP); @@ -702,7 +712,6 @@ function request_call(uri, options) { if (options.upload) { options.first = true; options.files.wait(function(file, next) { - // next(); request_writefile(req, options, file, next); }, function() { @@ -799,6 +808,13 @@ function request_response(res, uri, options) { res.req.removeAllListeners(); res.req = null; + if (options.proxy && tmp.protocol === 'https:') { + // TLS? + options.uri = tmp; + request_proxy(options, request_call); + return + } + if (!options.resolve) { res.removeAllListeners(); res = null; @@ -1065,7 +1081,10 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi if (proxy) { options.proxy = proxy; - request_proxy(options, download_call); + if (uri.protocol === 'http:') + download_call(uri, options); + else + request_proxy(options, download_call); } else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); @@ -1079,10 +1098,24 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi function download_call(uri, options) { + var opt; options.length = 0; + if (options.proxy && !options.proxy.tls) { + opt = PROXYOPTIONS; + opt.port = options.proxy.port; + opt.host = options.proxy.hostname; + opt.path = uri.href; + opt.headers = uri.headers; + opt.method = uri.method; + + if (options.proxy._auth) + opt.headers['Proxy-Authorization'] = options.proxy._auth; + } else + opt = uri; + var connection = uri.protocol === 'https:' ? Https : Http; - var req = options.post ? connection.request(uri, (res) => download_response(res, uri, options)) : connection.get(uri, (res) => download_response(res, uri, options)); + var req = options.post ? connection.request(opt, (res) => download_response(res, uri, options)) : connection.get(opt, (res) => download_response(res, uri, options)); if (!options.callback) { req.on('error', NOOP); @@ -1090,21 +1123,21 @@ function download_call(uri, options) { } req.on('error', function(err) { - if (!options.callback) - return; - options.callback(err); - options.callback = null; - options.evt.removeAllListeners(); - options.evt = null; + if (options.callback) { + options.callback(err); + options.callback = null; + options.evt.removeAllListeners(); + options.evt = null; + } }); req.setTimeout(options.timeout, function() { - if (!options.callback) - return; - options.callback(new Error(exports.httpStatus(408))); - options.callback = null; - options.evt.removeAllListeners(); - options.evt = null; + if (options.callback) { + options.callback(new Error(exports.httpStatus(408))); + options.callback = null; + options.evt.removeAllListeners(); + options.evt = null; + } }); req.on('response', function(response) { @@ -1141,6 +1174,13 @@ function download_response(res, uri, options) { res.req.removeAllListeners(); res.req = null; + if (options.proxy && tmp.protocol === 'https:') { + // TLS? + options.uri = tmp; + download_call(options, request_call); + return + } + if (!options.resolve) { res.removeAllListeners(); res = null; @@ -5877,5 +5917,31 @@ if (NODEVERSION > 699) { exports.createBuffer = (val, type) => new Buffer(val || '', type); } +function Callback(count, callback) { + this.pending = count; + this.$callback = callback; +} + +Callback.prototype.done = function(callback) { + this.$callback = callback; + return this; +}; + +Callback.prototype.next = function() { + var self = this; + self.pending--; + if (!self.pending && self.$callback) { + self.$callback(); + self.$callback = null; + } + return self; +}; + +global.Callback = Callback; + +exports.Callback = function(count, callback) { + return new Callback(count, callback); +}; + global.WAIT = exports.wait; !global.F && require('./index'); \ No newline at end of file From 76c5d9face4b306415dfbbebdbc12647f2f6f008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 May 2018 19:30:58 +0200 Subject: [PATCH 0312/1669] New changes. --- changes.txt | 7 ++++ index.js | 94 ++++++++++++++++++++++++++++++++++------------------ package.json | 4 +-- utils.js | 2 +- 4 files changed, 72 insertions(+), 35 deletions(-) diff --git a/changes.txt b/changes.txt index 39113335c..23b80762f 100755 --- a/changes.txt +++ b/changes.txt @@ -37,6 +37,7 @@ - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: config `security.txt` for auto-generating security.txt content (more in docs) - added: config `default-proxy` for default web proxy server +- added: config `allow-cache` disables clearing temporary cache (default: `false`) - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` @@ -100,6 +101,12 @@ - fixed: `U.get()` a problem with path with `-` - fixed: `U.set()` a problem with path with `-` +- replaced: config `disable-clear-temporary-directory` to `allow-clear-temp : true|false` +- replaced: config `disable-strict-server-certificate-validation` to `allow-ssc-validation : true|false` +- replaced: config `default-websocket-request-length` to `default-websocket-maxlength` +- replaced: config `default-request-length` to `default-request-maxlength` +- replaced: config `default-maximum-file-descriptors` to `default-maxopenfiles` + - improved: `debug` mode timing with improved consumption - improved: performance (+20%) NoSQL embedded database - improved: reading performance (+5%) in `U.streamer()` diff --git a/index.js b/index.js index 90c9b4e63..ac9bd3dba 100755 --- a/index.js +++ b/index.js @@ -610,7 +610,6 @@ function Framework() { secret: Os.hostname() + '-' + Os.platform() + '-' + Os.arch(), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', - 'default-xpoweredby': 'Total.js', 'etag-version': '', 'directory-src': '/.src/', 'directory-bundles': '/bundles/', @@ -651,16 +650,17 @@ function Framework() { // 'static-accepts-custom': [], + 'default-xpoweredby': 'Total.js', 'default-layout': 'layout', 'default-theme': '', 'default-proxy': '', // default maximum request size / length // default 10 kB - 'default-request-length': 10, - 'default-websocket-request-length': 2, + 'default-request-maxlength': 10, + 'default-websocket-maxlength': 2, 'default-websocket-encodedecode': true, - 'default-maximum-file-descriptors': 0, + 'default-maxopenfiles': 0, 'default-timezone': '', 'default-root': '', 'default-response-maxage': '11111111', @@ -694,8 +694,8 @@ function Framework() { 'allow-debug': false, 'allow-head': false, 'allow-filter-errors': true, - 'disable-strict-server-certificate-validation': true, - 'disable-clear-temporary-directory': false, + 'allow-clear-temp': true, + 'allow-ssc-validation': false, 'nosql-worker': false, 'nosql-inmemory': null, // String Array 'nosql-cleaner': 1440, @@ -705,9 +705,9 @@ function Framework() { // All values are in minutes 'default-interval-clear-resources': 20, 'default-interval-clear-cache': 10, + 'default-interval-clear-dnscache': 120, 'default-interval-precompile-views': 61, 'default-interval-websocket-ping': 3, - 'default-interval-clear-dnscache': 120, 'default-interval-uptodate': 5 }; @@ -2207,7 +2207,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { r.flags2 = flags_to_object(flags); r.method = method; r.execute = funcExecute; - r.length = (length || F.config['default-request-length']) * 1024; + r.length = (length || F.config['default-request-maxlength']) * 1024; r.middleware = middleware; r.timeout = timeout === undefined ? (isDELAY ? 0 : F.config['default-request-timeout']) : timeout; r.isGET = flags.indexOf('get') !== -1; @@ -2853,7 +2853,7 @@ F.websocket = function(url, funcInitialize, flags, length) { r.onInitialize = funcInitialize; r.protocols = protocols || EMPTYARRAY; r.allow = allow || []; - r.length = (length || F.config['default-websocket-request-length']) * 1024; + r.length = (length || F.config['default-websocket-maxlength']) * 1024; r.isWEBSOCKET = true; r.MEMBER = membertype; r.isJSON = isJSON; @@ -5192,7 +5192,8 @@ F.$onParseXML = function(req) { * @return {Object} */ F.onParseJSON = function(value) { - return JSON.parse(value); + if (value) + return JSON.parse(value); }; F.onParseJSON.$def = true; @@ -6904,18 +6905,21 @@ F.custom = function(mode, http, request, response, options) { F.console = function() { var memory = process.memoryUsage(); console.log('===================================================='); - console.log('PID : ' + process.pid); - console.log('Node.js : ' + process.version); - console.log('Total.js : v' + F.version_header); - console.log('OS : ' + Os.platform() + ' ' + Os.release()); - F.config['nosql-worker'] && console.log('NoSQL PID : ' + framework_nosql.pid()); - console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); + console.log('PID : ' + process.pid); + console.log('Node.js : ' + process.version); + console.log('Total.js : v' + F.version_header); + console.log('OS : ' + Os.platform() + ' ' + Os.release()); + F.config['nosql-worker'] && console.log('NoSQL PID : ' + framework_nosql.pid()); + console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); + console.log('===================================================='); + console.log('Name : ' + F.config.name); + console.log('Version : ' + F.config.version); + console.log('Author : ' + F.config.author); + console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); + console.log('Mode : ' + (F.config.debug ? 'debug' : 'release')); console.log('===================================================='); - console.log('Name : ' + F.config.name); - console.log('Version : ' + F.config.version); - console.log('Author : ' + F.config.author); - console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); - console.log('Mode : ' + (F.config.debug ? 'debug' : 'release')); + console.log('Directory : ' + process.cwd()); + console.log('node_modules : ' + PATHMODULES); console.log('====================================================\n'); if (!F.isWorker) { console.log('{2}://{0}:{1}/'.format(F.ip, F.port, F.isHTTPS ? 'https' : 'http')); @@ -7828,12 +7832,11 @@ F.test = function() { * @return {Framework} */ F.clear = function(callback, isInit) { - var dir = F.path.temp(); var plus = F.id ? 'i-' + F.id + '_' : ''; if (isInit) { - if (F.config['disable-clear-temporary-directory']) { + if (!F.config['allow-clear-temp']) { // clears only JS and CSS files U.ls(dir, function(files) { F.unlink(files, function() { @@ -8686,16 +8689,27 @@ F.$configure_configs = function(arr, rewrite) { subtype = ''; switch (name) { - case 'default-cors-maxage': case 'default-request-length': + OBSOLETE(name, 'You need to use "default-request-maxlength"'); + obj['default-request-maxlength'] = U.parseInt(value); + break; case 'default-websocket-request-length': + OBSOLETE(name, 'You need to use "default-websocket-maxlength"'); + obj['default-websocket-maxlength'] = U.parseInt(value); + break; + case 'default-maximum-file-descriptors': + OBSOLETE(name, 'You need to use "default-maxopenfiles"'); + obj['default-maxopenfiles'] = U.parseInt(value); + break; + case 'default-cors-maxage': case 'default-request-timeout': + case 'default-request-maxlength': + case 'default-websocket-maxlength': case 'default-interval-clear-cache': case 'default-interval-clear-resources': case 'default-interval-precompile-views': case 'default-interval-uptodate': case 'default-interval-websocket-ping': - case 'default-maximum-file-descriptors': case 'default-interval-clear-dnscache': case 'default-dependency-timeout': case 'nosql-cleaner': @@ -8743,18 +8757,28 @@ F.$configure_configs = function(arr, rewrite) { obj['allow-static-files'] = true; break; + case 'disable-clear-temporary-directory': + OBSOLETE('disable-clear-temporary-directory', 'You need to use "allow-clear-temp : true|false"'); + obj['allow-clear-temp'] = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); + break; + + case 'disable-strict-server-certificate-validation': + OBSOLETE('disable-strict-server-certificate-validation', 'You need to use "allow-ssc-validation : true|false"'); + obj['allow-ssc-validation'] = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); + break; + case 'allow-compile-html': case 'allow-compile-script': case 'allow-compile-style': + case 'allow-ssc-validation': case 'allow-debug': case 'allow-gzip': case 'allow-performance': case 'allow-static-files': case 'allow-websocket': - case 'disable-strict-server-certificate-validation': - case 'disable-clear-temporary-directory': - case 'trace': + case 'allow-clear-temp': case 'allow-cache-snapshot': + case 'trace': case 'nosql-worker': case 'nosql-logger': obj[name] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; @@ -8829,7 +8853,7 @@ F.$configure_configs = function(arr, rewrite) { F.config['nosql-inmemory'] && F.config['nosql-inmemory'].forEach(n => framework_nosql.inmemory(n)); accepts && accepts.length && accepts.forEach(accept => F.config['static-accepts'][accept] = true); - if (F.config['disable-strict-server-certificate-validation'] === true) + if (F.config['allow-ssc-validation'] === false) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; if (F.config['allow-performance']) @@ -12057,10 +12081,16 @@ Controller.prototype.custom = function() { * @return {Controller} */ Controller.prototype.noClear = function(enable) { + OBSOLETE('controller.noClear()', 'You need to use controller.autoclear(false)'); this.req._manual = enable === undefined ? true : enable; return this; }; +Controller.prototype.autoclear = function(enable) { + this.req._manual = enable === true; + return this; +}; + Controller.prototype.html = function(body, headers) { return this.content(body, 'text/html', headers); }; @@ -16188,7 +16218,7 @@ process.on('uncaughtException', function(e) { }); function fsFileRead(filename, callback, a, b, c) { - U.queue('F.files', F.config['default-maximum-file-descriptors'], function(next) { + U.queue('F.files', F.config['default-maxopenfiles'], function(next) { Fs.readFile(filename, function(err, result) { next(); callback(err, result, a, b, c); @@ -16197,7 +16227,7 @@ function fsFileRead(filename, callback, a, b, c) { } function fsFileExists(filename, callback, a, b, c) { - U.queue('F.files', F.config['default-maximum-file-descriptors'], function(next) { + U.queue('F.files', F.config['default-maxopenfiles'], function(next) { Fs.lstat(filename, function(err, stats) { next(); callback(!err && stats.isFile(), stats ? stats.size : 0, stats ? stats.isFile() : false, stats, a, b, c); @@ -16226,7 +16256,7 @@ function fsStreamRead(filename, options, callback, res) { } else opt = HEADERS.fsStreamRead; - U.queue('F.files', F.config['default-maximum-file-descriptors'], function(next) { + U.queue('F.files', F.config['default-maxopenfiles'], function(next) { var stream = Fs.createReadStream(filename, opt); stream.on('error', NOOP); callback(stream, next, res); diff --git a/package.json b/package.json index 0d1e4d2f1..a7291408b 100755 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test": "test" }, "engines": { - "node": ">=4.0.0" + "node": ">=8.0.0" }, "keywords": ["total", "iot", "framework", "web", "websocket", "mvc", "controller", "view", "angular.js", "upload", "picture", "graphicsmagick", "imagemagick", "eshop", "blog", "forum", "chat", "game", "nosql", "database", "streaming", "live", "server sent events", "sse", "multipart", "x-mixed-replace"], "license": "MIT", @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-56", + "version": "3.0.0-57", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/utils.js b/utils.js index ff89d5216..08d43ea92 100755 --- a/utils.js +++ b/utils.js @@ -3326,7 +3326,7 @@ String.prototype.parseConfig = function(def, onerr) { if (!str || str[0] === '#' || str.substring(0, 2) === '//') continue; - index = str.indexOf(' :'); + index = str.indexOf(':'); if (index === -1) { index = str.indexOf('\t:'); if (index === -1) From 0a8fa57f7d0ac509f893c41f8be9ea35f5c71c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 07:05:52 +0200 Subject: [PATCH 0313/1669] Updated changelog. --- changes.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/changes.txt b/changes.txt index 23b80762f..1c5c41e9a 100755 --- a/changes.txt +++ b/changes.txt @@ -37,7 +37,6 @@ - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: config `security.txt` for auto-generating security.txt content (more in docs) - added: config `default-proxy` for default web proxy server -- added: config `allow-cache` disables clearing temporary cache (default: `false`) - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` From 7fa363eb8eb649cb430a0ddad056cc19f2fc738e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 12:54:56 +0200 Subject: [PATCH 0314/1669] Updated TLS connection. --- utils.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 08d43ea92..9b38f5cb2 100755 --- a/utils.js +++ b/utils.js @@ -84,6 +84,7 @@ const REGISARR = /\[\d+\]$/; const PROXYBLACKLIST = { 'localhost': 1, '127.0.0.1': 1, '0.0.0.0': 1 }; const PROXYOPTIONS = {}; const PROXYHEADERS = {}; +const PROXYTLS = {}; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -643,7 +644,13 @@ function request_proxy(options, callback) { req.on('connect', function(res, socket) { if (res.statusCode === 200) { - var tls = Tls.connect(0, { servername: options.uri.hostname, headers: options.uri.headers, socket: socket }); + + PROXYTLS.servername = options.uri.hostname; + PROXYTLS.headers = options.uri.headers || {}; + PROXYTLS.headers.host = options.uri.hostname; + PROXYTLS.socket = socket; + + var tls = Tls.connect(443, PROXYTLS); tls.on('secureConnect', function() { options.uri.agent = options.uri.protocol === 'http:' ? new Http.Agent() : new Https.Agent(); From 946874a0624c61bd27c3fcbca465e421d469fcc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 14:23:59 +0200 Subject: [PATCH 0315/1669] Improved proxy. --- utils.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils.js b/utils.js index 9b38f5cb2..cfecaa163 100755 --- a/utils.js +++ b/utils.js @@ -625,7 +625,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, function request_proxy(options, callback) { - PROXYHEADERS.host = options.uri.hostname; + PROXYHEADERS.host = options.uri.hostname + ':443'; var proxy = options.proxy; proxy.path = options.uri.hostname + ':443'; @@ -653,9 +653,10 @@ function request_proxy(options, callback) { var tls = Tls.connect(443, PROXYTLS); tls.on('secureConnect', function() { - options.uri.agent = options.uri.protocol === 'http:' ? new Http.Agent() : new Https.Agent(); - options.uri.agent.reuseSocket(tls, req); - //req.onSocket(tls); + var a = options.uri.agent = new Https.Agent(); + a.defaultPort = 443; + a.reuseSocket(tls, req); + req.onSocket(tls); options.socket = tls; options.proxy.tls = tls; callback(options.uri, options); From 2bc16aaa568bde84e20daf383977560b5d067ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 19:24:53 +0200 Subject: [PATCH 0316/1669] New changes. --- changes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changes.txt b/changes.txt index 1c5c41e9a..0d66357de 100755 --- a/changes.txt +++ b/changes.txt @@ -25,6 +25,7 @@ - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - added: `DatabaseBuilder.query(code)` can contain a raw JS condition in the form e.g. `doc.age > 18 && doc.age < 33` - added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) +- added: `Database.stream(fn, [repository], [callback(err, repository, count)])` for streaming documents - added: `Database.lock(callback(next))` locks all internal DB operations - added: `Database.ready(callback)` executes a callback when DB is ready to use (only for special cases if you use indexes) - added: new directory `schemas` with a new configuration item `directory-schemas' @@ -105,6 +106,7 @@ - replaced: config `default-websocket-request-length` to `default-websocket-maxlength` - replaced: config `default-request-length` to `default-request-maxlength` - replaced: config `default-maximum-file-descriptors` to `default-maxopenfiles` +- replaced: `controller.proxy()` functionality (the name remains) via `controller.proxy2()` functionality - improved: `debug` mode timing with improved consumption - improved: performance (+20%) NoSQL embedded database From 39014b3260325a342ea971ffd63923c8973a2d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 19:25:09 +0200 Subject: [PATCH 0317/1669] Replace `controller.proxy` for `controller.proxy2()`. --- index.js | 64 +++++------------------------------------------------ internal.js | 5 ----- 2 files changed, 6 insertions(+), 63 deletions(-) diff --git a/index.js b/index.js index ac9bd3dba..f166b811a 100755 --- a/index.js +++ b/index.js @@ -66,7 +66,6 @@ const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; const REG_UTF8 = /[^\x20-\x7E]+/; -const FLAGS_PROXY = ['post', 'json']; const FLAGS_INSTALL = ['get']; const FLAGS_DOWNLOAD = ['get', 'dnscache']; const QUERYPARSEROPTIONS = { maxKeys: 69 }; @@ -128,8 +127,6 @@ HEADERS.mmr[HEADER_CACHE] = 'private, no-cache, no-store, max-age=0'; HEADERS.mmr['Pragma'] = 'no-cache'; HEADERS.mmr['Expires'] = '-1'; HEADERS.mmr['X-Powered-By'] = 'Total.js'; -HEADERS.proxy = {}; -HEADERS.proxy['X-Proxy'] = 'total.js'; HEADERS.file_lastmodified = {}; HEADERS.file_lastmodified['Access-Control-Allow-Origin'] = '*'; HEADERS.file_lastmodified[HEADER_CACHE] = 'public, max-age=11111111'; @@ -2129,11 +2126,6 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (url.indexOf('#') !== -1) priority -= 100; - if (flags.indexOf('proxy') !== -1) { - isJSON = true; - priority++; - } - if ((isJSON || flags.indexOf('xml') !== -1 || isRaw) && (flags.indexOf('delete') === -1 && flags.indexOf('post') === -1 && flags.indexOf('put') === -1) && flags.indexOf('patch') === -1) { flags.push('post'); method += (method ? ',' : '') + 'post'; @@ -2228,7 +2220,6 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { r.isHTTP = flags.indexOf('http') !== -1; r.isDEBUG = flags.indexOf('debug') !== -1; r.isRELEASE = flags.indexOf('release') !== -1; - r.isPROXY = flags.indexOf('proxy') !== -1; r.isBOTH = isNOXHR ? false : true; r.isXHR = flags.indexOf('xhr') !== -1; r.isUPLOAD = flags.indexOf('upload') !== -1; @@ -7236,12 +7227,6 @@ F.$requestcontinue = function(req, res, headers) { var flags = [req.method.toLowerCase()]; var multipart; - if (headers['x-proxy'] === 'total.js') { - req.isProxy = true; - req.$flags += 'f'; - flags.push('proxy'); - } - if (F._request_check_mobile && req.mobile) { req.$flags += 'a'; F.stats.request.mobile++; @@ -9927,10 +9912,6 @@ Controller.prototype = { return F.controllers; }, - get isProxy() { - return this.req.isProxy === true; - }, - get isDebug() { return F.config.debug; }, @@ -12579,40 +12560,6 @@ Controller.prototype.close = function(end) { return self; }; -/** - * Sends an object to another total.js application (POST + JSON) - * @param {String} url - * @param {Object} obj - * @param {Function(err, data, code, headers)} callback - * @param {Number} timeout Timeout, optional default 10 seconds. - * @return {EventEmitter} - */ -Controller.prototype.proxy = function(url, obj, callback, timeout) { - - var self = this; - var tmp; - - if (typeof(callback) === 'number') { - tmp = timeout; - timeout = callback; - callback = tmp; - } - - if (typeof(obj) === 'function') { - tmp = callback; - callback = obj; - obj = tmp; - } - - return U.request(url, FLAGS_PROXY, obj, function(err, data, code, headers) { - if (!callback) - return; - if ((headers['content-type'] || '').lastIndexOf('/json') !== -1) - data = F.onParseJSON(data); - callback.call(self, err, data, code, headers); - }, null, HEADERS.proxy, ENCODING, timeout || 10000); -}; - /** * Creates a proxy between current request and new URL * @param {String} url @@ -12621,7 +12568,7 @@ Controller.prototype.proxy = function(url, obj, callback, timeout) { * @param {Number} timeout Optional, timeout (default: 10000) * @return {EventEmitter} */ -Controller.prototype.proxy2 = function(url, callback, headers, timeout) { +Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callback, headers, timeout) { if (typeof(callback) === 'object') { timeout = headers; @@ -12687,6 +12634,7 @@ Controller.prototype.proxy2 = function(url, callback, headers, timeout) { self.status = code; callback && callback(err, data, code, headers); self.content(data, (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, '')); + }, null, h, ENCODING, timeout || 10000); }; @@ -15272,10 +15220,10 @@ function extend_response(PROTO) { }); client.on('close', function() { - if (res.success) - return; - F.stats.response.pipe++; - response_end(res); + if (!res.success) { + F.stats.response.pipe++; + response_end(res); + } }); }); diff --git a/internal.js b/internal.js index 64b2f7c5f..dd455056b 100755 --- a/internal.js +++ b/internal.js @@ -683,11 +683,6 @@ exports.routeCompareFlags2 = function(req, route, membertype) { continue; return 0; - case 'proxy': - if (!route.isPROXY) - return 0; - continue; - case 'debug': if (!route.isDEBUG && route.isRELEASE) return 0; From 4535a2eb48852b2162d8793636f24b05be0c9e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 19:25:17 +0200 Subject: [PATCH 0318/1669] Added `NoSQL.stream()`. --- nosql.js | 22 ++++++++++++++++++++-- nosqlworker.js | 10 ++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 33fc35bd4..4d3757a45 100755 --- a/nosql.js +++ b/nosql.js @@ -139,6 +139,9 @@ exports.worker = function() { case 'clear': item.callback && item.callback(err); break; + case 'stream': + item.callback && item.callback(err, EMPTYOBJECT, 0); + break; default: item.callback && item.callback(err, EMPTYOBJECT, EMPTYOBJECT); break; @@ -187,6 +190,10 @@ exports.worker = function() { var obj = FORKCALLBACKS[msg.id]; obj && obj.callback && obj.callback(msg.err, msg.response); break; + case 'stream': + var obj = FORKCALLBACKS[msg.id]; + obj && obj.callback && obj.callback(msg.err, msg.response || {}, msg.count); + break; case 'storage.scan': var obj = FORKCALLBACKS[msg.id]; obj && obj.callback && obj.callback(msg.err, msg.response, msg.repository); @@ -368,6 +375,17 @@ exports.worker = function() { return send(this, 'remove', filename).builder = new DatabaseBuilder(this); }; + Database.prototype.stream = function(fn, repository, callback) { + + if (typeof(repository) === 'function') { + callback = repository; + repository = undefined; + } + + send(this, 'stream', fn.toString(), repository).callback = callback; + return this; + }; + Counter.prototype.min = function(id, count) { notify(this.db, 'counter.min', id, count); return this; @@ -992,7 +1010,7 @@ Database.prototype.find2 = function(builder) { return builder; }; -Database.prototype.streamer = function(fn, repository, callback) { +Database.prototype.stream = function(fn, repository, callback) { var self = this; if (typeof(repository) === 'function') { @@ -5479,7 +5497,7 @@ Indexes.prototype.reindex = function(callback) { self.clear(function() { self.db.$events['indexing-begin'] && self.db.emit('indexing-begin'); var chunker = U.chunker(self.db.name + '_reindex', 10000); - self.db.streamer(function(doc) { + self.db.stream(function(doc) { chunker.write(doc); }, function(err, repository, count) { chunker.end(); diff --git a/nosqlworker.js b/nosqlworker.js index 530fd3e52..632c6ee1b 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -43,6 +43,7 @@ const RESSTORAGECLEAR = { TYPE: 'storage.clear' }; const RESINDEXESGET = { TYPE: 'indexes.get' }; const RESINDEXESCLEAR = { TYPE: 'indexes.clear' }; const RESINDEXESREINDEX = { TYPE: 'indexes.reindex' }; +const RESSTREAM = { TYPE: 'stream' }; function killprocess() { process.exit(0); @@ -271,6 +272,15 @@ process.on('message', function(msg) { case 'indexes.noreindex': db.indexes.noreindex(); break; + case 'stream': + db[msg.TYPE](eval('(' + msg.arg[0] + ')'), msg.arg[1], function(err, repository, count) { + RESSTREAM.id = msg.id; + RESSTREAM.err = err; + RESSTREAM.repository = repository; + RESSTREAM.count = count; + process.send(RESSTREAM); + }); + break; case 'clean': case 'clear': case 'ready': From 06d06232999b3a65be936cbd10aa64d969fc5d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 19:25:44 +0200 Subject: [PATCH 0319/1669] Improved hostname resolving. --- utils.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utils.js b/utils.js index cfecaa163..dbbaf55f4 100755 --- a/utils.js +++ b/utils.js @@ -603,6 +603,9 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, uri.headers = headers; options.uri = uri; + if (options.resolve && (uri.hostname === 'localhost' || uri.hostname.charCodeAt(0) < 64)) + options.resolve = null; + if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) proxy = parseProxy(F.config['default-proxy']); @@ -820,7 +823,7 @@ function request_response(res, uri, options) { // TLS? options.uri = tmp; request_proxy(options, request_call); - return + return; } if (!options.resolve) { @@ -1079,6 +1082,9 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi uri.headers = headers; options.uri = uri; + if (options.resolve && (uri.hostname === 'localhost' || uri.hostname.charCodeAt(0) < 64)) + options.resolve = null; + if (data.length) { options.data = exports.createBuffer(data, ENCODING); headers['Content-Length'] = options.data.length; From 6b834215e111878ea042a71904000753bc93852c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 19:34:29 +0200 Subject: [PATCH 0320/1669] Fixed callback. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 4d3757a45..22c84029d 100755 --- a/nosql.js +++ b/nosql.js @@ -192,7 +192,7 @@ exports.worker = function() { break; case 'stream': var obj = FORKCALLBACKS[msg.id]; - obj && obj.callback && obj.callback(msg.err, msg.response || {}, msg.count); + obj && obj.callback && obj.callback(msg.err, msg.repository || {}, msg.count); break; case 'storage.scan': var obj = FORKCALLBACKS[msg.id]; From 9cc4c474ba0730a010f13522a965a6e2eb4e892b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 22:48:33 +0200 Subject: [PATCH 0321/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a7291408b..54f829452 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-57", + "version": "3.0.0-58", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 213765582d060d328eafb8488a3409e794885ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 May 2018 23:43:17 +0200 Subject: [PATCH 0322/1669] Added new search methods. --- changes.txt | 2 ++ nosql.js | 47 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 0d66357de..f35fb1894 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,8 @@ - added: `route.groups` with defined groups - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - added: `DatabaseBuilder.query(code)` can contain a raw JS condition in the form e.g. `doc.age > 18 && doc.age < 33` +- added: `DatabaseBuilder.regexp(name, regexp)` RegExp search in strings +- added: `DatabaseBuilder.fulltext(name, regexp, [weight])` full text search in strings, more info in docs. - added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) - added: `Database.stream(fn, [repository], [callback(err, repository, count)])` for streaming documents - added: `Database.lock(callback(next))` locks all internal DB operations diff --git a/nosql.js b/nosql.js index 22c84029d..61659c496 100755 --- a/nosql.js +++ b/nosql.js @@ -3214,21 +3214,54 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam code = '$is=doc.{0}?doc.{0}.endsWith(arg.{1}):false;'; break; case '*': - code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){if(doc.{0} instanceof Array){for(var $i=0;$i> 0; + + var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Wed, 16 May 2018 11:12:08 +0200 Subject: [PATCH 0323/1669] Improved default values in Schemas. --- builders.js | 50 ++++++++++++++++++++++++++++++++++++++------------ changes.txt | 1 + 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/builders.js b/builders.js index ee4ba73a0..d9e49420b 100755 --- a/builders.js +++ b/builders.js @@ -321,7 +321,7 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { if (type instanceof SchemaBuilderEntity) type = type.name; - this.schema[name] = this.$parse(name, type, required, custom); + var a = this.schema[name] = this.$parse(name, type, required, custom); switch (this.schema[name].type) { case 7: @@ -339,7 +339,10 @@ SchemaBuilderEntity.prototype.define = function(name, type, required, custom) { else this.properties = this.properties.remove(name); - return this; + return function(val) { + a.def = val; + return this; + }; }; SchemaBuilderEntity.prototype.inherit = function(group, name) { @@ -1457,6 +1460,11 @@ SchemaBuilderEntity.prototype.default = function() { } } + if (type.def !== undefined) { + item[property] = typeof(type.def) === 'function' ? type.def() : type.def; + continue; + } + switch (type.type) { // undefined // object @@ -1613,18 +1621,24 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { for (var property in obj) { var val = model[property]; + var type = obj[property]; // IS PROTOTYPE? The problem was in e.g. "search" property, because search is in String prototypes. if (!hasOwnProperty.call(model, property)) val = undefined; - if (val === undefined && defaults) - val = self.$ondefault(property, false, self.name); + var def = type.def != undefined ? typeof(type.def) === 'function' ? type.def() : type.def : undefined; + + if (val === undefined) { + if (type.def !== undefined) + val = def; + else if (defaults) + val = self.$ondefault(property, false, self.name); + } if (val === undefined) val = ''; - var type = obj[property]; var typeval = typeof(val); if (typeval === 'function') @@ -1638,11 +1652,11 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { break; // number: integer case 1: - item[property] = self.$onprepare(property, framework_utils.parseInt(val), undefined, model); + item[property] = self.$onprepare(property, framework_utils.parseInt(val, def || 0), undefined, model); break; // number: float case 2: - item[property] = self.$onprepare(property, framework_utils.parseFloat(val), undefined, model); + item[property] = self.$onprepare(property, framework_utils.parseFloat(val, def || 0), undefined, model); break; // string @@ -1701,12 +1715,19 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { break; } + if (type.def && !tmp) + tmp = def; + item[property] = self.$onprepare(property, tmp, undefined, model); break; // boolean case 4: tmp = val ? val.toString().toLowerCase() : null; + + if (type.def && (tmp == null || tmp === '')) + tmp = def; + item[property] = self.$onprepare(property, tmp === 'true' || tmp === '1' || tmp === 'on', undefined, model); break; @@ -1725,14 +1746,19 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { if (framework_utils.isDate(tmp)) tmp = self.$onprepare(property, tmp, undefined, model); - else - tmp = (defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null); + else { + if (type.def !== undefined) + tmp = def; + else + tmp = (defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null); + } item[property] = tmp; break; // object case 6: + item[property] = self.$onprepare(property, model[property], undefined, model); if (item[property] === undefined) @@ -1758,8 +1784,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { case 7: if (!val) { - val = (defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null); - // val = defaults(property, false, self.name); + val = (type.def === undefined ? defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null : def); if (val === null) { item[property] = null; break; @@ -1780,6 +1805,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { dependencies && dependencies.push({ name: type.raw, value: self.$onprepare(property, item[property], undefined, model) }); } else item[property] = null; + break; } continue; @@ -1787,7 +1813,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { // ARRAY: if (!(val instanceof Array)) { - item[property] = (defaults ? isUndefined(self.$ondefault(property, false, self.name), EMPTYARRAY) : []); + item[property] = (type.def === undefined ? defaults ? isUndefined(self.$ondefault(property, false, self.name), EMPTYARRAY) : [] : def); continue; } diff --git a/changes.txt b/changes.txt index f35fb1894..fc1227961 100755 --- a/changes.txt +++ b/changes.txt @@ -66,6 +66,7 @@ - added: `RESTBuilder.proxy(proxy)` for HTTP proxy - added: `U.request()` supports a new flag `proxy`, for example `proxy 127.0.0.1:8080` - added: NoSQL database a new event `change`, more in docs +- added: `schema.define()(DEFAULT_VALUE)` added `DEFAULT_VALUE` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` From a1f23bb1f539610f654c062c6e742f9a9a6678ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 16 May 2018 11:30:59 +0200 Subject: [PATCH 0324/1669] Added missing header. --- utils.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.js b/utils.js index dbbaf55f4..3a547b0b3 100755 --- a/utils.js +++ b/utils.js @@ -686,6 +686,7 @@ function request_call(uri, options) { opt.path = uri.href; opt.headers = uri.headers; opt.method = uri.method; + opt.headers.host = options.uri.host; if (options.proxy._auth) opt.headers['Proxy-Authorization'] = options.proxy._auth; @@ -812,6 +813,7 @@ function request_response(res, uri, options) { options.redirect++; var tmp = Url.parse(res.headers['location']); + tmp.headers = uri.headers; tmp.agent = false; tmp.method = uri.method; From ef651649cf952f8fac2df1d021998c165b92d638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 16 May 2018 13:32:36 +0200 Subject: [PATCH 0325/1669] Improved default values. --- builders.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/builders.js b/builders.js index d9e49420b..f750434a8 100755 --- a/builders.js +++ b/builders.js @@ -1627,11 +1627,11 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { if (!hasOwnProperty.call(model, property)) val = undefined; - var def = type.def != undefined ? typeof(type.def) === 'function' ? type.def() : type.def : undefined; + var def = type.def != undefined ? typeof(type.def) === 'function' ? true : false : undefined; if (val === undefined) { if (type.def !== undefined) - val = def; + val = def ? type.def() : type.def; else if (defaults) val = self.$ondefault(property, false, self.name); } @@ -1652,11 +1652,11 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { break; // number: integer case 1: - item[property] = self.$onprepare(property, framework_utils.parseInt(val, def || 0), undefined, model); + item[property] = self.$onprepare(property, framework_utils.parseInt(val, (def ? type.def() : type.def) || 0), undefined, model); break; // number: float case 2: - item[property] = self.$onprepare(property, framework_utils.parseFloat(val, def || 0), undefined, model); + item[property] = self.$onprepare(property, framework_utils.parseFloat(val, (def ? type.def() : type.def) || 0), undefined, model); break; // string @@ -1716,7 +1716,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { } if (type.def && !tmp) - tmp = def; + tmp = def ? type.def() : type.def; item[property] = self.$onprepare(property, tmp, undefined, model); break; @@ -1726,7 +1726,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { tmp = val ? val.toString().toLowerCase() : null; if (type.def && (tmp == null || tmp === '')) - tmp = def; + tmp = def ? type.def() : type.def; item[property] = self.$onprepare(property, tmp === 'true' || tmp === '1' || tmp === 'on', undefined, model); break; @@ -1748,7 +1748,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { tmp = self.$onprepare(property, tmp, undefined, model); else { if (type.def !== undefined) - tmp = def; + tmp = def ? type.def() : type.def; else tmp = (defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null); } @@ -1784,7 +1784,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { case 7: if (!val) { - val = (type.def === undefined ? defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null : def); + val = (type.def === undefined ? defaults ? isUndefined(self.$ondefault(property, false, self.name), null) : null : (def ? type.def() : type.def)); if (val === null) { item[property] = null; break; @@ -1813,7 +1813,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { // ARRAY: if (!(val instanceof Array)) { - item[property] = (type.def === undefined ? defaults ? isUndefined(self.$ondefault(property, false, self.name), EMPTYARRAY) : [] : def); + item[property] = (type.def === undefined ? defaults ? isUndefined(self.$ondefault(property, false, self.name), EMPTYARRAY) : [] : (def ? type.def() : type.def)); continue; } From 9405c11d8c70575e7f9a8d853a161fbe09586023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 16 May 2018 15:08:09 +0200 Subject: [PATCH 0326/1669] Added port. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 3a547b0b3..cdc226fdc 100755 --- a/utils.js +++ b/utils.js @@ -650,7 +650,7 @@ function request_proxy(options, callback) { PROXYTLS.servername = options.uri.hostname; PROXYTLS.headers = options.uri.headers || {}; - PROXYTLS.headers.host = options.uri.hostname; + PROXYTLS.headers.host = options.uri.hostname + ':' + (options.uri.port || '443'); PROXYTLS.socket = socket; var tls = Tls.connect(443, PROXYTLS); From b8ffcaccc1628e5209333a378f736b500e829a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 17 May 2018 07:48:06 +0200 Subject: [PATCH 0327/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 54f829452..8af779d4c 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-58", + "version": "3.0.0-59", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 909e602e2bdf0654e457367b8468156135981f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 17 May 2018 08:20:10 +0200 Subject: [PATCH 0328/1669] Fixed `builder.search()`. --- nosql.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nosql.js b/nosql.js index 61659c496..121136e00 100755 --- a/nosql.js +++ b/nosql.js @@ -3215,6 +3215,8 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam break; case '*': code = '$is=false;if(doc.{0}){if(doc.{0} instanceof Array){for(var $i=0;$i Date: Fri, 18 May 2018 12:18:02 +0200 Subject: [PATCH 0329/1669] Improved `SchemaOptions`. --- builders.js | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ changes.txt | 11 +++++++++++ 2 files changed, 61 insertions(+) diff --git a/builders.js b/builders.js index f750434a8..2a3055b24 100755 --- a/builders.js +++ b/builders.js @@ -95,6 +95,56 @@ SchemaOptions.prototype.clean = function() { return this.model.$clean(); }; +SchemaOptions.prototype.$workflow = function(name, helper, callback, async) { + this.model.$workflow(name, helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$transform = function(name, helper, callback, async) { + this.model.$transform(name, helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$operation = function(name, helper, callback, async) { + this.model.$operation(name, helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$hook = function(name, helper, callback, async) { + this.model.$hook(name, helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$save = function(helper, callback, async) { + this.model.$save(helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$insert = function(helper, callback, async) { + this.model.$insert(helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$update = function(helper, callback, async) { + this.model.$update(helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$query = function(helper, callback, async) { + this.model.$query(helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$delete = SchemaOptions.prototype.$remove = function(helper, callback, async) { + this.model.$remove(helper, callback, async); + return this; +}; + +SchemaOptions.prototype.$get = SchemaOptions.prototype.$read = function(helper, callback, async) { + this.model.$get(helper, callback, async); + return this; +}; + SchemaOptions.prototype.push = function(type, name, helper, first) { return this.model.$push(type, name, helper, first); }; diff --git a/changes.txt b/changes.txt index fc1227961..9d878dccb 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,17 @@ - added: `SchemaOptions.output()` alias to `$.model.$output()` - added: `SchemaOptions.clean()` alias to `$.model.$clean()` - added: `SchemaOptions.response()` alias to `$.model.$response([index])` +- added: `SchemaOptions.$get([options], [callback])` alias to `$.model.$get()` +- added: `SchemaOptions.$insert([options], [callback])` alias to `$.model.$insert()` +- added: `SchemaOptions.$query([options], [callback])` alias to `$.model.$query()` +- added: `SchemaOptions.$remove([options], [callback])` alias to `$.model.$remove()` +- added: `SchemaOptions.$save([options], [callback])` alias to `$.model.$save()` +- added: `SchemaOptions.$update([options], [callback])` alias to `$.model.$update()` +- added: `SchemaOptions.$workflow(name, [options], [callback])` alias to `$.model.$workflow()` +- added: `SchemaOptions.$transform(name, [options], [callback])` alias to `$.model.$transform()` +- added: `SchemaOptions.$operation(name, [options], [callback])` alias to `$.model.$operation()` +- added: `SchemaOptions.$hook(name, [options], [callback])` alias to `$.model.$hook()` +- added: `SchemaOptions.stop()` alias to `$.model.$stop()` - added: a new route flag type `&group` something like `roles` but groups aren't evaluated - added: `route.groups` with defined groups - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode From ffb1591793946edf30cc2cc4f046d943d3c1e7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 12:18:13 +0200 Subject: [PATCH 0330/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8af779d4c..b32260f63 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-59", + "version": "3.0.0-60", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0383bd55bed833530093593cf4c72cba56c92d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 12:21:14 +0200 Subject: [PATCH 0331/1669] Added `SchemaOptions.$async()`. --- builders.js | 5 +++++ changes.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/builders.js b/builders.js index 2a3055b24..85613d6aa 100755 --- a/builders.js +++ b/builders.js @@ -95,6 +95,11 @@ SchemaOptions.prototype.clean = function() { return this.model.$clean(); }; +SchemaOptions.prototype.$async = function(callback, index) { + this.model.$async(callback, index); + return this; +}; + SchemaOptions.prototype.$workflow = function(name, helper, callback, async) { this.model.$workflow(name, helper, callback, async); return this; diff --git a/changes.txt b/changes.txt index 9d878dccb..2aa22337d 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - added: `SchemaOptions.output()` alias to `$.model.$output()` - added: `SchemaOptions.clean()` alias to `$.model.$clean()` - added: `SchemaOptions.response()` alias to `$.model.$response([index])` +- added: `SchemaOptions.$async(callback, [index])` alias to `$.model.$async()` - added: `SchemaOptions.$get([options], [callback])` alias to `$.model.$get()` - added: `SchemaOptions.$insert([options], [callback])` alias to `$.model.$insert()` - added: `SchemaOptions.$query([options], [callback])` alias to `$.model.$query()` From 200064f0c8ecb5b85efb6435fadbdcbd11f811dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 13:32:39 +0200 Subject: [PATCH 0332/1669] Improved unit-testing. --- changes.txt | 1 + index.js | 3 --- package.json | 2 +- test.js | 12 ++++++++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 2aa22337d..7bea5b1e5 100755 --- a/changes.txt +++ b/changes.txt @@ -79,6 +79,7 @@ - added: `U.request()` supports a new flag `proxy`, for example `proxy 127.0.0.1:8080` - added: NoSQL database a new event `change`, more in docs - added: `schema.define()(DEFAULT_VALUE)` added `DEFAULT_VALUE` +- added: `TESTUSER([user])` for faking of `F.onAuthorize` delegate, targeted for unit-testing only - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index f166b811a..1d11547e6 100755 --- a/index.js +++ b/index.js @@ -553,7 +553,6 @@ global.OBSOLETE = function(name, message) { }; global.DEBUG = false; -global.TEST = false; global.RELEASE = false; global.is_client = false; global.is_server = true; @@ -14829,10 +14828,8 @@ function extend_request(PROTO) { }; PROTO.$total_prepare = function() { - var req = this; var length = req.flags.length; - if (F.onAuthorize) { F.onAuthorize(req, req.res, req.flags, function(isAuthorized, user) { var hasRoles = length !== req.flags.length; diff --git a/package.json b/package.json index b32260f63..d255c6cfd 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-60", + "version": "3.0.0-61", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/test.js b/test.js index 48cf3c99e..3cc1bb198 100644 --- a/test.js +++ b/test.js @@ -129,6 +129,18 @@ global.OK = function(is, description) { T.immediate = setImmediate(NEXT); }; +global.TESTUSER = function(user) { + if (!T.auth) + T.auth = F.onAuthorize; + T.user = user; + if (user) { + F.onAuthorize = function(req, res, flags, next) { + next(true, F.tests.user); + }; + } else + F.onAuthorize = T.auth; +}; + exports.load = function() { U.ls(F.path.tests(), function(files) { files.waitFor(function(filename, next) { From c51df2b4ce4c8322e33db74d806229df2cd1fd18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 13:34:21 +0200 Subject: [PATCH 0333/1669] Fixed type. --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 1d11547e6..9d0bdce66 100755 --- a/index.js +++ b/index.js @@ -553,6 +553,7 @@ global.OBSOLETE = function(name, message) { }; global.DEBUG = false; +global.TEST = false; global.RELEASE = false; global.is_client = false; global.is_server = true; From 463cc7a2c898ae41cdd2a5834d9cc0b0a6fd743b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 19:17:04 +0200 Subject: [PATCH 0334/1669] Fixed `inspector`. --- debug.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debug.js b/debug.js index 9d7d74096..0a7e9cf88 100644 --- a/debug.js +++ b/debug.js @@ -319,6 +319,10 @@ function runwatching() { { isSkip = true; process.kill(app.pid); + if (options.inspector) { + setTimeout(restart, 1000); + return; + } } catch (err) {} app = null; } @@ -350,6 +354,7 @@ function runwatching() { app.on('message', function(msg) { switch (msg) { case 'total:eaddrinuse': + console.log('OK2'); process.exit(1); break; case 'total:ready': @@ -366,6 +371,7 @@ function runwatching() { // checks unexpected exit if (isSkip === false) { app = null; + console.log('OK'); process.exit(); return; } From cd3e68180d8148c67da0d75d25fd6b9220768276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 19:57:41 +0200 Subject: [PATCH 0335/1669] Fixed `livereloading`. --- debug.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/debug.js b/debug.js index 0a7e9cf88..d7363e4f8 100644 --- a/debug.js +++ b/debug.js @@ -283,9 +283,8 @@ function runwatching() { isLoaded = true; - reload && livereload(); - if (status !== 1 || !force) { + reload && livereload(); if (counter % 150 === 0) speed = isRELOAD ? 3000 : 6000; setTimeout(refresh_directory, speed); @@ -303,7 +302,6 @@ function runwatching() { changes = []; force = false; - livereload(); }, 3); } @@ -354,7 +352,6 @@ function runwatching() { app.on('message', function(msg) { switch (msg) { case 'total:eaddrinuse': - console.log('OK2'); process.exit(1); break; case 'total:ready': @@ -362,6 +359,7 @@ function runwatching() { app.send('total:debug'); status = 1; } + livereload(); break; } }); @@ -371,7 +369,6 @@ function runwatching() { // checks unexpected exit if (isSkip === false) { app = null; - console.log('OK'); process.exit(); return; } From 7b4d3dff2956e8526b7cddf7ab5a83244d4df4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 20:04:44 +0200 Subject: [PATCH 0336/1669] Improved code. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 9d0bdce66..6e93e5c53 100755 --- a/index.js +++ b/index.js @@ -1285,7 +1285,7 @@ F.stop = F.kill = function(signal) { F.cache.stop(); F.server && F.server.close && F.server.close(); - setTimeout(() => process.exit(signal), TEST ? 2000 : 100); + setTimeout(() => process.exit(signal), global.TEST ? 2000 : 100); return F; }; From 328be9fb32d7091b8d43c1e44c32b24011b1de04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 18 May 2018 20:05:03 +0200 Subject: [PATCH 0337/1669] Improved unit-testing. --- test.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test.js b/test.js index 3cc1bb198..46ae42b47 100644 --- a/test.js +++ b/test.js @@ -118,15 +118,27 @@ global.TEST = function(name, url, scope) { }; global.FAIL = function(is, description) { - logger(is ? true : false, T.currentname, description); - T.immediate && clearImmediate(T.immediate); - T.immediate = setImmediate(NEXT); + if (arguments.length) { + logger(is ? true : false, T.currentname, description); + T.immediate && clearImmediate(T.immediate); + T.immediate = setImmediate(NEXT); + } else { + return function(err) { + FAIL(err == null); + }; + } }; global.OK = function(is, description) { - logger(is ? false : true, T.currentname, description); - T.immediate && clearImmediate(T.immediate); - T.immediate = setImmediate(NEXT); + if (arguments.length) { + logger(is ? false : true, T.currentname, description); + T.immediate && clearImmediate(T.immediate); + T.immediate = setImmediate(NEXT); + } else { + return function(err) { + OK(err == null); + }; + } }; global.TESTUSER = function(user) { From 2789eba3ed87da0907c092ad2b079c22503eee17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 19 May 2018 09:22:30 +0200 Subject: [PATCH 0338/1669] Extended `TESTUSER()` by adding `flags` argument. --- test.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test.js b/test.js index 46ae42b47..55d4a2b3b 100644 --- a/test.js +++ b/test.js @@ -141,12 +141,26 @@ global.OK = function(is, description) { } }; -global.TESTUSER = function(user) { +global.TESTUSER = function(user, flags) { + if (!T.auth) T.auth = F.onAuthorize; + T.user = user; + T.flags = flags; + if (user) { F.onAuthorize = function(req, res, flags, next) { + + if (T.flags && T.flags.length) { + for (var i = 0; i < T.flags.length; i++) { + var f = T.flags[i]; + if (f[0] !== '@') + f = '@' + f; + flags.push(f); + } + } + next(true, F.tests.user); }; } else From 58c13710e7ec7070902cc86c4bdcf905e12591dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 20 May 2018 23:56:21 +0200 Subject: [PATCH 0339/1669] Improved CSS compressor. --- internal.js | 139 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/internal.js b/internal.js index dd455056b..56afbbd92 100755 --- a/internal.js +++ b/internal.js @@ -76,9 +76,11 @@ const REG_CSS_8 = /\s\{/g; const REG_CSS_9 = /;\}/g; const REG_CSS_10 = /\$[a-z0-9-_]+:.*?;/gi; const REG_CSS_11 = /\$[a-z0-9-_]+/gi; +const REG_CSS_12 = /(margin|padding|border-radius):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; const EMPTYBUFFER = framework_utils.createBufferSize(0); +const ALLOWEDMARKUP = { G: 1, M: 1, R: 1, repository: 1, model: 1, config: 1, global: 1, resource: 1, RESOURCE: 1, CONFIG: 1, author: 1, root: 1, functions: 1, NOW: 1, F: 1 }; var INDEXFILE = 0; var INDEXMIXED = 0; @@ -905,13 +907,58 @@ function compile_autovendor(css) { var isAuto = css.substring(0, 100).indexOf(avp) !== -1; if (isAuto) css = autoprefixer(css.replace(avp, '')); + return css.replace(REG_CSS_1, '').replace(REG_CSS_2, '{').replace(REG_CSS_3, '}').replace(REG_CSS_4, ':').replace(REG_CSS_5, ';').replace(REG_CSS_6, function(search, index, text) { for (var i = index; i > 0; i--) { if ((text[i] === '\'' || text[i] === '"') && (text[i - 1] === ':')) return search; } return ','; - }).replace(REG_CSS_7, '}').replace(REG_CSS_8, '{').replace(REG_CSS_9, '}').trim(); + }).replace(REG_CSS_7, '}').replace(REG_CSS_8, '{').replace(REG_CSS_9, '}').replace(REG_CSS_12, cssmarginpadding).trim(); +} + +function cssmarginpadding(text) { + + // margin + // padding + + var prop = ''; + var val; + var l = text.length - 1; + var last = text[l]; + + if (text[0] === 'm') { + prop = 'margin:'; + val = text.substring(7, l); + } else { + prop = 'padding:'; + val = text.substring(8, l); + } + + var a = val.split(' '); + + for (var i = 0; i < a.length; i++) { + if (a[i][0] === '0' && a[i].charCodeAt(1) > 58) + a[i] = '0'; + } + + // 0 0 0 0 --> 0 + if (a[0] === '0' && a[1] === '0' && a[2] === '0' && a[3] === '0') + return prop + '0' + last; + + // 20px 0 0 0 --> 20px 0 0 + if (a[0] !== '0' && a[1] === '0' && a[2] === '0' && a[3] === '0') + return prop + a[0] + ' 0 0' + last; + + // 20px 30px 20px 30px --> 20px 30px + if (a[1] && a[2] && a[3] && a[0] === a[3] && a[1] === a[2]) + return prop + a[0] + ' ' + a[1] + last; + + // 20px 30px 10px 30px --> 20px 30px 10px + if (a[2] && a[3] && a[1] === a[3] && a[0] !== a[2]) + return prop + a[0] + ' ' + a[1] + ' ' + a[2] + last; + + return text; } function autoprefixer(value) { @@ -945,7 +992,8 @@ function autoprefixer(value) { continue; // text-transform - var isPrefix = value.substring(index - 1, index) === '-'; + var before = value.substring(index - 1, index); + var isPrefix = before === '-'; if (isPrefix) continue; @@ -955,7 +1003,7 @@ function autoprefixer(value) { if (end === -1 || css.substring(0, end + 1).replace(/\s/g, '') !== property + ':') continue; - builder.push({ name: property, property: css }); + builder.push({ name: property, property: before + css, css: css }); } } @@ -965,7 +1013,10 @@ function autoprefixer(value) { for (var i = 0; i < length; i++) { var name = builder[i].name; - property = builder[i].property; + var replace = builder[i].property; + var before = replace[0]; + + property = builder[i].css.trim(); var plus = property; var delimiter = ';'; @@ -973,11 +1024,11 @@ function autoprefixer(value) { if (name === 'opacity') { var opacity = +plus.replace('opacity', '').replace(':', '').replace(/\s/g, ''); - if (isNaN(opacity)) - continue; - updated += 'filter:alpha(opacity=' + Math.floor(opacity * 100) + ')'; - value = value.replacer(property, '@[[' + output.length + ']]'); - output.push(updated); + if (!isNaN(opacity)) { + updated += 'filter:alpha(opacity=' + Math.floor(opacity * 100) + ')'; + value = value.replacer(replace, before + '@[[' + output.length + ']]'); + output.push(updated); + } continue; } @@ -985,7 +1036,7 @@ function autoprefixer(value) { updated = plus + delimiter; updated += plus.replacer('font-smoothing', '-webkit-font-smoothing') + delimiter; updated += plus.replacer('font-smoothing', '-moz-osx-font-smoothing'); - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); continue; } @@ -996,50 +1047,49 @@ function autoprefixer(value) { updated += plus.replacer('repeating-linear-', '-moz-repeating-linear-') + delimiter; updated += plus.replacer('repeating-linear-', '-ms-repeating-linear-') + delimiter; updated += plus; - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); } else if (property.indexOf('repeating-radial-gradient') !== -1) { updated = plus.replacer('repeating-radial-', '-webkit-repeating-radial-') + delimiter; updated += plus.replacer('repeating-radial-', '-moz-repeating-radial-') + delimiter; updated += plus.replacer('repeating-radial-', '-ms-repeating-radial-') + delimiter; updated += plus; - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); } else if (property.indexOf('linear-gradient') !== -1) { updated = plus.replacer('linear-', '-webkit-linear-') + delimiter; updated += plus.replacer('linear-', '-moz-linear-') + delimiter; updated += plus.replacer('linear-', '-ms-linear-') + delimiter; updated += plus; - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); } else if (property.indexOf('radial-gradient') !== -1) { updated = plus.replacer('radial-', '-webkit-radial-') + delimiter; updated += plus.replacer('radial-', '-moz-radial-') + delimiter; updated += plus.replacer('radial-', '-ms-radial-') + delimiter; updated += plus; - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); } - continue; } if (name === 'text-overflow') { updated = plus + delimiter; updated += plus.replacer('text-overflow', '-ms-text-overflow'); - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); continue; } if (name === 'display') { - if (property.indexOf('box') === -1) - continue; - updated = plus + delimiter; - updated += plus.replacer('box', '-webkit-box') + delimiter; - updated += plus.replacer('box', '-moz-box'); - value = value.replacer(property, '@[[' + output.length + ']]'); - output.push(updated); + if (property.indexOf('box') !== -1) { + updated = plus + delimiter; + updated += plus.replacer('box', '-webkit-box') + delimiter; + updated += plus.replacer('box', '-moz-box'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); + output.push(updated); + } continue; } @@ -1049,7 +1099,7 @@ function autoprefixer(value) { if (name.indexOf('animation') === -1) updated += delimiter + '-ms-' + plus; - value = value.replacer(property, '@[[' + output.length + ']]'); + value = value.replacer(replace, before + '@[[' + output.length + ']]'); output.push(updated); } @@ -3323,6 +3373,7 @@ exports.restart = function() { }; function markup(body) { + var command = view_find_command(body, 0, true); if (!command) return body; @@ -3333,21 +3384,39 @@ function markup(body) { var r = []; while (command) { + var cmd = command.command; + var name = cmd; - switch (cmd) { - case 'author': - cmd = 'F.config.author'; - break; - case 'root': - cmd = 'F.config[\'default-root\']'; - break; + if (name.substring(0, 2) === '\'%') { + name = 'config'; + cmd = 'config[\'' + cmd.substring(2) + ']'; + } else { + var index = name.indexOf('.'); + if (index !== -1) + name = name.substring(0, index); + else { + index = name.indexOf('('); + if (index !== -1) + name = name.substring(0, index); + } } - try { - r.push({ cmd: command.phrase, value: eval('(' + cmd + ')') }); - } catch (e) { - console.log('A markup compilation error -->', cmd, e); + if (ALLOWEDMARKUP[name]) { + switch (cmd) { + case 'author': + cmd = 'F.config.author'; + break; + case 'root': + cmd = 'F.config[\'default-root\']'; + break; + } + + try { + r.push({ cmd: command.phrase, value: eval('(' + cmd + ')') }); + } catch (e) { + console.log('A markup compilation error -->', cmd, e); + } } command = view_find_command(body, command.end, true); From 17d51211aa233ff3003c75fddcfa7c781cec4511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 20 May 2018 23:56:29 +0200 Subject: [PATCH 0340/1669] Added new changes. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 7bea5b1e5..bfe4a84d8 100755 --- a/changes.txt +++ b/changes.txt @@ -127,6 +127,7 @@ - improved: `debug` mode timing with improved consumption - improved: performance (+20%) NoSQL embedded database - improved: reading performance (+5%) in `U.streamer()` +- improved: CSS compressor ======= 2.9.4 (HOTFIX) From 8d1d1b8846eff70ef464085639311230579b9e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 00:09:43 +0200 Subject: [PATCH 0341/1669] Fixed CSS optimizer. --- internal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal.js b/internal.js index 56afbbd92..3d5e2bf5c 100755 --- a/internal.js +++ b/internal.js @@ -951,7 +951,7 @@ function cssmarginpadding(text) { return prop + a[0] + ' 0 0' + last; // 20px 30px 20px 30px --> 20px 30px - if (a[1] && a[2] && a[3] && a[0] === a[3] && a[1] === a[2]) + if (a[1] && a[2] && a[3] && a[0] === a[2] && a[1] === a[3]) return prop + a[0] + ' ' + a[1] + last; // 20px 30px 10px 30px --> 20px 30px 10px From 342a0edb36cd1c89504e8f068479de81ff77cb85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 07:01:52 +0200 Subject: [PATCH 0342/1669] Imroved code. --- utils.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils.js b/utils.js index cdc226fdc..8d347cec1 100755 --- a/utils.js +++ b/utils.js @@ -194,12 +194,9 @@ const hasOwnProperty = Object.prototype.hasOwnProperty; */ exports.isEmpty = function(obj) { - if (!obj) + if (!obj || obj instanceof Array) return true; - if (obj.length) - return false; - for (var key in obj) { if (hasOwnProperty.call(obj, key)) return false; From 2851d6b5ba8a033f89165a52af861eb95f9e02ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 07:09:54 +0200 Subject: [PATCH 0343/1669] Fixed new CSS optimization. --- internal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal.js b/internal.js index 3d5e2bf5c..9330906a1 100755 --- a/internal.js +++ b/internal.js @@ -76,7 +76,7 @@ const REG_CSS_8 = /\s\{/g; const REG_CSS_9 = /;\}/g; const REG_CSS_10 = /\$[a-z0-9-_]+:.*?;/gi; const REG_CSS_11 = /\$[a-z0-9-_]+/gi; -const REG_CSS_12 = /(margin|padding|border-radius):.*?(;|})/g; +const REG_CSS_12 = /(margin|padding):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; const EMPTYBUFFER = framework_utils.createBufferSize(0); From dec5343ecf16651c521cad54ccd5a84bef590604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 07:15:23 +0200 Subject: [PATCH 0344/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d255c6cfd..91944df5b 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-61", + "version": "3.0.0-62", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From ab133cac421ed6b1e4915903834a4952c4073eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 08:35:04 +0200 Subject: [PATCH 0345/1669] Fixed boolean overrides. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 121136e00..db1cfc45e 100755 --- a/nosql.js +++ b/nosql.js @@ -60,7 +60,7 @@ const FLAGS_READ = ['get']; const COUNTER_MMA = [0, 0]; const REGNUMBER = /^\d+$/; const REGINDEXCHAR = /[a-z]{1,2}/; -const REGBOOL = /":true/; // for updates of boolean types +const REGBOOL = /":true/g; // for updates of boolean types const JSONBOOL = '":true '; const DIRECTORYLENGTH = 9; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; From d7dd4b92535c4da6245c2a8ffb0a9ca85c37daee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 12:24:04 +0200 Subject: [PATCH 0346/1669] Fixed async operations. --- builders.js | 127 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/builders.js b/builders.js index 85613d6aa..fa26fbd3d 100755 --- a/builders.js +++ b/builders.js @@ -96,58 +96,47 @@ SchemaOptions.prototype.clean = function() { }; SchemaOptions.prototype.$async = function(callback, index) { - this.model.$async(callback, index); - return this; + return this.model.$async(callback, index); }; SchemaOptions.prototype.$workflow = function(name, helper, callback, async) { - this.model.$workflow(name, helper, callback, async); - return this; + return this.model.$workflow(name, helper, callback, async); }; SchemaOptions.prototype.$transform = function(name, helper, callback, async) { - this.model.$transform(name, helper, callback, async); - return this; + return this.model.$transform(name, helper, callback, async); }; SchemaOptions.prototype.$operation = function(name, helper, callback, async) { - this.model.$operation(name, helper, callback, async); - return this; + return this.model.$operation(name, helper, callback, async); }; SchemaOptions.prototype.$hook = function(name, helper, callback, async) { - this.model.$hook(name, helper, callback, async); - return this; + return this.model.$hook(name, helper, callback, async); }; SchemaOptions.prototype.$save = function(helper, callback, async) { - this.model.$save(helper, callback, async); - return this; + return this.model.$save(helper, callback, async); }; SchemaOptions.prototype.$insert = function(helper, callback, async) { - this.model.$insert(helper, callback, async); - return this; + return this.model.$insert(helper, callback, async); }; SchemaOptions.prototype.$update = function(helper, callback, async) { - this.model.$update(helper, callback, async); - return this; + return this.model.$update(helper, callback, async); }; SchemaOptions.prototype.$query = function(helper, callback, async) { - this.model.$query(helper, callback, async); - return this; + return this.model.$query(helper, callback, async); }; SchemaOptions.prototype.$delete = SchemaOptions.prototype.$remove = function(helper, callback, async) { - this.model.$remove(helper, callback, async); - return this; + return this.model.$remove(helper, callback, async); }; SchemaOptions.prototype.$get = SchemaOptions.prototype.$read = function(helper, callback, async) { - this.model.$get(helper, callback, async); - return this; + return this.model.$get(helper, callback, async); }; SchemaOptions.prototype.push = function(type, name, helper, first) { @@ -2543,6 +2532,7 @@ SchemaInstance.prototype.$async = function(callback, index) { a.pending = 0; a.next = function() { + a.running = true; var fn = a.fn ? a.fn.shift() : null; if (fn) { a.pending++; @@ -2552,6 +2542,7 @@ SchemaInstance.prototype.$async = function(callback, index) { }; a.done = function() { + a.running = false; a.pending--; if (a.fn.length) setImmediate(a.next); @@ -2712,9 +2703,14 @@ SchemaInstance.prototype.$controller = function(controller) { }; SchemaInstance.prototype.$save = function(helper, callback, async) { - if (this.$$async) { - if (callback === true) { + if (this.$$async && !this.$$async.running) { + + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2728,9 +2724,14 @@ SchemaInstance.prototype.$save = function(helper, callback, async) { }; SchemaInstance.prototype.$insert = function(helper, callback, async) { - if (this.$$async) { - if (callback === true) { + if (this.$$async && !this.$$async.running) { + + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2744,9 +2745,14 @@ SchemaInstance.prototype.$insert = function(helper, callback, async) { }; SchemaInstance.prototype.$update = function(helper, callback, async) { - if (this.$$async) { - if (callback === true) { + if (this.$$async && !this.$$async.running) { + + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2760,9 +2766,14 @@ SchemaInstance.prototype.$update = function(helper, callback, async) { }; SchemaInstance.prototype.$query = function(helper, callback, async) { - if (this.$$async) { - if (callback === true) { + if (this.$$async && !this.$$async.running) { + + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2771,14 +2782,19 @@ SchemaInstance.prototype.$query = function(helper, callback, async) { this.$push('query', null, helper, null, async, callback); } else this.$$schema.query(this, helper, callback, this.$$controller); + return this; }; SchemaInstance.prototype.$read = SchemaInstance.prototype.$get = function(helper, callback, async) { - if (this.$$async) { + if (this.$$async && !this.$$async.running) { - if (callback === true) { + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2793,9 +2809,13 @@ SchemaInstance.prototype.$read = SchemaInstance.prototype.$get = function(helper SchemaInstance.prototype.$delete = SchemaInstance.prototype.$remove = function(helper, callback, async) { - if (this.$$async) { + if (this.$$async && !this.$$async.running) { - if (callback === true) { + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2819,9 +2839,13 @@ SchemaInstance.prototype.$destroy = function() { SchemaInstance.prototype.$transform = function(name, helper, callback, async) { - if (this.$$async) { + if (this.$$async && !this.$$async.running) { - if (callback === true) { + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2837,9 +2861,13 @@ SchemaInstance.prototype.$transform = function(name, helper, callback, async) { SchemaInstance.prototype.$workflow = function(name, helper, callback, async) { - if (this.$$async) { + if (this.$$async && !this.$$async.running) { - if (callback === true) { + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2855,9 +2883,13 @@ SchemaInstance.prototype.$workflow = function(name, helper, callback, async) { SchemaInstance.prototype.$hook = function(name, helper, callback, async) { - if (this.$$async) { + if (this.$$async && !this.$$async.running) { - if (callback === true) { + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { var a = async; async = true; callback = a; @@ -2873,9 +2905,20 @@ SchemaInstance.prototype.$hook = function(name, helper, callback, async) { SchemaInstance.prototype.$operation = function(name, helper, callback, async) { - if (this.$$async) + if (this.$$async && !this.$$async.running) { + + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { + var a = async; + async = true; + callback = a; + } + this.$push('operation', name, helper, null, async, callback); - else + } else this.$$schema.operation(name, this, helper, callback, undefined, this.$$controller); return this; From ba90d466cc4196f1c66f485bd00bbdd1e9db3c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 21 May 2018 22:15:14 +0200 Subject: [PATCH 0347/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 91944df5b..7c0299a9f 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-62", + "version": "3.0.0-63", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 3c8d5dd86353cbbb45d7ef4df8cca46b3f206264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 22 May 2018 08:34:55 +0200 Subject: [PATCH 0348/1669] Some improvements. --- readme.md | 3 ++- utils.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index e677043b6..e655b5c5b 100755 --- a/readme.md +++ b/readme.md @@ -3,7 +3,7 @@ [![Professional Support](https://www.totaljs.com/img/badge-support.svg)](https://www.totaljs.com/support/) [![Chat with contributors](https://www.totaljs.com/img/badge-chat.svg)](https://messenger.totaljs.com) [![NPM version][npm-version-image]][npm-url] [![NPM quality][npm-quality]](http://packagequality.com/#?package=total.js) [![NPM downloads][npm-downloads-image]][npm-url] [![MIT License][license-image]][license-url] [![Build Status][travis-image]][travis-url] [![Gitter chat](https://badges.gitter.im/totaljs/framework.png)](https://messenger.totaljs.com) [![Known Vulnerabilities](https://snyk.io/test/npm/total.js/badge.svg)](https://snyk.io/test/npm/total.js) Node.js framework -===================== +================= __Total.js framework__ is a framework for Node.js platfrom written in `pure` JavaScript similar to PHP's Laravel or Python's Django or ASP.NET MVC. It can be used as web, desktop, service or IoT application. @@ -132,6 +132,7 @@ Please support the framework on social networks. - [Follow Total.js on LinkedIn](https://www.linkedin.com/groups/totaljs-8109884) ## Contact + - (c) 2012-2018 by Peter Širka - - contact form - diff --git a/utils.js b/utils.js index 8d347cec1..c692b9752 100755 --- a/utils.js +++ b/utils.js @@ -631,6 +631,7 @@ function request_proxy(options, callback) { proxy.path = options.uri.hostname + ':443'; proxy.headers = PROXYHEADERS; proxy.method = 'CONNECT'; + proxy.agent = false; if (proxy._auth) proxy.headers['Proxy-Authorization'] = proxy._auth; From 508686bba0cfb9aff2060ed589cbca268b4b45b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 22 May 2018 16:02:44 +0200 Subject: [PATCH 0349/1669] Updated proxy. --- utils.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/utils.js b/utils.js index c692b9752..e2e2d5b47 100755 --- a/utils.js +++ b/utils.js @@ -638,12 +638,15 @@ function request_proxy(options, callback) { var req = Http.request(proxy); + req.setTimeout(2000); + req.on('error', function(e) { options.callback(new Error('Proxy error: ' + e.toString()), '', 0, EMPTYOBJECT, proxy.hostname, EMPTYOBJECT); options.callback = null; }); req.on('connect', function(res, socket) { + if (res.statusCode === 200) { PROXYTLS.servername = options.uri.hostname; @@ -651,15 +654,14 @@ function request_proxy(options, callback) { PROXYTLS.headers.host = options.uri.hostname + ':' + (options.uri.port || '443'); PROXYTLS.socket = socket; - var tls = Tls.connect(443, PROXYTLS); + var tls = Tls.connect(0, PROXYTLS); + + options.proxy.tls = tls; + req.agent.defaultPort = 443; + req.onSocket(tls); tls.on('secureConnect', function() { - var a = options.uri.agent = new Https.Agent(); - a.defaultPort = 443; - a.reuseSocket(tls, req); - req.onSocket(tls); - options.socket = tls; - options.proxy.tls = tls; + tls.end(); callback(options.uri, options); }); From e52a2bb0530f369a0d134f6ed0a3442b6d5baee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 22 May 2018 22:42:00 +0200 Subject: [PATCH 0350/1669] Fixed `503`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 6e93e5c53..9445e7be6 100755 --- a/index.js +++ b/index.js @@ -6424,7 +6424,7 @@ F.response503 = function(req, res) { res.options.code = 503; res.options.headers = HEADERS.response503; res.options.body = F.view('.' + PATHMODULES + '503', F.waits); - res.$throw(); + res.$text(); return F; }; From 4863f9f563a0968a93a405fdbb9acf1bf54b17e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 22 May 2018 23:53:32 +0200 Subject: [PATCH 0351/1669] Fixed proxy in TLS mode. --- utils.js | 155 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 94 insertions(+), 61 deletions(-) diff --git a/utils.js b/utils.js index e2e2d5b47..82ac70f0e 100755 --- a/utils.js +++ b/utils.js @@ -82,9 +82,9 @@ const CT = 'Content-Type'; const CRC32TABLE = '00000000,77073096,EE0E612C,990951BA,076DC419,706AF48F,E963A535,9E6495A3,0EDB8832,79DCB8A4,E0D5E91E,97D2D988,09B64C2B,7EB17CBD,E7B82D07,90BF1D91,1DB71064,6AB020F2,F3B97148,84BE41DE,1ADAD47D,6DDDE4EB,F4D4B551,83D385C7,136C9856,646BA8C0,FD62F97A,8A65C9EC,14015C4F,63066CD9,FA0F3D63,8D080DF5,3B6E20C8,4C69105E,D56041E4,A2677172,3C03E4D1,4B04D447,D20D85FD,A50AB56B,35B5A8FA,42B2986C,DBBBC9D6,ACBCF940,32D86CE3,45DF5C75,DCD60DCF,ABD13D59,26D930AC,51DE003A,C8D75180,BFD06116,21B4F4B5,56B3C423,CFBA9599,B8BDA50F,2802B89E,5F058808,C60CD9B2,B10BE924,2F6F7C87,58684C11,C1611DAB,B6662D3D,76DC4190,01DB7106,98D220BC,EFD5102A,71B18589,06B6B51F,9FBFE4A5,E8B8D433,7807C9A2,0F00F934,9609A88E,E10E9818,7F6A0DBB,086D3D2D,91646C97,E6635C01,6B6B51F4,1C6C6162,856530D8,F262004E,6C0695ED,1B01A57B,8208F4C1,F50FC457,65B0D9C6,12B7E950,8BBEB8EA,FCB9887C,62DD1DDF,15DA2D49,8CD37CF3,FBD44C65,4DB26158,3AB551CE,A3BC0074,D4BB30E2,4ADFA541,3DD895D7,A4D1C46D,D3D6F4FB,4369E96A,346ED9FC,AD678846,DA60B8D0,44042D73,33031DE5,AA0A4C5F,DD0D7CC9,5005713C,270241AA,BE0B1010,C90C2086,5768B525,206F85B3,B966D409,CE61E49F,5EDEF90E,29D9C998,B0D09822,C7D7A8B4,59B33D17,2EB40D81,B7BD5C3B,C0BA6CAD,EDB88320,9ABFB3B6,03B6E20C,74B1D29A,EAD54739,9DD277AF,04DB2615,73DC1683,E3630B12,94643B84,0D6D6A3E,7A6A5AA8,E40ECF0B,9309FF9D,0A00AE27,7D079EB1,F00F9344,8708A3D2,1E01F268,6906C2FE,F762575D,806567CB,196C3671,6E6B06E7,FED41B76,89D32BE0,10DA7A5A,67DD4ACC,F9B9DF6F,8EBEEFF9,17B7BE43,60B08ED5,D6D6A3E8,A1D1937E,38D8C2C4,4FDFF252,D1BB67F1,A6BC5767,3FB506DD,48B2364B,D80D2BDA,AF0A1B4C,36034AF6,41047A60,DF60EFC3,A867DF55,316E8EEF,4669BE79,CB61B38C,BC66831A,256FD2A0,5268E236,CC0C7795,BB0B4703,220216B9,5505262F,C5BA3BBE,B2BD0B28,2BB45A92,5CB36A04,C2D7FFA7,B5D0CF31,2CD99E8B,5BDEAE1D,9B64C2B0,EC63F226,756AA39C,026D930A,9C0906A9,EB0E363F,72076785,05005713,95BF4A82,E2B87A14,7BB12BAE,0CB61B38,92D28E9B,E5D5BE0D,7CDCEFB7,0BDBDF21,86D3D2D4,F1D4E242,68DDB3F8,1FDA836E,81BE16CD,F6B9265B,6FB077E1,18B74777,88085AE6,FF0F6A70,66063BCA,11010B5C,8F659EFF,F862AE69,616BFFD3,166CCF45,A00AE278,D70DD2EE,4E048354,3903B3C2,A7672661,D06016F7,4969474D,3E6E77DB,AED16A4A,D9D65ADC,40DF0B66,37D83BF0,A9BCAE53,DEBB9EC5,47B2CF7F,30B5FFE9,BDBDF21C,CABAC28A,53B39330,24B4A3A6,BAD03605,CDD70693,54DE5729,23D967BF,B3667A2E,C4614AB8,5D681B02,2A6F2B94,B40BBE37,C30C8EA1,5A05DF1B,2D02EF8D'.split(',').map(s => parseInt(s, 16)); const REGISARR = /\[\d+\]$/; const PROXYBLACKLIST = { 'localhost': 1, '127.0.0.1': 1, '0.0.0.0': 1 }; -const PROXYOPTIONS = {}; -const PROXYHEADERS = {}; -const PROXYTLS = {}; +const PROXYOPTIONS = { headers: {}, method: 'CONNECT', agent: false }; +const PROXYTLS = { headers: {}}; +const PROXYOPTIONSHTTP = {}; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -426,6 +426,7 @@ function parseProxy(p) { if (obj.auth) obj._auth = 'Basic ' + U.createBuffer(obj.auth).toString('base64'); + obj.port = +obj.port; return F.temporary.other[key] = obj; } @@ -606,13 +607,19 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) proxy = parseProxy(F.config['default-proxy']); - if (proxy) { - options.proxy = proxy; - if (uri.protocol === 'http:') - request_call(uri, options); - else - request_proxy(options, request_call); - } else if (options.resolve) { + options.proxy = proxy; + + if (uri.protocol === 'https:') { + proxy.tls = true; + uri.agent = new ProxyAgent(options); + uri.agent.request = Http.request; + uri.agent.createSocket = createSecureSocket; + uri.agent.defaultPort = 443; + } + + if (proxy) + request_call(uri, options); + else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); request_call(uri, options); @@ -623,77 +630,95 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, return options.evt; }; -function request_proxy(options, callback) { +function ProxyAgent(options) { + var self = this; + self.options = options; + self.maxSockets = Http.Agent.defaultMaxSockets; + self.requests = []; +} - PROXYHEADERS.host = options.uri.hostname + ':443'; +ProxyAgent.prototype.createConnection = function(pending) { + var self = this + self.createSocket(pending, function(socket) { + pending.request.onSocket(socket); + }); +} - var proxy = options.proxy; - proxy.path = options.uri.hostname + ':443'; - proxy.headers = PROXYHEADERS; - proxy.method = 'CONNECT'; - proxy.agent = false; +ProxyAgent.prototype.createSocket = function(options, callback) { - if (proxy._auth) - proxy.headers['Proxy-Authorization'] = proxy._auth; + var self = this; + var proxy = self.options.proxy; + var uri = self.options.uri; - var req = Http.request(proxy); + PROXYOPTIONS.host = proxy.hostname; + PROXYOPTIONS.port = proxy.port; + PROXYOPTIONS.path = PROXYOPTIONS.headers.host = uri.hostname + ':' + (uri.port || '443'); - req.setTimeout(2000); + if (proxy._auth) + PROXYOPTIONS.headers['Proxy-Authorization'] = proxy._auth; - req.on('error', function(e) { - options.callback(new Error('Proxy error: ' + e.toString()), '', 0, EMPTYOBJECT, proxy.hostname, EMPTYOBJECT); - options.callback = null; - }); + var req = self.request(PROXYOPTIONS); - req.on('connect', function(res, socket) { + req.setTimeout(3000); + req.on('response', proxyagent_response); + req.on('upgrade', function(res, socket, head) { + setImmediate(onConnect, res, socket, head) + }); + req.on('connect', function(res, socket, head) { if (res.statusCode === 200) { + callback(socket); + } else { + var err = new Error('Proxy could not be established, code: ' + res.statusCode); + err.code = 'ECONNRESET'; + options.request.emit('error', err); + } + }); - PROXYTLS.servername = options.uri.hostname; - PROXYTLS.headers = options.uri.headers || {}; - PROXYTLS.headers.host = options.uri.hostname + ':' + (options.uri.port || '443'); - PROXYTLS.socket = socket; + req.on('error', function(err) { + options.request.emit('error', err); + }); - var tls = Tls.connect(0, PROXYTLS); + req.end(); +} - options.proxy.tls = tls; - req.agent.defaultPort = 443; - req.onSocket(tls); +function proxyagent_response() { + res.upgrade = true; +} - tls.on('secureConnect', function() { - tls.end(); - callback(options.uri, options); - }); +ProxyAgent.prototype.addRequest = function(req, options) { + this.createConnection({ host: options.host, port: options.port, request: req }); +} - } else { - options.callback(new Error((res.statusMessage || 'Proxy error') + ': ' + res.statusCode), '', res.statusCode, res.headers, proxy.hostname, EMPTYOBJECT); - options.callback = null; - } +function createSecureSocket(options, callback) { + var self = this; + ProxyAgent.prototype.createSocket.call(self, options, function(socket) { + PROXYTLS.servername = self.options.uri.hostname; + PROXYTLS.headers = self.options.uri.headers; + PROXYTLS.socket = socket; + var tls = Tls.connect(0, PROXYTLS); + //tls.on('data', console.log); + callback(tls); }); - - req.end(); } function request_call(uri, options) { - var connection = uri.protocol === 'https:' ? Https : Http; var opt; if (options.proxy && !options.proxy.tls) { - opt = PROXYOPTIONS; + opt = PROXYOPTIONSHTTP; opt.port = options.proxy.port; opt.host = options.proxy.hostname; opt.path = uri.href; opt.headers = uri.headers; opt.method = uri.method; - opt.headers.host = options.uri.host; - if (options.proxy._auth) opt.headers['Proxy-Authorization'] = options.proxy._auth; - } else opt = uri; + var connection = uri.protocol === 'https:' ? Https : Http; var req = options.post ? connection.request(opt, (res) => request_response(res, uri, options)) : connection.get(opt, (res) => request_response(res, uri, options)); if (!options.callback) { @@ -823,9 +848,12 @@ function request_response(res, uri, options) { if (options.proxy && tmp.protocol === 'https:') { // TLS? + options.proxy.tls = true; options.uri = tmp; - request_proxy(options, request_call); - return; + options.uri.agent = new ProxyAgent(options); + options.uri.agent.request = Http.request; + options.uri.agent.createSocket = createSecureSocket; + options.uri.agent.defaultPort = 443; } if (!options.resolve) { @@ -888,7 +916,7 @@ function request_response(res, uri, options) { if (options.socket) { options.uri.agent.destroy(); - options.socket.destroy(); + //options.socket.destroy(); } var self = this; @@ -1095,13 +1123,19 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) proxy = parseProxy(F.config['default-proxy']); - if (proxy) { - options.proxy = proxy; - if (uri.protocol === 'http:') - download_call(uri, options); - else - request_proxy(options, download_call); - } else if (options.resolve) { + options.proxy = proxy; + + if (uri.protocol === 'https:') { + proxy.tls = true; + uri.agent = new ProxyAgent(options); + uri.agent.request = Http.request; + uri.agent.createSocket = createSecureSocket; + uri.agent.defaultPort = 443; + } + + if (proxy) + download_call(uri, options); + else if (options.resolve) { exports.resolve(url, function(err, u) { !err && (uri.host = u.host); download_call(uri, options); @@ -1118,13 +1152,12 @@ function download_call(uri, options) { options.length = 0; if (options.proxy && !options.proxy.tls) { - opt = PROXYOPTIONS; + opt = PROXYOPTIONSHTTP; opt.port = options.proxy.port; opt.host = options.proxy.hostname; opt.path = uri.href; opt.headers = uri.headers; opt.method = uri.method; - if (options.proxy._auth) opt.headers['Proxy-Authorization'] = options.proxy._auth; } else From f861ea1079e13a0f3e10922f6201b9df79134291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 22 May 2018 23:55:16 +0200 Subject: [PATCH 0352/1669] Fixed empty proxy. --- utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 82ac70f0e..890564f2f 100755 --- a/utils.js +++ b/utils.js @@ -609,7 +609,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, options.proxy = proxy; - if (uri.protocol === 'https:') { + if (proxy && uri.protocol === 'https:') { proxy.tls = true; uri.agent = new ProxyAgent(options); uri.agent.request = Http.request; @@ -1125,7 +1125,7 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi options.proxy = proxy; - if (uri.protocol === 'https:') { + if (proxy && uri.protocol === 'https:') { proxy.tls = true; uri.agent = new ProxyAgent(options); uri.agent.request = Http.request; From 8296d4692c7c0440dcfa2e4f8342988887e84297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 08:31:14 +0200 Subject: [PATCH 0353/1669] Added a new global alias `G`. --- changes.txt | 1 + index.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index bfe4a84d8..d0afdc311 100755 --- a/changes.txt +++ b/changes.txt @@ -80,6 +80,7 @@ - added: NoSQL database a new event `change`, more in docs - added: `schema.define()(DEFAULT_VALUE)` added `DEFAULT_VALUE` - added: `TESTUSER([user])` for faking of `F.onAuthorize` delegate, targeted for unit-testing only +- added: `G` as a global alias for `F.global` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index 9445e7be6..0efd4daf1 100755 --- a/index.js +++ b/index.js @@ -708,7 +708,7 @@ function Framework() { 'default-interval-uptodate': 5 }; - this.global = {}; + global.G = this.global = {}; this.resources = {}; this.connections = {}; this.functions = {}; @@ -4412,7 +4412,7 @@ F.$restart = function() { F.cache.clear(); F.cache.stop(); F.$events = {}; - F.global = {}; + global.G = F.global = {}; F.resources = {}; F.connections = {}; F.functions = {}; From 47976c6a50f4c993f40f922426b597ce8f3b8cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 10:23:44 +0200 Subject: [PATCH 0354/1669] Improved `ErrorBuilder` error messages. --- builders.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index fa26fbd3d..f43b55866 100755 --- a/builders.js +++ b/builders.js @@ -3260,6 +3260,8 @@ ErrorBuilder.prototype.add = function(name, error, path, index) { return this.push(name, error, path, index); }; +const ERRORBUILDERWHITE = { ' ': 1, ':': 1, ',': 1 }; + /** * Add an error (@alias for add) * @param {String} name Property name. @@ -3311,9 +3313,22 @@ ErrorBuilder.prototype.push = function(name, error, path, index, prefix) { path = undefined; } - if (!error) + if (!error && typeof(name) === 'string') { + var m = name.length; + if (m > 15) + m = 15; + error = '@'; + for (var i = 0; i < m; i++) { + if (ERRORBUILDERWHITE[name[i]]) { + error = name; + name = ''; + break; + } + } + } + if (error instanceof Error) { // Why? The answer is in controller.callback(); It's a reason for throwing 500 - internal server error this.unexpected = true; From 1271d46aa8ad71649e63ecf955212d80257c47f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 14:04:21 +0200 Subject: [PATCH 0355/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7c0299a9f..83c8cb391 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-63", + "version": "3.0.0-64", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 15d0f4d8fe12dc674a0fa3b9724b58d1d00613bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 16:05:10 +0200 Subject: [PATCH 0356/1669] Improved `CORS()`. --- changes.txt | 2 + index.js | 156 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 103 insertions(+), 55 deletions(-) diff --git a/changes.txt b/changes.txt index d0afdc311..b62c5ff6f 100755 --- a/changes.txt +++ b/changes.txt @@ -98,6 +98,8 @@ - updated: `middleware`, now Total.js supports new declaration `F.middleware(function($) {})` - updated: `F.wait()` HTML template - updated: JavaScript compressor, now optimizes multiple `var` declarations +- updated: `CORS()` without arguments for all routes, methods and origins +- updated: `CORS()` tries to join multiple same preferences to one - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index 0efd4daf1..ebad0042c 100755 --- a/index.js +++ b/index.js @@ -730,6 +730,7 @@ function Framework() { system: {}, files: [], cors: [], + corsall: false, websockets: [], middleware: {}, redirects: {}, @@ -1561,6 +1562,12 @@ F.restful2 = function(url, flags, onQuery, onGet, onSave, onDelete) { */ global.CORS = F.cors = function(url, flags, credentials) { + if (!arguments.length) { + F.routes.corsall = true; + PERF.OPTIONS = true; + return F; + } + var route = {}; var origins = []; var methods = []; @@ -1614,11 +1621,17 @@ global.CORS = F.cors = function(url, flags, credentials) { route.isWILDCARD = url.lastIndexOf('*') !== -1; + var index = url.indexOf('{'); + if (index !== -1) + url = url.substring(0, index); + if (route.isWILDCARD) url = url.replace('*', ''); - url = framework_internal.preparePath(framework_internal.encodeUnicodeURL(url.trim())); + if (url[url.length - 1] !== '/') + url += '/'; + url = framework_internal.preparePath(framework_internal.encodeUnicodeURL(url.trim())); route.hash = url.hash(); route.owner = _owner; route.url = framework_internal.routeSplitCreate(url); @@ -1627,9 +1640,36 @@ global.CORS = F.cors = function(url, flags, credentials) { route.headers = headers.length ? headers : null; route.credentials = credentials; route.age = age || F.config['default-cors-maxage']; - route.id = id; - F.routes.cors.push(route); + var e = F.routes.cors.findItem(function(item) { + return item.hash === route.hash; + }); + + if (e) { + + // Extends existing + if (route.origins && e.origins) + corsextend(route.origins, e.origins); + else if (e.origins && !route.origins) + e.origins = null; + + if (route.methods && e.methods) + corsextend(route.methods, e.methods); + + if (route.headers && e.headers) + corsextend(route.headers, e.headers); + + if (route.credentials && !e.credentials) + e.credentials = true; + + if (route.isWILDCARD && !e.isWILDCARD) + e.isWILDCARD = true; + + } else { + F.routes.cors.push(route); + route.id = id; + } + F._length_cors = F.routes.cors.length; F.routes.cors.sort(function(a, b) { @@ -1642,6 +1682,11 @@ global.CORS = F.cors = function(url, flags, credentials) { return F; }; +function corsextend(a, b) { + for (var i = 0; i < a.length; i++) + b.indexOf(a[i]) === -1 && b.push(a[i]); +} + global.GROUP = F.group = function() { var fn = null; @@ -4431,6 +4476,7 @@ F.$restart = function() { system: {}, files: [], cors: [], + corsall: false, websockets: [], middleware: {}, redirects: {}, @@ -7317,7 +7363,7 @@ F.$requestcontinue = function(req, res, headers) { req.flags = flags; F.$events['request-begin'] && F.emit('request-begin', req, res); - var isCORS = F._length_cors && req.headers['origin']; + var isCORS = (F._length_cors || F.routes.corsall) && req.headers['origin'] != null; switch (first) { case 'G': @@ -7408,90 +7454,90 @@ F.$requestcontinue_mmr = function(req, res, header) { F.$cors = function(req, res, fn, arg) { - var isAllowed = false; - var cors; - - for (var i = 0; i < F._length_cors; i++) { - cors = F.routes.cors[i]; - if (framework_internal.routeCompare(req.path, cors.url, false, cors.isWILDCARD)) { - isAllowed = true; - break; - } - } - - if (!isAllowed) - return fn(req, res, arg); - - var stop = false; + var isAllowed = F.routes.corsall; + var cors, origin; var headers = req.headers; - - if (!isAllowed) - stop = true; - - isAllowed = false; - - if (!stop && cors.headers) { - isAllowed = false; - for (var i = 0, length = cors.headers.length; i < length; i++) { - if (headers[cors.headers[i]]) { + if (!isAllowed) { + for (var i = 0; i < F._length_cors; i++) { + cors = F.routes.cors[i]; + if (framework_internal.routeCompare(req.path, cors.url, false, cors.isWILDCARD)) { isAllowed = true; break; } } + + if (!isAllowed) + return fn(req, res, arg); + + var stop = false; + if (!isAllowed) stop = true; - } - if (!stop && cors.methods) { isAllowed = false; - var current = headers['access-control-request-method'] || req.method; - if (current !== 'OPTIONS') { - for (var i = 0, length = cors.methods.length; i < length; i++) { - if (current.indexOf(cors.methods[i]) !== -1) { + + if (!stop && cors.headers) { + isAllowed = false; + for (var i = 0, length = cors.headers.length; i < length; i++) { + if (headers[cors.headers[i]]) { isAllowed = true; break; } } - if (!isAllowed) stop = true; } - } - var origin = headers['origin'].toLowerCase(); - if (!stop && cors.origins) { - isAllowed = false; - for (var i = 0, length = cors.origins.length; i < length; i++) { - if (cors.origins[i].indexOf(origin) !== -1) { - isAllowed = true; - break; + if (!stop && cors.methods) { + isAllowed = false; + var current = headers['access-control-request-method'] || req.method; + if (current !== 'OPTIONS') { + for (var i = 0, length = cors.methods.length; i < length; i++) { + if (current === cors.methods[i]) { + isAllowed = true; + break; + } + } + if (!isAllowed) + stop = true; } } - if (!isAllowed) - stop = true; + + origin = headers['origin'].toLowerCase(); + if (!stop && cors.origins) { + isAllowed = false; + for (var i = 0, length = cors.origins.length; i < length; i++) { + if (cors.origins[i].indexOf(origin) !== -1) { + isAllowed = true; + break; + } + } + if (!isAllowed) + stop = true; + } } - var name; - var isOPTIONS = req.method === 'OPTIONS'; + res.setHeader('Access-Control-Allow-Origin', F.routes.corsall ? headers['origin'] : (cors.origins ? cors.origins : cors.credentials ? isAllowed ? origin : cors.origins ? cors.origins : origin : headers['origin'])); - res.setHeader('Access-Control-Allow-Origin', cors.origins ? cors.origins : cors.credentials ? isAllowed ? origin : cors.origins ? cors.origins : origin : headers['origin']); - cors.credentials && res.setHeader('Access-Control-Allow-Credentials', 'true'); + if (!cors || cors.credentials) + res.setHeader('Access-Control-Allow-Credentials', 'true'); - name = 'Access-Control-Allow-Methods'; + var name = 'Access-Control-Allow-Methods'; + var isOPTIONS = req.method === 'OPTIONS'; - if (cors.methods) + if (cors && cors.methods) res.setHeader(name, cors.methods.join(', ')); else res.setHeader(name, isOPTIONS ? headers['access-control-request-method'] || '*' : req.method); name = 'Access-Control-Allow-Headers'; - if (cors.headers) + if (cors && cors.headers) res.setHeader(name, cors.headers.join(', ')); else res.setHeader(name, headers['access-control-request-headers'] || '*'); - cors.age && res.setHeader('Access-Control-Max-Age', cors.age); + cors && cors.age && res.setHeader('Access-Control-Max-Age', cors.age); if (stop) { fn = null; @@ -16572,4 +16618,4 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; +global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file From 29198292a888debdf0f47a320e53f622fa2d0524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 16:15:58 +0200 Subject: [PATCH 0357/1669] Added wildcard conditions to CORS. --- index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index ebad0042c..6e3c0e561 100755 --- a/index.js +++ b/index.js @@ -1622,8 +1622,10 @@ global.CORS = F.cors = function(url, flags, credentials) { route.isWILDCARD = url.lastIndexOf('*') !== -1; var index = url.indexOf('{'); - if (index !== -1) + if (index !== -1) { + route.isWILDCARD = true; url = url.substring(0, index); + } if (route.isWILDCARD) url = url.replace('*', ''); From 4b8d02c8f7d126081680856c890c4321c319d516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 21:51:54 +0200 Subject: [PATCH 0358/1669] Improved `cookies` in view engine. --- internal.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal.js b/internal.js index 9330906a1..f35d4a96b 100755 --- a/internal.js +++ b/internal.js @@ -1820,6 +1820,7 @@ function view_parse(content, minify, filename, controller) { var isFirst = false; var txtindex = -1; var index = 0; + var isCookie = false; if ((controller.$hasComponents || REG_COMPONENTS.test(content)) && REG_HEAD.test(content)) { @@ -1898,6 +1899,9 @@ function view_parse(content, minify, filename, controller) { while (command) { + if (!isCookie && command.command.indexOf('cookie') !== -1) + isCookie = true; + if (old) { text = content.substring(old.end + 1, command.beg); if (text) { @@ -2087,7 +2091,7 @@ function view_parse(content, minify, filename, controller) { if (RELEASE) builder = builder.replace(/(\+\$EMPTY\+)/g, '+').replace(/(\$output=\$EMPTY\+)/g, '$output=').replace(/(\$output\+=\$EMPTY\+)/g, '$output+=').replace(/(\}\$output\+=\$EMPTY)/g, '}').replace(/(\{\$output\+=\$EMPTY;)/g, '{').replace(/(\+\$EMPTY\+)/g, '+').replace(/(>'\+'<)/g, '><').replace(/'\+'/g, ''); - var fn = '(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;var cookie=function(name){return self.req.cookie(name)};' + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'; + var fn = '(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'; try { fn = eval(fn); } catch (e) { @@ -2273,7 +2277,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { return 'self.' + command; case 'sitemap': + case 'breadcrumb': case 'place': + if (name === 'breadcrumb') + name = 'sitemap'; return command.indexOf('(') === -1 ? '(repository[\'$' + command + '\'] || \'\')' : 'self.$' + command; case 'meta': From d3b05daf0dd9143ca51cd5f1a08a80340daac5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 22:22:22 +0200 Subject: [PATCH 0359/1669] Updates changelog. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index b62c5ff6f..502dd02c2 100755 --- a/changes.txt +++ b/changes.txt @@ -131,6 +131,7 @@ - improved: performance (+20%) NoSQL embedded database - improved: reading performance (+5%) in `U.streamer()` - improved: CSS compressor +- improved: CORS processing ======= 2.9.4 (HOTFIX) From 5af5f12d175eb87e495f4b0a932ad143982d82e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 23 May 2018 22:34:24 +0200 Subject: [PATCH 0360/1669] Added `@{breacrumb_` aliases for `@{sitemap_`. --- internal.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal.js b/internal.js index f35d4a96b..f03388dc0 100755 --- a/internal.js +++ b/internal.js @@ -2275,6 +2275,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'sitemap_name': case 'sitemap_navigation': return 'self.' + command; + case 'breadcrumb_url': + case 'breadcrumb_name': + case 'breadcrumb_navigation': + return 'self.sitemap_' + command.substring(10); case 'sitemap': case 'breadcrumb': From b4d4cb97a14065348b312ccfdc70fea9e64f6041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 28 May 2018 07:17:12 +0200 Subject: [PATCH 0361/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 83c8cb391..f8e7df751 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-64", + "version": "3.0.0-65", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 70dbb6fed5c00e5cb0345348998c0771a4cb7681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 28 May 2018 17:51:05 +0200 Subject: [PATCH 0362/1669] Improved roles in routing. --- index.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/index.js b/index.js index 6e3c0e561..53286126e 100755 --- a/index.js +++ b/index.js @@ -2051,6 +2051,13 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { break; } } + + if (isROLE && !membertype) { + tmp.push('authorize'); + priority += 2; + membertype = 1; + } + flags = tmp; priority += (count * 2); } else { @@ -2865,6 +2872,12 @@ F.websocket = function(url, funcInitialize, flags, length) { } } + if (isROLE && !membertype) { + tmp.push('authorize'); + membertype = 1; + priority++; + } + flags = tmp; flags.indexOf('get') === -1 && flags.unshift('get'); From 9286d0bb45d6ba39a6652cc0c6a7bf3748a6b438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 28 May 2018 18:48:00 +0200 Subject: [PATCH 0363/1669] Improved routing's cache. --- index.js | 12 +++++++++--- internal.js | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 53286126e..787867ba9 100755 --- a/index.js +++ b/index.js @@ -2056,6 +2056,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { tmp.push('authorize'); priority += 2; membertype = 1; + count++; } flags = tmp; @@ -2876,6 +2877,7 @@ F.websocket = function(url, funcInitialize, flags, length) { tmp.push('authorize'); membertype = 1; priority++; + count++; } flags = tmp; @@ -9056,7 +9058,7 @@ F.lookup = function(req, url, flags, membertype) { req.$isAuthorized = true; if (!isSystem) { - key = '1' + url + '$' + membertype + req.$flags + (subdomain ? '$' + subdomain : ''); + key = '1' + url + '$' + membertype + req.$flags + (subdomain ? '$' + subdomain : '') + (req.$roles ? 'R' : ''); if (F.temporary.other[key]) return F.temporary.other[key]; } @@ -9114,7 +9116,7 @@ F.lookup = function(req, url, flags, membertype) { continue; } - if (key && route.isCACHE && req.$isAuthorized) + if (key && route.isCACHE && (req.$isAuthorized || membertype === 1)) F.temporary.other[key] = route; return route; @@ -14895,8 +14897,12 @@ function extend_request(PROTO) { if (F.onAuthorize) { F.onAuthorize(req, req.res, req.flags, function(isAuthorized, user) { var hasRoles = length !== req.flags.length; - if (hasRoles) + + if (hasRoles) { req.$flags += req.flags.slice(length).join(''); + req.$roles = true; + } + if (typeof(isAuthorized) !== 'boolean') { user = isAuthorized; isAuthorized = !user; diff --git a/internal.js b/internal.js index f03388dc0..f9c76899d 100755 --- a/internal.js +++ b/internal.js @@ -669,6 +669,8 @@ exports.routeCompareFlags2 = function(req, route, membertype) { return 0; if ((route.isREFERER && req.flags.indexOf('referer') === -1) || (!route.isMULTIPLE && route.isJSON && req.flags.indexOf('json') === -1)) return 0; + if (route.isROLE && !req.$roles) + return -1; } var isRole = false; From d0c06bb90c2f761f421aa5046b711aecf9176b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 28 May 2018 19:35:18 +0200 Subject: [PATCH 0364/1669] Fixed comparing flags. --- internal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal.js b/internal.js index f9c76899d..8a8ab3ffd 100755 --- a/internal.js +++ b/internal.js @@ -669,7 +669,7 @@ exports.routeCompareFlags2 = function(req, route, membertype) { return 0; if ((route.isREFERER && req.flags.indexOf('referer') === -1) || (!route.isMULTIPLE && route.isJSON && req.flags.indexOf('json') === -1)) return 0; - if (route.isROLE && !req.$roles) + if (route.isROLE && !req.$roles && membertype) return -1; } From 3d7b8d9a331e61df675087e7a32cb8fd203f1083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 28 May 2018 21:33:27 +0200 Subject: [PATCH 0365/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f8e7df751..d8d98f952 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-65", + "version": "3.0.0-66", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 3b54320a4af250cb1fa66ad9f0657c503568ee95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 29 May 2018 09:17:15 +0200 Subject: [PATCH 0366/1669] Fixed tabs in routes. --- index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 6e3c0e561..a066aff1c 100755 --- a/index.js +++ b/index.js @@ -1750,6 +1750,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { url = '/'; if (url) { + url = url.replace(/\t/g, ' '); var index = url.indexOf(' '); if (index !== -1) { method = url.substring(0, index).toLowerCase().trim(); @@ -1896,8 +1897,9 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { continue; } - var first = flags[i][0]; + flags[i] = flags[i].replace(/\t/g, ' '); + var first = flags[i][0]; if (first === '&') { groups.push(flags[i].substring(1).trim()); continue; From add0b2695f6d3f8eba55a78beca5c9ad0cfaf07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 29 May 2018 10:13:28 +0200 Subject: [PATCH 0367/1669] Fixed `ROUTE()` with no flags. --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index 3cd9a6e81..dd17b59d7 100755 --- a/index.js +++ b/index.js @@ -1744,6 +1744,11 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { return F; } + if (typeof(flags) === 'number') { + length = flags; + flags = null; + } + var method = ''; var CUSTOM = typeof(url) === 'function' ? url : null; if (CUSTOM) From f321ce70f90657b23ade0c5b48e3bb0c5dde2e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 29 May 2018 10:45:30 +0200 Subject: [PATCH 0368/1669] Fixed replacing chars. --- internal.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal.js b/internal.js index 8a8ab3ffd..db62bc962 100755 --- a/internal.js +++ b/internal.js @@ -65,7 +65,8 @@ const REG_NOCOMPRESS = /@\{nocompress\s\w+}/gi; const REG_TAGREMOVE = /[^>](\r)\n\s{1,}$/; const REG_HELPERS = /helpers\.[a-z0-9A-Z_$]+\(.*?\)+/g; const REG_SITEMAP = /\s+(sitemap_navigation\(|sitemap\()+/g; -const REG_CSS_1 = /\n|\s{2,}/g; +const REG_CSS_0 = /\s{2,}/g; +const REG_CSS_1 = /\n/g; const REG_CSS_2 = /\s?\{\s{1,}/g; const REG_CSS_3 = /\s?\}\s{1,}/g; const REG_CSS_4 = /\s?:\s{1,}/g; @@ -909,8 +910,7 @@ function compile_autovendor(css) { var isAuto = css.substring(0, 100).indexOf(avp) !== -1; if (isAuto) css = autoprefixer(css.replace(avp, '')); - - return css.replace(REG_CSS_1, '').replace(REG_CSS_2, '{').replace(REG_CSS_3, '}').replace(REG_CSS_4, ':').replace(REG_CSS_5, ';').replace(REG_CSS_6, function(search, index, text) { + return css.replace(REG_CSS_0, ' ').replace(REG_CSS_1, '').replace(REG_CSS_2, '{').replace(REG_CSS_3, '}').replace(REG_CSS_4, ':').replace(REG_CSS_5, ';').replace(REG_CSS_6, function(search, index, text) { for (var i = index; i > 0; i--) { if ((text[i] === '\'' || text[i] === '"') && (text[i - 1] === ':')) return search; From 54233b85ae6256e7696bec814cbc023c0cfade99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 2 Jun 2018 21:37:24 +0200 Subject: [PATCH 0369/1669] Added `MailMessage.attachmentnosql()`. --- changes.txt | 1 + mail.js | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- nosql.js | 46 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/changes.txt b/changes.txt index 502dd02c2..96b501989 100755 --- a/changes.txt +++ b/changes.txt @@ -14,6 +14,7 @@ - added: `MailMessage.high()` sets `high` priority of the email messsage - added: `MailMessage.low()` sets `low` priority of the email messsage - added: `MailMessage.confidential()` sets `Sensitivity` header with `confidential` value +- added: `MailMessage.attachmentnosql(db, id, [name])` sends a file from NoSQL embedded database - added: `SchemaBuilderEntity.$stop()` stops the async list - added: `SchemaOptions.stop()` alias to `$.model.$stop()` - added: `SchemaOptions.next()` alias to `$.model.$next()` diff --git a/mail.js b/mail.js index 0130d625a..01fb3253c 100755 --- a/mail.js +++ b/mail.js @@ -267,6 +267,21 @@ Message.prototype.attachment = function(filename, name) { return this; }; +Message.prototype.attachmentnosql = function(db, id, name) { + + var extension; + var type; + + if (name) { + extension = framework_utils.getExtension(name); + type = framework_utils.getContentType(extension); + } + + !this.files && (this.files = []); + this.files.push({ nosql: db, name: name, filename: id, type: type, extension: extension }); + return this; +}; + /** * Clears a timeout for sending emails (if the email is sent through the F.onMail) * @return {Message} @@ -368,6 +383,7 @@ Mailer.prototype.destroy = function(obj) { }; const ATTACHMENT_SO = { encoding: 'base64' }; +const ATTACHMENT_SO_NOSQL = { encoding: 'base64', start: 2000 }; Mailer.prototype.$writeattachment = function(obj) { @@ -379,11 +395,37 @@ Mailer.prototype.$writeattachment = function(obj) { return this; } + var stream; + + if (attachment.nosql) { + NOSQL(attachment.nosql).binary.readbase64(attachment.filename, function(err, stream, meta) { + if (err) { + F.error(err, 'Mail.attachment()', attachment.filename); + mailer.$writeattachment(obj); + } else { + + if (!attachment.name) { + attachment.name = meta.name; + attachment.type = meta.type; + attachment.extension = U.getExtension(meta.name); + } + + writeattachemnt_stream(attachment, obj, stream); + } + }); + } else { + stream = Fs.createReadStream(attachment.filename, ATTACHMENT_SO); + writeattachemnt_stream(attachment, obj, stream); + } + + return this; +}; + +function writeattachemnt_stream(attachment, obj, stream) { + var name = attachment.name; - var stream = Fs.createReadStream(attachment.filename, ATTACHMENT_SO); + var isCalendar = attachment.extension === 'ics'; var message = []; - var extension = attachment.extension; - var isCalendar = extension === 'ics'; message.push('--' + obj.boundary); @@ -409,8 +451,7 @@ Mailer.prototype.$writeattachment = function(obj) { mailer.$writeattachment(obj); }); - return this; -}; +} function writeattachment_data(chunk) { diff --git a/nosql.js b/nosql.js index db1cfc45e..e915f37d4 100755 --- a/nosql.js +++ b/nosql.js @@ -65,6 +65,7 @@ const JSONBOOL = '":true '; const DIRECTORYLENGTH = 9; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; +const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' }; const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; const CLEANDBTICKS = 86400000 * 2; // 48 hours @@ -3376,7 +3377,7 @@ DatabaseBuilder.prototype.repository = function(key, value) { DatabaseBuilder.prototype.compile = function() { var self = this; var raw = self.$code.join(''); - var code = 'var repository=$F.repository;var options=$F.options;var arg=$F.arg;var fn=$F.fn;var $is=false;var $tmp;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + 'if(options.fields){var $doc={};for(var $i=0;$i 3) { + callback(new Error('File not found.')); + return self; + } + + var isnew = false; + + if (id > 0) + isnew = true; + else if (id[0] === 'B' || id[0] === 'b') { + id = +id.substring(id.length - DIRECTORYLENGTH); + isnew = true; + } else if (id.indexOf('#') === -1) + id = self.db.name + '#' + id; + + var filename; + + if (isnew) { + filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + } else + filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); + + var stream = Fs.createReadStream(filename, BINARYREADMETA); + stream.on('error', err => callback(err)); + stream.on('data', function(buffer) { + var json = buffer.toString('utf8').replace(REG_CLEAN, ''); + if (json) { + var meta = JSON.parse(json, jsonparser); + stream = Fs.createReadStream(filename, BINARYREADDATABASE64); + callback(null, stream, meta); + CLEANUP(stream); + } else + setTimeout(readfileattempt, 100, self, id, callback, count || 1); + }); + + return self; +}; + + function readfileattempt(self, id, callback, count) { self.read(id, callback, count + 1); } From 5131c5fefdeff9cc852d1633740cbb96d2260032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 2 Jun 2018 22:59:55 +0200 Subject: [PATCH 0370/1669] Improved `U.keywords()` for Chinese / Japan chars. --- changes.txt | 1 + nosql.js | 33 ++++++++++++++++++------------- utils.js | 57 ++++++++++++++++++++++++++++++++++++----------------- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/changes.txt b/changes.txt index 96b501989..a3836e345 100755 --- a/changes.txt +++ b/changes.txt @@ -101,6 +101,7 @@ - updated: JavaScript compressor, now optimizes multiple `var` declarations - updated: `CORS()` without arguments for all routes, methods and origins - updated: `CORS()` tries to join multiple same preferences to one +- updated: `U.keywords()` for Chinese/Japan characters - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/nosql.js b/nosql.js index e915f37d4..8dd846d4e 100755 --- a/nosql.js +++ b/nosql.js @@ -52,17 +52,18 @@ const EXTENSION_META = '.meta'; const EXTENSION_COUNTER = '-counter2'; const EXTENSION_INDEXES = '-indexes'; const BINARY_HEADER_LENGTH = 2000; -const NEWLINE = '\n'; +const COUNTER_MMA = [0, 0]; +const DIRECTORYLENGTH = 9; const EMPTYARRAY = []; -const REG_CLEAN = /^[\s]+|[\s]+$/g; -const INMEMORY = {}; const FLAGS_READ = ['get']; -const COUNTER_MMA = [0, 0]; -const REGNUMBER = /^\d+$/; -const REGINDEXCHAR = /[a-z]{1,2}/; -const REGBOOL = /":true/g; // for updates of boolean types +const INMEMORY = {}; const JSONBOOL = '":true '; -const DIRECTORYLENGTH = 9; +const NEWLINE = '\n'; +const REGBOOL = /":true/g; // for updates of boolean types +const REGCHINA = /[\u3400-\u9FBF]/; +const REGCLEAN = /^[\s]+|[\s]+$/g; +const REGINDEXCHAR = /[a-z]{1,2}/; +const REGNUMBER = /^\d+$/; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' }; @@ -3250,8 +3251,12 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { if (value instanceof Array) { for (var i = 0; i < value.length; i++) value[i] = value[i].toLowerCase(); - } else - value = value.toLowerCase().split(' '); + } else { + if (REGCHINA.test(value)) + value = value.split(''); + else + value = value.toLowerCase().split(' '); + } var count = 1; @@ -5028,7 +5033,7 @@ Binary.prototype.read = function(id, callback, count) { var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); stream.on('data', function(buffer) { - var json = buffer.toString('utf8').replace(REG_CLEAN, ''); + var json = buffer.toString('utf8').replace(REGCLEAN, ''); if (json) { var meta = JSON.parse(json, jsonparser); stream = Fs.createReadStream(filename, BINARYREADDATA); @@ -5070,7 +5075,7 @@ Binary.prototype.readbase64 = function(id, callback, count) { var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); stream.on('data', function(buffer) { - var json = buffer.toString('utf8').replace(REG_CLEAN, ''); + var json = buffer.toString('utf8').replace(REGCLEAN, ''); if (json) { var meta = JSON.parse(json, jsonparser); stream = Fs.createReadStream(filename, BINARYREADDATABASE64); @@ -5211,7 +5216,7 @@ Binary.prototype.browse = function(directory, callback) { var stream = Fs.createReadStream(target + '/' + item, BINARYREADMETA); stream.on('data', function(buffer) { - var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, '').parseJSON(true); + var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REGCLEAN, '').parseJSON(true); if (json) { var id = item.substring(0, item.length - le); json.id = 'B' + json.date + 'T' + id; @@ -5264,7 +5269,7 @@ Binary.prototype.all = function(callback) { var stream = Fs.createReadStream(target + '/' + item, BINARYREADMETA); stream.on('data', function(buffer) { - var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REG_CLEAN, '').parseJSON(true); + var json = framework_utils.createBuffer(buffer, 'binary').toString('utf8').replace(REGCLEAN, '').parseJSON(true); if (json) { json.id = item.substring(l, item.length - le); json.ctime = stat.ctime; diff --git a/utils.js b/utils.js index 890564f2f..b19aa4196 100755 --- a/utils.js +++ b/utils.js @@ -66,6 +66,10 @@ const regexpDECRYPT = /-|_/g; const regexpENCRYPT = /\/|\+/g; const regexpUNICODE = /\\u([\d\w]{4})/gi; const regexpTERMINAL = /[\w\S]+/g; +const regexpY = /y/g; +const regexpN = /\n/g; +const regexpCHARS = /\W|_/g; +const regexpCHINA = /[\u3400-\u9FBF]/; const SOUNDEX = { a: '', e: '', i: '', o: '', u: '', b: 1, f: 1, p: 1, v: 1, c: 2, g: 2, j: 2, k: 2, q: 2, s: 2, x: 2, z: 2, d: 3, t: 3, l: 4, m: 5, n: 5, r: 6 }; const ENCODING = 'utf8'; const NEWLINE = '\r\n'; @@ -353,14 +357,14 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng for (var i = 0, length = content.length; i < length; i++) { if (!content[i]) continue; - var tmp = (forSearch ? content[i].removeDiacritics().toLowerCase().replace(/y/g, 'i') : content[i].toLowerCase()).replace(/\n/g, ' ').split(' '); + var tmp = (forSearch ? content[i].removeDiacritics().toLowerCase().replace(regexpY, 'i') : content[i].toLowerCase()).replace(regexpN, ' ').split(' '); if (!tmp || !tmp.length) continue; for (var j = 0, jl = tmp.length; j < jl; j++) words.push(tmp[j]); } } else - words = (forSearch ? content.removeDiacritics().toLowerCase().replace(/y/g, 'i') : content.toLowerCase()).replace(/\n/g, ' ').split(' '); + words = (forSearch ? content.removeDiacritics().toLowerCase().replace(regexpY, 'i') : content.toLowerCase()).replace(regexpN, ' ').split(' '); if (!words) words = []; @@ -369,7 +373,27 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng var counter = 0; for (var i = 0, length = words.length; i < length; i++) { - var word = words[i].trim(); + + var word = words[i].trim().replace(regexpCHARS, keywordscleaner); + + if (regexpCHINA.test(word)) { + + var tmpw = word.split('', max_count); + + for (var j = 0; j < tmpw.length; j++) { + word = tmpw[j]; + if (dic[word]) + dic[word]++; + else + dic[word] = 1; + counter++; + } + + if (counter >= max_count) + break; + + continue; + } if (word.length < min_length) continue; @@ -377,9 +401,6 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng if (counter >= max_count) break; - if (forSearch) - word = word.replace(/\W|_/g, ''); - // Gets 80% length of word if (alternative) { if (isSoundex) @@ -413,6 +434,10 @@ exports.keywords = function(content, forSearch, alternative, max_count, max_leng return keys; }; +function keywordscleaner(c) { + return c.charCodeAt(0) < 200 ? '' : c; +} + function parseProxy(p) { var key = 'proxy_' + p; if (F.temporary.other[key]) @@ -638,11 +663,11 @@ function ProxyAgent(options) { } ProxyAgent.prototype.createConnection = function(pending) { - var self = this + var self = this; self.createSocket(pending, function(socket) { pending.request.onSocket(socket); }); -} +}; ProxyAgent.prototype.createSocket = function(options, callback) { @@ -661,16 +686,12 @@ ProxyAgent.prototype.createSocket = function(options, callback) { req.setTimeout(3000); req.on('response', proxyagent_response); - req.on('upgrade', function(res, socket, head) { - setImmediate(onConnect, res, socket, head) - }); - - req.on('connect', function(res, socket, head) { + req.on('connect', function(res, socket) { if (res.statusCode === 200) { callback(socket); } else { - var err = new Error('Proxy could not be established, code: ' + res.statusCode); - err.code = 'ECONNRESET'; + var err = new Error('Proxy could not be established, code: ' + res.statusCode); + err.code = 'ECONNRESET'; options.request.emit('error', err); } }); @@ -680,15 +701,15 @@ ProxyAgent.prototype.createSocket = function(options, callback) { }); req.end(); -} +}; -function proxyagent_response() { +function proxyagent_response(res) { res.upgrade = true; } ProxyAgent.prototype.addRequest = function(req, options) { this.createConnection({ host: options.host, port: options.port, request: req }); -} +}; function createSecureSocket(options, callback) { var self = this; From daa5e1fc63d55893b10ca2a2ff1ff4f38e616f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 2 Jun 2018 23:01:50 +0200 Subject: [PATCH 0371/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d8d98f952..e233fe818 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-66", + "version": "3.0.0-67", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From e0386f4dbfc26be50a08dbb7d6665f23aea80e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 3 Jun 2018 22:50:31 +0200 Subject: [PATCH 0372/1669] Improved code. --- nosql.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/nosql.js b/nosql.js index 8dd846d4e..70189b533 100755 --- a/nosql.js +++ b/nosql.js @@ -54,7 +54,6 @@ const EXTENSION_INDEXES = '-indexes'; const BINARY_HEADER_LENGTH = 2000; const COUNTER_MMA = [0, 0]; const DIRECTORYLENGTH = 9; -const EMPTYARRAY = []; const FLAGS_READ = ['get']; const INMEMORY = {}; const JSONBOOL = '":true '; @@ -93,8 +92,6 @@ function promise(fn) { }); } -Object.freeze(EMPTYARRAY); - exports.kill = function(signal) { FORK && TRY(() => FORK && FORK.kill && FORK.kill(signal || 'SIGTERM')); }; From 0cc3a224f64df8b1318af6996449d6a2b25bb320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 3 Jun 2018 22:51:07 +0200 Subject: [PATCH 0373/1669] Improved code. --- utils.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/utils.js b/utils.js index b19aa4196..361a43473 100755 --- a/utils.js +++ b/utils.js @@ -78,8 +78,6 @@ const DIACRITICSMAP = {}; const STREAM_READONLY = { flags: 'r' }; const STREAM_END = { end: false }; const ALPHA_INDEX = { '<': '<', '>': '>', '"': '"', '&apos': '\'', '&': '&', '<': '<', '>': '>', '"': '"', ''': '\'', '&': '&' }; -const EMPTYARRAY = []; -const EMPTYOBJECT = {}; const NODEVERSION = parseFloat(process.version.toString().replace('v', '').replace(/\./g, '')); const STREAMPIPE = { end: false }; const CT = 'Content-Type'; @@ -93,9 +91,6 @@ const PROXYOPTIONSHTTP = {}; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; -Object.freeze(EMPTYARRAY); -Object.freeze(EMPTYOBJECT); - var DIACRITICS=[{b:' ',c:'\u00a0'},{b:'0',c:'\u07c0'},{b:'A',c:'\u24b6\uff21\u00c0\u00c1\u00c2\u1ea6\u1ea4\u1eaa\u1ea8\u00c3\u0100\u0102\u1eb0\u1eae\u1eb4\u1eb2\u0226\u01e0\u00c4\u01de\u1ea2\u00c5\u01fa\u01cd\u0200\u0202\u1ea0\u1eac\u1eb6\u1e00\u0104\u023a\u2c6f'},{b:'AA',c:'\ua732'},{b:'AE',c:'\u00c6\u01fc\u01e2'},{b:'AO',c:'\ua734'},{b:'AU',c:'\ua736'},{b:'AV',c:'\ua738\ua73a'},{b:'AY',c:'\ua73c'},{b:'B',c:'\u24b7\uff22\u1e02\u1e04\u1e06\u0243\u0181'},{b:'C',c:'\u24b8\uff23\ua73e\u1e08\u0106C\u0108\u010a\u010c\u00c7\u0187\u023b'},{b:'D',c:'\u24b9\uff24\u1e0a\u010e\u1e0c\u1e10\u1e12\u1e0e\u0110\u018a\u0189\u1d05\ua779'},{b:'Dh',c:'\u00d0'},{b:'DZ',c:'\u01f1\u01c4'},{b:'Dz',c:'\u01f2\u01c5'},{b:'E',c:'\u025b\u24ba\uff25\u00c8\u00c9\u00ca\u1ec0\u1ebe\u1ec4\u1ec2\u1ebc\u0112\u1e14\u1e16\u0114\u0116\u00cb\u1eba\u011a\u0204\u0206\u1eb8\u1ec6\u0228\u1e1c\u0118\u1e18\u1e1a\u0190\u018e\u1d07'},{b:'F',c:'\ua77c\u24bb\uff26\u1e1e\u0191\ua77b'}, {b:'G',c:'\u24bc\uff27\u01f4\u011c\u1e20\u011e\u0120\u01e6\u0122\u01e4\u0193\ua7a0\ua77d\ua77e\u0262'},{b:'H',c:'\u24bd\uff28\u0124\u1e22\u1e26\u021e\u1e24\u1e28\u1e2a\u0126\u2c67\u2c75\ua78d'},{b:'I',c:'\u24be\uff29\u00cc\u00cd\u00ce\u0128\u012a\u012c\u0130\u00cf\u1e2e\u1ec8\u01cf\u0208\u020a\u1eca\u012e\u1e2c\u0197'},{b:'J',c:'\u24bf\uff2a\u0134\u0248\u0237'},{b:'K',c:'\u24c0\uff2b\u1e30\u01e8\u1e32\u0136\u1e34\u0198\u2c69\ua740\ua742\ua744\ua7a2'},{b:'L',c:'\u24c1\uff2c\u013f\u0139\u013d\u1e36\u1e38\u013b\u1e3c\u1e3a\u0141\u023d\u2c62\u2c60\ua748\ua746\ua780'}, {b:'LJ',c:'\u01c7'},{b:'Lj',c:'\u01c8'},{b:'M',c:'\u24c2\uff2d\u1e3e\u1e40\u1e42\u2c6e\u019c\u03fb'},{b:'N',c:'\ua7a4\u0220\u24c3\uff2e\u01f8\u0143\u00d1\u1e44\u0147\u1e46\u0145\u1e4a\u1e48\u019d\ua790\u1d0e'},{b:'NJ',c:'\u01ca'},{b:'Nj',c:'\u01cb'},{b:'O',c:'\u24c4\uff2f\u00d2\u00d3\u00d4\u1ed2\u1ed0\u1ed6\u1ed4\u00d5\u1e4c\u022c\u1e4e\u014c\u1e50\u1e52\u014e\u022e\u0230\u00d6\u022a\u1ece\u0150\u01d1\u020c\u020e\u01a0\u1edc\u1eda\u1ee0\u1ede\u1ee2\u1ecc\u1ed8\u01ea\u01ec\u00d8\u01fe\u0186\u019f\ua74a\ua74c'}, {b:'OE',c:'\u0152'},{b:'OI',c:'\u01a2'},{b:'OO',c:'\ua74e'},{b:'OU',c:'\u0222'},{b:'P',c:'\u24c5\uff30\u1e54\u1e56\u01a4\u2c63\ua750\ua752\ua754'},{b:'Q',c:'\u24c6\uff31\ua756\ua758\u024a'},{b:'R',c:'\u24c7\uff32\u0154\u1e58\u0158\u0210\u0212\u1e5a\u1e5c\u0156\u1e5e\u024c\u2c64\ua75a\ua7a6\ua782'},{b:'S',c:'\u24c8\uff33\u1e9e\u015a\u1e64\u015c\u1e60\u0160\u1e66\u1e62\u1e68\u0218\u015e\u2c7e\ua7a8\ua784'},{b:'T',c:'\u24c9\uff34\u1e6a\u0164\u1e6c\u021a\u0162\u1e70\u1e6e\u0166\u01ac\u01ae\u023e\ua786'}, {b:'Th',c:'\u00de'},{b:'TZ',c:'\ua728'},{b:'U',c:'\u24ca\uff35\u00d9\u00da\u00db\u0168\u1e78\u016a\u1e7a\u016c\u00dc\u01db\u01d7\u01d5\u01d9\u1ee6\u016e\u0170\u01d3\u0214\u0216\u01af\u1eea\u1ee8\u1eee\u1eec\u1ef0\u1ee4\u1e72\u0172\u1e76\u1e74\u0244'},{b:'V',c:'\u24cb\uff36\u1e7c\u1e7e\u01b2\ua75e\u0245'},{b:'VY',c:'\ua760'},{b:'W',c:'\u24cc\uff37\u1e80\u1e82\u0174\u1e86\u1e84\u1e88\u2c72'},{b:'X',c:'\u24cd\uff38\u1e8a\u1e8c'},{b:'Y',c:'\u24ce\uff39\u1ef2\u00dd\u0176\u1ef8\u0232\u1e8e\u0178\u1ef6\u1ef4\u01b3\u024e\u1efe'}, {b:'Z',c:'\u24cf\uff3a\u0179\u1e90\u017b\u017d\u1e92\u1e94\u01b5\u0224\u2c7f\u2c6b\ua762'},{b:'a',c:'\u24d0\uff41\u1e9a\u00e0\u00e1\u00e2\u1ea7\u1ea5\u1eab\u1ea9\u00e3\u0101\u0103\u1eb1\u1eaf\u1eb5\u1eb3\u0227\u01e1\u00e4\u01df\u1ea3\u00e5\u01fb\u01ce\u0201\u0203\u1ea1\u1ead\u1eb7\u1e01\u0105\u2c65\u0250\u0251'},{b:'aa',c:'\ua733'},{b:'ae',c:'\u00e6\u01fd\u01e3'},{b:'ao',c:'\ua735'},{b:'au',c:'\ua737'},{b:'av',c:'\ua739\ua73b'},{b:'ay',c:'\ua73d'}, {b:'b',c:'\u24d1\uff42\u1e03\u1e05\u1e07\u0180\u0183\u0253\u0182'},{b:'c',c:'\uff43\u24d2\u0107\u0109\u010b\u010d\u00e7\u1e09\u0188\u023c\ua73f\u2184'},{b:'d',c:'\u24d3\uff44\u1e0b\u010f\u1e0d\u1e11\u1e13\u1e0f\u0111\u018c\u0256\u0257\u018b\u13e7\u0501\ua7aa'},{b:'dh',c:'\u00f0'},{b:'dz',c:'\u01f3\u01c6'},{b:'e',c:'\u24d4\uff45\u00e8\u00e9\u00ea\u1ec1\u1ebf\u1ec5\u1ec3\u1ebd\u0113\u1e15\u1e17\u0115\u0117\u00eb\u1ebb\u011b\u0205\u0207\u1eb9\u1ec7\u0229\u1e1d\u0119\u1e19\u1e1b\u0247\u01dd'}, {b:'f',c:'\u24d5\uff46\u1e1f\u0192'},{b:'ff',c:'\ufb00'},{b:'fi',c:'\ufb01'},{b:'fl',c:'\ufb02'},{b:'ffi',c:'\ufb03'},{b:'ffl',c:'\ufb04'},{b:'g',c:'\u24d6\uff47\u01f5\u011d\u1e21\u011f\u0121\u01e7\u0123\u01e5\u0260\ua7a1\ua77f\u1d79'},{b:'h',c:'\u24d7\uff48\u0125\u1e23\u1e27\u021f\u1e25\u1e29\u1e2b\u1e96\u0127\u2c68\u2c76\u0265'},{b:'hv',c:'\u0195'},{b:'i',c:'\u24d8\uff49\u00ec\u00ed\u00ee\u0129\u012b\u012d\u00ef\u1e2f\u1ec9\u01d0\u0209\u020b\u1ecb\u012f\u1e2d\u0268\u0131'}, {b:'j',c:'\u24d9\uff4a\u0135\u01f0\u0249'},{b:'k',c:'\u24da\uff4b\u1e31\u01e9\u1e33\u0137\u1e35\u0199\u2c6a\ua741\ua743\ua745\ua7a3'},{b:'l',c:'\u24db\uff4c\u0140\u013a\u013e\u1e37\u1e39\u013c\u1e3d\u1e3b\u017f\u0142\u019a\u026b\u2c61\ua749\ua781\ua747\u026d'},{b:'lj',c:'\u01c9'},{b:'m',c:'\u24dc\uff4d\u1e3f\u1e41\u1e43\u0271\u026f'},{b:'n',c:'\u24dd\uff4e\u01f9\u0144\u00f1\u1e45\u0148\u1e47\u0146\u1e4b\u1e49\u019e\u0272\u0149\ua791\ua7a5\u043b\u0509'},{b:'nj', c:'\u01cc'},{b:'o',c:'\u24de\uff4f\u00f2\u00f3\u00f4\u1ed3\u1ed1\u1ed7\u1ed5\u00f5\u1e4d\u022d\u1e4f\u014d\u1e51\u1e53\u014f\u022f\u0231\u00f6\u022b\u1ecf\u0151\u01d2\u020d\u020f\u01a1\u1edd\u1edb\u1ee1\u1edf\u1ee3\u1ecd\u1ed9\u01eb\u01ed\u00f8\u01ff\ua74b\ua74d\u0275\u0254\u1d11'},{b:'oe',c:'\u0153'},{b:'oi',c:'\u01a3'},{b:'oo',c:'\ua74f'},{b:'ou',c:'\u0223'},{b:'p',c:'\u24df\uff50\u1e55\u1e57\u01a5\u1d7d\ua751\ua753\ua755\u03c1'},{b:'q',c:'\u24e0\uff51\u024b\ua757\ua759'}, {b:'r',c:'\u24e1\uff52\u0155\u1e59\u0159\u0211\u0213\u1e5b\u1e5d\u0157\u1e5f\u024d\u027d\ua75b\ua7a7\ua783'},{b:'s',c:'\u24e2\uff53\u015b\u1e65\u015d\u1e61\u0161\u1e67\u1e63\u1e69\u0219\u015f\u023f\ua7a9\ua785\u1e9b\u0282'},{b:'ss',c:'\u00df'},{b:'t',c:'\u24e3\uff54\u1e6b\u1e97\u0165\u1e6d\u021b\u0163\u1e71\u1e6f\u0167\u01ad\u0288\u2c66\ua787'},{b:'th',c:'\u00fe'},{b:'tz',c:'\ua729'},{b:'u',c:'\u24e4\uff55\u00f9\u00fa\u00fb\u0169\u1e79\u016b\u1e7b\u016d\u00fc\u01dc\u01d8\u01d6\u01da\u1ee7\u016f\u0171\u01d4\u0215\u0217\u01b0\u1eeb\u1ee9\u1eef\u1eed\u1ef1\u1ee5\u1e73\u0173\u1e77\u1e75\u0289'}, {b:'v',c:'\u24e5\uff56\u1e7d\u1e7f\u028b\ua75f\u028c'},{b:'vy',c:'\ua761'},{b:'w',c:'\u24e6\uff57\u1e81\u1e83\u0175\u1e87\u1e85\u1e98\u1e89\u2c73'},{b:'x',c:'\u24e7\uff58\u1e8b\u1e8d'},{b:'y',c:'\u24e8\uff59\u1ef3\u00fd\u0177\u1ef9\u0233\u1e8f\u00ff\u1ef7\u1e99\u1ef5\u01b4\u024f\u1eff'},{b:'z',c:'\u24e9\uff5a\u017a\u1e91\u017c\u017e\u1e93\u1e95\u01b6\u0225\u0240\u2c6c\ua763'}]; for (var i=0; i Date: Sun, 3 Jun 2018 22:51:22 +0200 Subject: [PATCH 0374/1669] Updated test. --- test/test-css.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test-css.js b/test/test-css.js index c6e16ff95..eff390911 100755 --- a/test/test-css.js +++ b/test/test-css.js @@ -13,8 +13,6 @@ buffer.push('div{background:linear-gradient(90deg, #000000, #FFFFFF);}'); var css = buffer.join('\n'); assert.ok(internal.compile_css(css) === 'b{border-radius:1px}a{border-radius:1px 2px 3px 4px}a{text-overflow:ellipsis}span{opacity:0;filter:alpha(opacity=0)}@keyframes test{border-radius:5px}@-webkit-keyframes test{border-radius:5px}@-moz-keyframes test{border-radius:5px}@-o-keyframes test{border-radius:5px}div{background:-webkit-linear-gradient(90deg,#000000,#FFFFFF);background:-moz-linear-gradient(90deg,#000000,#FFFFFF);background:-ms-linear-gradient(90deg,#000000,#FFFFFF);background:linear-gradient(90deg,#000000,#FFFFFF)}', 'automated CSS vendor prefixes'); -// console.log(internal.compile_css('/*auto*/\ndiv{background:repeating-linear-gradient(90deg, #000000, #FFFFFF);}')); - css = '.input{ }, .input:disabled, .input:hover { background-color: red; } .required{content:"This, field is required"}'; assert.ok(internal.compile_css(css) === '.input{},.input:disabled,.input:hover{background-color:red}.required{content:"This, field is required"}', 'Problem with content.'); From c719cc3735a65d09afd790bc2774cf311d8f640d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 5 Jun 2018 10:45:46 +0200 Subject: [PATCH 0375/1669] Fixed `F.path.mkdir()`. --- index.js | 25 +++++++++++++++++++------ package.json | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index dd17b59d7..621ac0e34 100755 --- a/index.js +++ b/index.js @@ -9436,31 +9436,44 @@ FrameworkPath.prototype.verify = function(name) { FrameworkPath.prototype.mkdir = function(p) { - if (p[0] === '/') + var is = F.isWindows; + var s = ''; + + if (p[0] === '/') { + s = is ? '\\' : '/'; p = p.substring(1); + } - var is = F.isWindows; var l = p.length - 1; + var beg = 0; if (is) { if (p[l] === '\\') p = p.substring(0, l); + + if (p[1] === ':') + beg = 1; + } else { if (p[l] === '/') p = p.substring(0, l); } var arr = is ? p.replace(/\//g, '\\').split('\\') : p.split('/'); - var directory = is ? '' : '/'; + var directory = s; for (var i = 0, length = arr.length; i < length; i++) { var name = arr[i]; if (is) - directory += (directory ? '\\' : '') + name; + directory += (i && directory ? '\\' : '') + name; else - directory += (directory ? '/' : '') + name; - !existsSync(directory) && Fs.mkdirSync(directory); + directory += (i && directory ? '/' : '') + name; + + if (i >= beg && !existsSync(directory)) + Fs.mkdirSync(directory); } + + return F; }; FrameworkPath.prototype.exists = function(path, callback) { diff --git a/package.json b/package.json index e233fe818..b32de88ae 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-67", + "version": "3.0.0-68", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From d6470ab6fc7f089e6f4ab28397cd803d1bd6b550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 5 Jun 2018 10:46:11 +0200 Subject: [PATCH 0376/1669] Added a new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index a3836e345..f9c986027 100755 --- a/changes.txt +++ b/changes.txt @@ -121,6 +121,7 @@ - fixed: `controller.href()` with Array values - fixed: `U.get()` a problem with path with `-` - fixed: `U.set()` a problem with path with `-` +- fixed: `F.path.mkdir()` in Windows and Linux - replaced: config `disable-clear-temporary-directory` to `allow-clear-temp : true|false` - replaced: config `disable-strict-server-certificate-validation` to `allow-ssc-validation : true|false` From 1496d5bdaaace8a2d0bf76aea436a618cf205c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 5 Jun 2018 18:53:00 +0200 Subject: [PATCH 0377/1669] Fixed `/versions` with `auto` value. --- index.js | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index 621ac0e34..b7aa74dc7 100755 --- a/index.js +++ b/index.js @@ -2375,6 +2375,19 @@ global.MERGE = F.merge = function(url) { if (url[0] === '#') url = sitemapurl(url.substring(1)); + url = F.$version(url); + + if (url === 'auto') { + // auto-generating + var arg = arguments; + setTimeout(function(arg) { + F.merge.apply(F, arg); + }, 500, arg); + return F; + } + + url = framework_internal.preparePath(url); + var arr = []; for (var i = 1, length = arguments.length; i < length; i++) { @@ -2396,8 +2409,6 @@ global.MERGE = F.merge = function(url) { } } - url = framework_internal.preparePath(F.$version(url)); - if (url[0] !== '/') url = '/' + url; @@ -5897,7 +5908,7 @@ function compile_content(extension, content, filename) { for (var i = 0, length = matches.length; i < length; i++) { var key = matches[i]; var url = key.substring(4, key.length - 1); - content = content.replace(key, 'url(' + F.$version(url) + ')'); + content = content.replace(key, 'url(' + F.$version(url, true) + ')'); } } return content; @@ -8615,6 +8626,8 @@ F.$configure_versions = function(arr, clean) { if (ismap) throw new Error('/versions: "auto" value can\'t be used with mapping'); + F.versions[key] = filename; + (function(key, filename) { ON('ready', function() { F.consoledebug('"versions" is getting checksum of ' + key); @@ -8628,6 +8641,7 @@ F.$configure_versions = function(arr, clean) { }); }); })(key, filename); + } else { F.versions[key] = filename; ismap && F.map(filename, F.path.public(key)); @@ -8653,6 +8667,7 @@ function makehash(url, callback) { hash.end(); callback(hash.read().crc32(true)); }); + stream.on('error', () => callback('')); }); } @@ -9001,22 +9016,23 @@ F.$routeStatic = function(name, directory, theme) { if (REG_ROUTESTATIC.test(name)) filename = name; else if (name[0] === '/') - filename = U.join(theme, F.$version(name)); + filename = U.join(theme, F.$version(name, true)); else { - filename = U.join(theme, directory, F.$version(name)); + filename = U.join(theme, directory, F.$version(name, true)); if (REG_HTTPHTTPS.test(filename) && filename[0] === '/') filename = filename.substring(1); } - return F.temporary.other[key] = framework_internal.preparePath(F.$version(filename)); + return F.temporary.other[key] = framework_internal.preparePath(F.$version(filename, true)); }; -F.$version = function(name) { +F.$version = function(name, def) { + var tmp; if (F.versions) - name = F.versions[name] || name; + tmp = F.versions[name] || name; if (F.onVersion) - name = F.onVersion(name) || name; - return name; + tmp = F.onVersion(name) || name; + return tmp === 'auto' && def ? name : (tmp || name); }; F.$versionprepare = function(html) { @@ -9034,7 +9050,7 @@ F.$versionprepare = function(html) { end = 6; var name = src.substring(end, src.length - 1); - html = html.replace(match[i], src.substring(0, end) + F.$version(name) + '"'); + html = html.replace(match[i], src.substring(0, end) + F.$version(name, true) + '"'); } return html; From 60956274f674695c4a6b87715407cba124c47d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 5 Jun 2018 18:53:12 +0200 Subject: [PATCH 0378/1669] Improved NOSQL responses. --- nosql.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nosql.js b/nosql.js index 70189b533..158c9fc9d 100755 --- a/nosql.js +++ b/nosql.js @@ -1809,7 +1809,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else - output = item.response || EMPTYARRAY; + output = item.response || []; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); continue; @@ -1835,7 +1835,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { if (builder.$options.first) output = item.response ? item.response[0] : undefined; else - output = item.response || EMPTYARRAY; + output = item.response || []; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); builder.done(); @@ -2003,7 +2003,7 @@ Database.prototype.$reader3 = function() { else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else - output = item.response || EMPTYARRAY; + output = item.response || []; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); continue; @@ -2029,7 +2029,7 @@ Database.prototype.$reader3 = function() { if (builder.$options.first) output = item.response ? item.response[0] : undefined; else - output = item.response || EMPTYARRAY; + output = item.response || []; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); builder.done(); @@ -2248,7 +2248,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { else if (builder.$options.first) output = item.response ? item.response[0] : undefined; else - output = item.response || EMPTYARRAY; + output = item.response || []; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); continue; @@ -2273,7 +2273,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { if (builder.$options.first) output = item.response ? item.response[0] : undefined; else - output = item.response || EMPTYARRAY; + output = item.response || []; builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); builder.done(); @@ -2903,7 +2903,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { }; db.find(join.view, join.builder).in(join.a, unique); } else { - join.items = join.builder.$options.first ? null : EMPTYARRAY; + join.items = join.builder.$options.first ? null : []; next(); } } From b1aa50e0634dda96d36c29753e3bcced866bb9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 6 Jun 2018 12:40:09 +0200 Subject: [PATCH 0379/1669] Added missing block scope. --- builders.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/builders.js b/builders.js index f43b55866..a9c42c4d3 100755 --- a/builders.js +++ b/builders.js @@ -177,13 +177,10 @@ SchemaOptions.prototype.success = function(a, b) { SchemaOptions.prototype.done = function(arg) { var self = this; return function(err, response) { - if (err && !(err instanceof ErrorBuilder)) { self.error.push(err); self.callback(); - } - - if (arg) + } else if (arg) self.callback(SUCCESS(err == null, response)); else self.callback(SUCCESS(err == null)); From 5212bfd1e1aa231f895ad1159d0eed05885c8c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 6 Jun 2018 12:40:22 +0200 Subject: [PATCH 0380/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b32de88ae..4464af01c 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-68", + "version": "3.0.0-69", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From d569e2a26db272f191e90d466fc5bc522b9bdcd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 7 Jun 2018 21:11:12 +0200 Subject: [PATCH 0381/1669] Fixed redirects in request. --- utils.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/utils.js b/utils.js index 361a43473..e35b8a562 100755 --- a/utils.js +++ b/utils.js @@ -853,7 +853,13 @@ function request_response(res, uri, options) { options.redirect++; - var tmp = Url.parse(res.headers['location']); + var loc = res.headers['location']; + var proto = loc.substring(0, 6); + + if (proto !== 'http:/' && proto !== 'https:/') + loc = uri.protocol + '//' + uri.hostname + loc; + + var tmp = Url.parse(loc); tmp.headers = uri.headers; tmp.agent = false; @@ -868,8 +874,8 @@ function request_response(res, uri, options) { options.uri = tmp; options.uri.agent = new ProxyAgent(options); options.uri.agent.request = Http.request; - options.uri.agent.createSocket = createSecureSocket; - options.uri.agent.defaultPort = 443; + options.uri.agent.createSocket = createSecureSocket; + options.uri.agent.defaultPort = 443; } if (!options.resolve) { @@ -878,7 +884,7 @@ function request_response(res, uri, options) { return request_call(tmp, options); } - exports.resolve(res.headers['location'], function(err, u) { + exports.resolve(tmp, function(err, u) { if (!err) tmp.host = u.host; res.removeAllListeners(); From 3e6ff05abc35ea83793cb27e41e16a7f3981ddd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 7 Jun 2018 21:28:05 +0200 Subject: [PATCH 0382/1669] Fixed redirects. --- utils.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils.js b/utils.js index e35b8a562..a39e57eb8 100755 --- a/utils.js +++ b/utils.js @@ -856,11 +856,10 @@ function request_response(res, uri, options) { var loc = res.headers['location']; var proto = loc.substring(0, 6); - if (proto !== 'http:/' && proto !== 'https:/') + if (proto !== 'http:/' && proto !== 'https:') loc = uri.protocol + '//' + uri.hostname + loc; var tmp = Url.parse(loc); - tmp.headers = uri.headers; tmp.agent = false; tmp.method = uri.method; From 3052278f43d08f9075979f85388dbf5faf3b087a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 11 Jun 2018 21:52:04 +0200 Subject: [PATCH 0383/1669] Updated removing of comments. --- internal.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal.js b/internal.js index db62bc962..1bdd1d328 100755 --- a/internal.js +++ b/internal.js @@ -1768,8 +1768,7 @@ function localize(language, command) { function view_parse(content, minify, filename, controller) { - if (minify) - content = removeComments(content); + content = removeComments(content); var nocompressHTML = false; var nocompressJS = false; @@ -1868,6 +1867,10 @@ function view_parse(content, minify, filename, controller) { if (!value) return '$EMPTY'; + if (!minify) { + + } + if (!nocompressHTML && is) value += ' '; From d5992f1b7099ee4e99b432ec8b82550da80b2626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 11 Jun 2018 21:52:09 +0200 Subject: [PATCH 0384/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4464af01c..19a5deacb 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-69", + "version": "3.0.0-70", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 364276c1526bf0d50ddd4106ad095a9ba7c3167b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 12 Jun 2018 10:07:50 +0200 Subject: [PATCH 0385/1669] Fixed rendering of components. --- index.js | 4 ++-- internal.js | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index b7aa74dc7..e1a9757b2 100755 --- a/index.js +++ b/index.js @@ -3864,7 +3864,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.uninstall(type, uptodateName || name, uptodateName ? 'uptodate' : undefined); - var hash = '\n/*' + name.hash() + '*/\n'; + var hash = '\n/*' + name.crc32(true) + '*/\n'; var temporary = (F.id ? 'i-' + F.id + '_' : '') + 'components'; content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); @@ -3929,7 +3929,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } if (obj && obj.group) { - key = obj.group.hash(); + key = obj.group.crc32(true); temporary += '_g' + key; tmp = F.components.groups[obj.group]; if (!tmp) diff --git a/internal.js b/internal.js index 1bdd1d328..bd7c3c085 100755 --- a/internal.js +++ b/internal.js @@ -1823,7 +1823,10 @@ function view_parse(content, minify, filename, controller) { var index = 0; var isCookie = false; - if ((controller.$hasComponents || REG_COMPONENTS.test(content)) && REG_HEAD.test(content)) { + if (!controller.$hasComponents) + controller.$hasComponents = REG_COMPONENTS.test(content) && REG_HEAD.test(content); + + if (controller.$hasComponents) { index = content.indexOf('@{import('); From e8c4cc49743f19c971f52f80de7297639783f7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 12 Jun 2018 10:09:24 +0200 Subject: [PATCH 0386/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 19a5deacb..b4bf3a855 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-70", + "version": "3.0.0-71", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From be09e226cb96117311ce6d816149eed8a9f18533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 16:23:52 +0200 Subject: [PATCH 0387/1669] Added nested files in `components`, etc. --- changes.txt | 2 + index.js | 91 ++++++++++++++++++++++++++------ test/components/contactform.html | 10 +++- test/test-framework-debug.js | 18 +++++++ test/test-framework-release.js | 18 +++++++ utils.js | 2 + 6 files changed, 124 insertions(+), 17 deletions(-) diff --git a/changes.txt b/changes.txt index f9c986027..5f17604ed 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: (IMPORTANT) bundles - added: (IMPORTANT) Total.js components can have async delegate +- added: (IMPORTANT) Total.js components support nested public files encoded in base64 - added: (IMPORTANT) NoSQL worker - added: (IMPORTANT) NoSQL embedded storage for smaller big data / IoT - added: (IMPORTANT) NoSQL indexes @@ -82,6 +83,7 @@ - added: `schema.define()(DEFAULT_VALUE)` added `DEFAULT_VALUE` - added: `TESTUSER([user])` for faking of `F.onAuthorize` delegate, targeted for unit-testing only - added: `G` as a global alias for `F.global` +- added: a simple support for `.heic` and `.heif` image format - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index e1a9757b2..dad63261b 100755 --- a/index.js +++ b/index.js @@ -49,7 +49,7 @@ const COMPRESSIONSPECIAL = { 'js': 1, 'css': 1 }; const REG_TEMPORARY = /\//g; const REG_MOBILE = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|Tablet/i; const REG_ROBOT = /search|agent|bot|crawler|spider/i; -const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.]+\.(jpg|js|css|png|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; +const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; const REG_NEWIMPL = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; @@ -91,6 +91,7 @@ const CLUSTER_CACHE_CLEAR = { TYPE: 'cache-clear' }; const GZIPFILE = { memLevel: 9 }; const GZIPSTREAM = { memLevel: 1 }; const MODELERROR = {}; +const IMAGES = { jpg: 1, png: 1, gif: 1, apng: 1, jpeg: 1, heif: 1, heic: 1, webp: 1 }; var PATHMODULES = require.resolve('./index'); PATHMODULES = PATHMODULES.substring(0, PATHMODULES.length - 8); @@ -643,7 +644,7 @@ function Framework() { 'static-url-font': '/fonts/', 'static-url-download': '/download/', 'static-url-components': '/components.', - 'static-accepts': { 'flac': true, 'jpg': true, 'jpeg': true, 'png': true, 'gif': true, 'ico': true, 'js': true, 'css': true, 'txt': true, 'xml': true, 'woff': true, 'woff2': true, 'otf': true, 'ttf': true, 'eot': true, 'svg': true, 'zip': true, 'rar': true, 'pdf': true, 'docx': true, 'xlsx': true, 'doc': true, 'xls': true, 'html': true, 'htm': true, 'appcache': true, 'manifest': true, 'map': true, 'ogv': true, 'ogg': true, 'mp4': true, 'mp3': true, 'webp': true, 'webm': true, 'swf': true, 'package': true, 'json': true, 'md': true, 'm4v': true, 'jsx': true }, + 'static-accepts': { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, js: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, md: true, m4v: true, jsx: true, heif: true, heic: true }, // 'static-accepts-custom': [], @@ -1401,7 +1402,7 @@ F.resize = function(url, fn, flags) { fn = tmp; } - var ext = url.match(/\*.\*$|\*?\.(jpg|png|gif|jpeg)$/gi); + var ext = url.match(/\*.\*$|\*?\.(jpg|png|gif|jpeg|heif|heic|apng)$/gi); if (ext) { url = url.replace(ext, ''); switch (ext.toString().toLowerCase()) { @@ -1411,6 +1412,9 @@ F.resize = function(url, fn, flags) { case '*.jpg': case '*.gif': case '*.png': + case '*.heif': + case '*.heic': + case '*.apng': case '*.jpeg': extensions[ext.toString().toLowerCase().replace(/\*/g, '').substring(1)] = true; break; @@ -1436,6 +1440,9 @@ F.resize = function(url, fn, flags) { extensions['jpeg'] = true; extensions['png'] = true; extensions['gif'] = true; + extensions['heic'] = true; + extensions['heif'] = true; + extensions['apng'] = true; } if (extensions['jpg'] && !extensions['jpeg']) @@ -5121,8 +5128,12 @@ F.onMapping = function(url, def, ispublic, encode) { if (url[0] !== '/') url = '/' + url; + // component files + if (url[1] === '~') + return F.path.temp() + url.substring(1); + if (F._length_themes) { - var index = url.indexOf('/', 1); + var index = url.indexOf('/', 2); if (index !== -1) { var themeName = url.substring(1, index); if (F.themes[themeName]) @@ -7936,8 +7947,19 @@ F.clear = function(callback, isInit) { continue; (filename.indexOf('/') === -1 || filename.indexOf('.package/') !== -1) && !filename.endsWith('.jsoncache') && arr.push(files[i]); } + files = arr; - directories = directories.remove(n => n.indexOf('.package') === -1); + directories = directories.remove(function(name) { + name = U.getName(name); + + if (name[0] === '~') + return false; + + if (name.endsWith('.package')) + return false; + + return true; + }); } F.unlink(files, () => F.rmdir(directories, callback)); @@ -11702,6 +11724,10 @@ Controller.prototype.$import = function() { case '.svg': case '.png': case '.jpeg': + case '.heif': + case '.webp': + case '.heic': + case '.apng': builder += self.routeImage(filename); break; case '.mp4': @@ -15404,24 +15430,27 @@ function extend_response(PROTO) { return res; } - var name = req.uri.pathname; - var index = name.lastIndexOf('/'); - var resizer = F.routes.resize[name.substring(0, index + 1)]; - var canResize = false; + var canresize = false; var filename = null; + var name = req.uri.pathname; - if (resizer) { - name = name.substring(index + 1); - canResize = resizer.extension['*'] || resizer.extension[req.extension]; - if (canResize) { - name = resizer.path + $decodeURIComponent(name); - filename = F.onMapping(name, name, false, false); + if (IMAGES[req.extension]) { + var index = name.lastIndexOf('/'); + var resizer = F.routes.resize[name.substring(0, index + 1)]; + if (resizer) { + name = name.substring(index + 1); + canresize = resizer.extension['*'] || resizer.extension[req.extension]; + if (canresize) { + name = resizer.path + $decodeURIComponent(name); + filename = F.onMapping(name, name, false, false); + } else + filename = F.onMapping(name, name, true, true); } else filename = F.onMapping(name, name, true, true); } else filename = F.onMapping(name, name, true, true); - if (!canResize) { + if (!canresize) { if (F.components.has && F.components[req.extension] && req.uri.pathname === F.config['static-url-components'] + req.extension) { res.noCompress = true; @@ -16557,9 +16586,39 @@ function parseComponent(body, filename) { response.css = ''; response.js = ''; response.install = ''; + response.files = {}; var beg = 0; var end = 0; + var comname = U.getName(filename); + + // Files + while (true) { + beg = body.indexOf(''); diff --git a/test/components/contactform.html b/test/components/contactform.html index 3bd7df67c..2700857f0 100644 --- a/test/components/contactform.html +++ b/test/components/contactform.html @@ -1,3 +1,11 @@ + + VG90YWwuanMgdjM= + + + + UGV0ZXIgU2lya2E= + + @@ -6,4 +14,4 @@ \ No newline at end of file + diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index a7b40761c..833a3d715 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -645,6 +645,24 @@ function test_routing(next) { }); }); + async.await('component-filename-1', function(complete) { + utils.request(url + '~contactform/a.txt', [], function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === 'Total.js v3', 'problem with component files 1'); + complete(); + }); + }); + + async.await('component-filename-2', function(complete) { + utils.request(url + '~contactform/b.txt', [], function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === 'Peter Sirka', 'problem with component files 2'); + complete(); + }); + }); + async.await('static-file-notfound-because-directory1', function(complete) { utils.request(url + 'directory.txt', [], function(error, data, code, headers) { if (error) diff --git a/test/test-framework-release.js b/test/test-framework-release.js index c0c9ec8d8..6b28ef1bf 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -645,6 +645,24 @@ function test_routing(next) { }); }); + async.await('component-filename-1', function(complete) { + utils.request(url + '~contactform/a.txt', [], function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === 'Total.js v3', 'problem with component files 1'); + complete(); + }); + }); + + async.await('component-filename-2', function(complete) { + utils.request(url + '~contactform/b.txt', [], function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === 'Peter Sirka', 'problem with component files 2'); + complete(); + }); + }); + async.await('static-file-notfound-because-directory1', function(complete) { utils.request(url + 'directory.txt', [], function(error, data, code, headers) { if (error) diff --git a/utils.js b/utils.js index a39e57eb8..11dda857d 100755 --- a/utils.js +++ b/utils.js @@ -118,6 +118,8 @@ var CONTENTTYPES = { 'geojson': 'application/json', 'gif': 'image/gif', 'gzip': 'application/x-gzip', + 'heic': 'image/heic', + 'heif': 'image/heif', 'htm': 'text/html', 'html': 'text/html', 'ico': 'image/x-icon', From 6930c5a75881a7e8288a52a4559d797bab5f769f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 17:37:26 +0200 Subject: [PATCH 0388/1669] Improved static fiels. --- index.js | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index dad63261b..7542c9909 100755 --- a/index.js +++ b/index.js @@ -755,7 +755,7 @@ function Framework() { this.controllers = {}; this.dependencies = {}; this.isomorphic = {}; - this.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {} }; + this.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; this.convertors = []; this.convertors2 = null; this.tests = []; @@ -3875,6 +3875,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe var temporary = (F.id ? 'i-' + F.id + '_' : '') + 'components'; content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); + content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); @@ -3884,6 +3885,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (content.css) F.components.css = true; + if (content.files) + F.components.files[name] = content.files; + else + delete F.components.files[name]; + if (content.body) { F.components.views[name] = '.' + F.path.temp('component_' + name); Fs.writeFile(F.components.views[name].substring(1) + '.html', U.minifyHTML(content.body), NOOP); @@ -4509,7 +4515,7 @@ F.$restart = function() { F.schedules = []; F.isLoaded = false; F.isRestarted = false; - F.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {} }; + F.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; PERF = {}; F.routes = { @@ -4889,6 +4895,7 @@ F.uninstall = function(type, name, options, skipEmit, packageName) { delete F.components.instances[name]; delete F.components.views[name]; + delete F.components.files[name]; delete F.temporary.ready[type + '#' + name]; var temporary = (F.id ? 'i-' + F.id + '_' : '') + 'components'; @@ -5129,8 +5136,11 @@ F.onMapping = function(url, def, ispublic, encode) { url = '/' + url; // component files - if (url[1] === '~') - return F.path.temp() + url.substring(1); + if (url[1] === '~') { + var index = url.indexOf('/', 2); + var name = url.substring(2, index); + return F.components.files[name] && F.components.files[name][url.substring(index + 1)] ? (F.path.temp() + url.substring(1)) : null; + } if (F._length_themes) { var index = url.indexOf('/', 2); @@ -15450,6 +15460,11 @@ function extend_response(PROTO) { } else filename = F.onMapping(name, name, true, true); + if (!filename) { + res.throw404(); + return; + } + if (!canresize) { if (F.components.has && F.components[req.extension] && req.uri.pathname === F.config['static-url-components'] + req.extension) { From e4cfc893c632e64b6378092f443f7847cdd2c3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 20:01:29 +0200 Subject: [PATCH 0389/1669] Improved routing and schemas. --- changes.txt | 2 ++ index.js | 61 +++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 5f17604ed..692bdad29 100755 --- a/changes.txt +++ b/changes.txt @@ -57,6 +57,7 @@ - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` +- added: `VIEW()` a global alias for `F.view()` - added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue - added: `$SAVE(schema, model, [options], [callback], [controller])` performs `schema.save()` - added: `$INSERT(schema, model, [options], [callback], [controller])` performs `schema.insert()` @@ -90,6 +91,7 @@ - updated: `NEWSCHEMA()` supports `NEWSCHEMA('group/name')` - updated: `ROUTE()`, extended syntax for schemas, for example: `Schema --> @name` (more in docs.) - updated: `ROUTE()` supports a new HTTP method definition `ROUTE('GET /api/users/')`, `ROUTE('POST /api/users/')`, etc. +- updated: `ROUTE()` supports a schema definition directly in the URL `ROUTE('GET /api/users/ *User --> @save')`, etc. - updated: `tpm` supports a new command called `bundle`, for example: `tpm bundle cms` - updated: `F.restore()` filter can return a new filename (for files only) - updated: `@{import('livereload')}` or `@{import('livereload wss://mywebsite.com')}` supports `livereload` value and it's rendered in `debug` mode only diff --git a/index.js b/index.js index 7542c9909..5e3a6147a 100755 --- a/index.js +++ b/index.js @@ -1768,6 +1768,12 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { method = url.substring(0, index).toLowerCase().trim(); url = url.substring(index + 1).trim(); } + + url = url.replace(/\s\*[a-z0-9].*?$/i, function(text) { + !flags && (flags = []); + flags.push(text.trim()); + return ''; + }); } if (url[0] === '#') { @@ -2080,8 +2086,17 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { method = 'get'; } - if (workflow && workflow[0] === '@') - workflow = { id: workflow.substring(1) }; + if (workflow && workflow[0] === '@') { + var tmpa = workflow.split('@').trim(); + var rindex = null; + for (var i = 0; i < tmpa.length; i++) { + var a = tmpa[i].split(' '); + if (a[1] && (/response|res/i).test(a[1])) + rindex = i; + tmpa[i] = a[0]; + } + workflow = { id: tmpa.length > 1 ? tmpa : tmpa[0], index: rindex }; + } if (type === 'string') { viewname = funcExecute; @@ -2145,7 +2160,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { }; })(viewname, sitemap, language); } else if (workflow) - funcExecute = controller_json_workflow; + funcExecute = workflow.id instanceof Array ? controller_json_workflow_multiple : controller_json_workflow; } if (!isGENERATOR) @@ -7844,7 +7859,7 @@ global.MAIL = F.mail = function(address, subject, view, model, callback, languag * @param {String} language Optional. * @return {String} */ -F.view = function(name, model, layout, repository, language) { +global.VIEW = F.view = function(name, model, layout, repository, language) { var controller = EMPTYCONTROLLER; @@ -16722,6 +16737,44 @@ function controller_json_workflow(id) { self.$exec(w, null, self.callback()); } +// Default action for workflow routing +function controller_json_workflow_multiple(id) { + var self = this; + self.id = id; + var w = self.route.workflow; + if (w instanceof Object) { + if (!w.type) { + var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); + var op = []; + for (var i = 0; i < w.id.length; i++) { + var id = w.id[i]; + if (schema.meta[id] !== undefined) { + op.push({ name: '$' + id }); + } else if (schema.meta['workflow#' + id] !== undefined) { + op.push({ name: '$workflow', id: id }); + } else if (schema.meta['workflow#' + id] !== undefined) { + op.push({ name: '$transform', id: id }); + } else if (schema.meta['operation#' + id] !== undefined) { + op.push({ name: '$operation', id: id }); + } else if (schema.meta['hook#' + id] !== undefined) { + op.push({ name: '$hook', id: id }); + } + } + w.async = op; + } + + var async = self.$async(self.callback(), w.index); + for (var i = 0; i < w.async.length; i++) { + var a = w.async[i]; + if (a.id) + async[a.name](a.id); + else + async[a.name](); + } + } else + self.$exec(w, null, self.callback()); +} + // Parses schema group and schema name from string e.g. "User" or "Company/User" function parseSchema(name) { var schema = F.temporary.internal['$$$' + name]; From d15b18422b0f9b9a35c69a89a8a885712b153158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 20:01:57 +0200 Subject: [PATCH 0390/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b4bf3a855..0d2a24244 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-71", + "version": "3.0.0-72", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 558c32ff716c9153ec719b2d7957a0dd873edef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 20:24:04 +0200 Subject: [PATCH 0391/1669] Fixed schema options. --- builders.js | 2 +- index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index a9c42c4d3..bb357dea3 100755 --- a/builders.js +++ b/builders.js @@ -47,7 +47,7 @@ function SchemaBuilder(name) { function SchemaOptions(error, model, options, callback, controller) { this.error = error; this.value = this.model = model; - this.options = options; + this.options = options || EMPTYOBJECT; this.callback = this.next = callback; this.controller = controller; } diff --git a/index.js b/index.js index 5e3a6147a..1d8db1fdc 100755 --- a/index.js +++ b/index.js @@ -2087,7 +2087,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { } if (workflow && workflow[0] === '@') { - var tmpa = workflow.split('@').trim(); + var tmpa = workflow.replace(/,/g, ' ').split('@').trim(); var rindex = null; for (var i = 0; i < tmpa.length; i++) { var a = tmpa[i].split(' '); From f0acf5f78dc25219912e90c3e999c39159c9a57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 21:55:00 +0200 Subject: [PATCH 0392/1669] Fixed routing prioritization. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 1d8db1fdc..c9992fcc3 100755 --- a/index.js +++ b/index.js @@ -2204,7 +2204,7 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { regIndex.push(i); }); - priority -= arr.length; + priority -= arr.length + 1; } if (url.indexOf('#') !== -1) From dbd680b9bbea6e01644d425276e2b6a818d56c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 13 Jun 2018 23:29:51 +0200 Subject: [PATCH 0393/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0d2a24244..bffa99be5 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-72", + "version": "3.0.0-73", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 18ff16b92fe585dcfe918ad31fe8a1be0f2201f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 14 Jun 2018 14:15:41 +0200 Subject: [PATCH 0394/1669] Fixed `RESTBuilder.accept()` with custom type. --- builders.js | 11 ++++++++++- package.json | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index bb357dea3..fff636882 100755 --- a/builders.js +++ b/builders.js @@ -4291,10 +4291,19 @@ RESTBuilder.prototype.urlencoded = function(data) { }; RESTBuilder.prototype.accept = function(ext) { - var type = framework_utils.getContentType(ext); + + var type; + + if (ext.length > 8) + type = ext; + else + type = framework_utils.getContentType(ext); + if (this.$headers['Accept'] !== type) this.$flags = null; + this.$headers['Accept'] = type; + return this; }; diff --git a/package.json b/package.json index bffa99be5..eda885e10 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-73", + "version": "3.0.0-74", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From e81ebe20129edb937e118877c4e7ac71a71f24fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 14 Jun 2018 14:23:10 +0200 Subject: [PATCH 0395/1669] Updated header. --- builders.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index fff636882..5887542ed 100755 --- a/builders.js +++ b/builders.js @@ -4299,10 +4299,11 @@ RESTBuilder.prototype.accept = function(ext) { else type = framework_utils.getContentType(ext); - if (this.$headers['Accept'] !== type) + if (this.$headers.Accept !== type) this.$flags = null; - this.$headers['Accept'] = type; + this.$flags = null; + this.$headers.Accept = type; return this; }; From 0e66af23594e18d0b3b5b7d05bb40a03b4bbe35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 14 Jun 2018 22:14:01 +0200 Subject: [PATCH 0396/1669] Fixed backup filename. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 158c9fc9d..339fbe5e2 100755 --- a/nosql.js +++ b/nosql.js @@ -522,6 +522,7 @@ function Database(name, filename, readonly) { var http = filename.substring(0, 6); self.readonly = http === 'http:/' || http === 'https:'; self.filename = self.readonly ? filename.format('') : filename + EXTENSION; + self.directory = Path.dirname(filename); if (!readonly) { self.filenameCounter = self.readonly ? filename.format('counter', '-') : filename + EXTENSION + EXTENSION_COUNTER; @@ -538,7 +539,6 @@ function Database(name, filename, readonly) { } self.filenameTemp = filename + EXTENSION_TMP; - self.directory = Path.dirname(filename); self.name = name; self.pending_update = []; self.pending_append = []; From 7ef4f974a267d66deba11ae5ffe90af056f0d9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 15 Jun 2018 13:06:16 +0200 Subject: [PATCH 0397/1669] Updated JS compressor. --- internal.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/internal.js b/internal.js index bd7c3c085..7e904103e 100755 --- a/internal.js +++ b/internal.js @@ -1299,21 +1299,28 @@ function minify_javascript(data) { } } else { if (vtmp) { - - vindex = index + 1; - while (true) { if (!data[vindex] || !white.test(data[vindex])) break; vindex++; } - if (c === '(' || c === ')' || (c === ';' && !regvar.test(data.substring(vindex, vindex + 20)))) vtmp = false; } } + if (c === '+' || c === '-' && next === ' ') { + if (data[index + 1] === c) { + index += 2; + output.push(c); + output.push(' '); + output.push(c); + last = c; + continue; + } + } + if ((c === '}' && last === ';') || ((c === '}' || c === ']') && output[output.length - 1] === ' ' && alpha.test(output[output.length - 2]))) output.pop(); From cf4ed6b8ee300b167c026f59a953b739390ce336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 20 Jun 2018 09:38:37 +0200 Subject: [PATCH 0398/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eda885e10..123b5c675 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-74", + "version": "3.0.0-75", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0d77fcf280695f74e20645333980ba5ea7a8b7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 21 Jun 2018 09:27:20 +0200 Subject: [PATCH 0399/1669] Fixed reading in `find2()`. --- nosqlstream.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosqlstream.js b/nosqlstream.js index d44562219..620a72fc1 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -147,8 +147,8 @@ NoSQLStream.prototype.readhelpers = function() { var tmp = self.buffer.toString('utf8', index); if (tmp[1] === '-') { - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); + self.buffer = self.buffer.slice(0, index); + index = self.buffer.lastIndexOf(NEWLINEBUFFER); if (index === -1) break; continue; From aad116a4d59b2e921d923e3258dd9f1ed77099ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 22 Jun 2018 13:48:23 +0200 Subject: [PATCH 0400/1669] Fixed `flags` in `req.authorize()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index c9992fcc3..1af2274d1 100755 --- a/index.js +++ b/index.js @@ -14525,7 +14525,7 @@ function extend_request(PROTO) { var req = this; - auth(req, req.res, req.flags, function(isAuthorized, user) { + auth(req, req.res, req.flags || [], function(isAuthorized, user) { if (typeof(isAuthorized) !== 'boolean') { user = isAuthorized; isAuthorized = !user; From 3cde2c42bbbadb92c2f322042605314415650aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 00:10:52 +0200 Subject: [PATCH 0401/1669] Added SchemaOptions `$.params` alias to `controller.params`. --- builders.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builders.js b/builders.js index 5887542ed..da24198c3 100755 --- a/builders.js +++ b/builders.js @@ -88,6 +88,10 @@ SchemaOptions.prototype = { get query() { return this.controller ? this.controller.query : null; + }, + + get params() { + return this.controller ? this.controller.params : null; } }; From 2df56e32f297d3703694ce3af507cbef2b0bddc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 00:15:39 +0200 Subject: [PATCH 0402/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 123b5c675..b8b8adaf8 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-75", + "version": "3.0.0-76", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 150032b27a32e1e16fe397c5c38ab5d3d2d9b311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 00:17:56 +0200 Subject: [PATCH 0403/1669] Extend `ErrorBuilder` by adding `.is` property. --- builders.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builders.js b/builders.js index da24198c3..465030f84 100755 --- a/builders.js +++ b/builders.js @@ -3193,6 +3193,10 @@ ErrorBuilder.prototype = { var self = this; !self.isPrepared && self.prepare(); return self._transform(); + }, + + get is() { + return this.items.length > 0; } }; From eb308e339058e33cae28a212bb4eba6826da6cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 09:02:24 +0200 Subject: [PATCH 0404/1669] Improved global aliases. --- builders.js | 4 ++++ index.js | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/builders.js b/builders.js index 465030f84..8aab44136 100755 --- a/builders.js +++ b/builders.js @@ -2002,6 +2002,7 @@ SchemaBuilderEntity.prototype.transform = function(name, model, options, callbac SchemaBuilderEntity.prototype.transform2 = function(name, options, callback, controller) { if (typeof(options) === 'function') { + controller = callback; callback = options; options = undefined; } @@ -2051,6 +2052,7 @@ SchemaBuilderEntity.prototype.workflow = function(name, model, options, callback SchemaBuilderEntity.prototype.workflow2 = function(name, options, callback, controller) { if (typeof(options) === 'function') { + controller = callback; callback = options; options = undefined; } @@ -2190,6 +2192,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk SchemaBuilderEntity.prototype.hook2 = function(name, options, callback, controller) { if (typeof(options) === 'function') { + controller = callback; callback = options; options = undefined; } @@ -2419,6 +2422,7 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac SchemaBuilderEntity.prototype.operation2 = function(name, options, callback, controller) { if (typeof(options) === 'function') { + controller = callback; callback = options; options = undefined; } diff --git a/index.js b/index.js index 1af2274d1..948c0f5a6 100755 --- a/index.js +++ b/index.js @@ -386,6 +386,13 @@ global.$TRANSFORM = function(schema, name, options, callback, controller) { global.$REMOVE = function(schema, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); + + if (typeof(options) === 'function') { + controller = callback; + callback = options; + options = EMPTYOBJECT; + } + o && o.remove(options, callback, controller); return !!o; }; From 4c663aa399cf6944612932728adc57815cbd83ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 11:27:34 +0200 Subject: [PATCH 0405/1669] Improved skip JS/CSS compression for files. --- index.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 948c0f5a6..ce42d50bc 100755 --- a/index.js +++ b/index.js @@ -61,7 +61,7 @@ const REG_WEBSOCKET_ERROR = /ECONNRESET|EHOSTUNREACH|EPIPE|is closed/i; const REG_WINDOWSPATH = /\\/g; const REG_SCRIPTCONTENT = /<|>|;/; const REG_HTTPHTTPS = /^(\/)?(http|https):\/\//i; -const REG_NOCOMPRESS = /[.|-]+min\.(css|js)$/i; +const REG_NOCOMPRESS = /[.|-]+min(@\d+)\.(css|js)$/i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; diff --git a/package.json b/package.json index b8b8adaf8..1076a4530 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-76", + "version": "3.0.0-77", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 09a009b3a2ce4fb32039f4dc8db836959a79b394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 11:36:40 +0200 Subject: [PATCH 0406/1669] Fixed a condintion for compression. --- index.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index ce42d50bc..d8a0c580d 100755 --- a/index.js +++ b/index.js @@ -61,7 +61,7 @@ const REG_WEBSOCKET_ERROR = /ECONNRESET|EHOSTUNREACH|EPIPE|is closed/i; const REG_WINDOWSPATH = /\\/g; const REG_SCRIPTCONTENT = /<|>|;/; const REG_HTTPHTTPS = /^(\/)?(http|https):\/\//i; -const REG_NOCOMPRESS = /[.|-]+min(@\d+)\.(css|js)$/i; +const REG_NOCOMPRESS = /[.|-]+min(@\d+)?\.(css|js)$/i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; diff --git a/package.json b/package.json index 1076a4530..2d9c38197 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-77", + "version": "3.0.0-78", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 25fe53a331e55a18e880e1547970d85d12ecb756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 23 Jun 2018 13:18:25 +0200 Subject: [PATCH 0407/1669] Fixed schema operations in controllers. --- index.js | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++- package.json | 2 +- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index d8a0c580d..07e011b66 100755 --- a/index.js +++ b/index.js @@ -10143,16 +10143,34 @@ Controller.prototype = { // Schema operations Controller.prototype.$get = Controller.prototype.$read = function(helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + this.getSchema().get(helper, callback, this); return this; }; Controller.prototype.$query = function(helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + this.getSchema().query(helper, callback, this); return this; }; Controller.prototype.$save = function(helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + var self = this; if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; @@ -10166,6 +10184,12 @@ Controller.prototype.$save = function(helper, callback) { }; Controller.prototype.$insert = function(helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + var self = this; if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; @@ -10179,6 +10203,12 @@ Controller.prototype.$insert = function(helper, callback) { }; Controller.prototype.$update = function(helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + var self = this; if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; @@ -10192,6 +10222,12 @@ Controller.prototype.$update = function(helper, callback) { }; Controller.prototype.$remove = function(helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + var self = this; self.getSchema().remove(helper, callback, self); return this; @@ -10199,6 +10235,12 @@ Controller.prototype.$remove = function(helper, callback) { Controller.prototype.$workflow = function(name, helper, callback) { var self = this; + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; self.body.$workflow(name, helper, callback); @@ -10208,6 +10250,12 @@ Controller.prototype.$workflow = function(name, helper, callback) { }; Controller.prototype.$workflow2 = function(name, helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = null; + } + var self = this; self.getSchema().workflow2(name, helper, callback, self); return self; @@ -10215,21 +10263,40 @@ Controller.prototype.$workflow2 = function(name, helper, callback) { Controller.prototype.$hook = function(name, helper, callback) { var self = this; + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = EMPTYOBJECT; + } + if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; self.body.$hook(name, helper, callback); } else self.getSchema().hook2(name, helper, callback, self); + return self; }; Controller.prototype.$hook2 = function(name, helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = EMPTYOBJECT; + } + var self = this; self.getSchema().hook2(name, helper, callback, self); return self; }; Controller.prototype.$transform = function(name, helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = EMPTYOBJECT; + } + var self = this; if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; @@ -10240,12 +10307,24 @@ Controller.prototype.$transform = function(name, helper, callback) { }; Controller.prototype.$transform2 = function(name, helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = EMPTYOBJECT; + } + var self = this; self.getSchema().transform2(name, helper, callback, self); return self; }; Controller.prototype.$operation = function(name, helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = EMPTYOBJECT; + } + var self = this; if (framework_builders.isSchema(self.body)) { self.body.$$controller = self; @@ -10256,6 +10335,12 @@ Controller.prototype.$operation = function(name, helper, callback) { }; Controller.prototype.$operation2 = function(name, helper, callback) { + + if (callback == null && typeof(helper) === 'function') { + callback = helper; + helper = EMPTYOBJECT; + } + var self = this; self.getSchema().operation2(name, helper, callback, self); return self; @@ -10264,7 +10349,7 @@ Controller.prototype.$operation2 = function(name, helper, callback) { Controller.prototype.$exec = function(name, helper, callback) { var self = this; - if (typeof(helper) === 'function') { + if (callback == null && typeof(helper) === 'function') { callback = helper; helper = EMPTYOBJECT; } diff --git a/package.json b/package.json index 2d9c38197..5fd98d8ef 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-78", + "version": "3.0.0-79", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From cc34bb24302a934e736e46394a32e502dd838b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 24 Jun 2018 09:26:27 +0200 Subject: [PATCH 0408/1669] Updated `nocompress` condition again. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 07e011b66..fa8ee5602 100755 --- a/index.js +++ b/index.js @@ -61,7 +61,7 @@ const REG_WEBSOCKET_ERROR = /ECONNRESET|EHOSTUNREACH|EPIPE|is closed/i; const REG_WINDOWSPATH = /\\/g; const REG_SCRIPTCONTENT = /<|>|;/; const REG_HTTPHTTPS = /^(\/)?(http|https):\/\//i; -const REG_NOCOMPRESS = /[.|-]+min(@\d+)?\.(css|js)$/i; +const REG_NOCOMPRESS = /[.|-]+min(@[a-z0-9]*)?\.(css|js)$/i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; From 2e0b6702369d7c992f6480cf498616c328384967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 24 Jun 2018 22:54:42 +0200 Subject: [PATCH 0409/1669] Updated `@{import()}` by adding `manifest` value. --- changes.txt | 1 + index.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 692bdad29..c907c3586 100755 --- a/changes.txt +++ b/changes.txt @@ -106,6 +106,7 @@ - updated: `CORS()` without arguments for all routes, methods and origins - updated: `CORS()` tries to join multiple same preferences to one - updated: `U.keywords()` for Chinese/Japan characters +- updated: `@{import()}` by adding `manifest` value linked to `/manifest.json` - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index fa8ee5602..fe648d589 100755 --- a/index.js +++ b/index.js @@ -11805,6 +11805,11 @@ Controller.prototype.$import = function() { continue; } + if (filename === 'manifest' || filename === 'manifest.json') { + builder += ''; + continue; + } + if (filename === 'favicon.ico' || filename === 'favicon.png') { builder += self.$favicon(filename); continue; From d324ded45fb092d432f3cd64abe0bf53a9f6dafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 24 Jun 2018 23:09:58 +0200 Subject: [PATCH 0410/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5fd98d8ef..83d701c7d 100755 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" }], - "version": "3.0.0-79", + "version": "3.0.0-80", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 90f26e8eec09c2adbcd9bf119ead2667009e0d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 12:18:04 +0200 Subject: [PATCH 0411/1669] Added `TABLE()` extension for NoSQL embedded database. --- index.js | 15 +- nosql.js | 1134 +++++++++++++++++++++++++++++++++++++++++++++++- nosqlstream.js | 9 +- nosqlworker.js | 3 +- 4 files changed, 1148 insertions(+), 13 deletions(-) diff --git a/index.js b/index.js index fe648d589..114fb103b 100755 --- a/index.js +++ b/index.js @@ -1270,6 +1270,16 @@ global.NOSQL = F.nosql = function(name) { return db; }; +global.TABLE = function(name) { + var db = F.databases['$' + name]; + if (db) + return db; + F.path.verify('databases'); + db = framework_nosql.table(name, F.path.databases(name)); + F.databases['$' + name] = db; + return db; +}; + F.stop = F.kill = function(signal) { if (F.isKilled) @@ -7204,7 +7214,10 @@ F.service = function(count) { if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { keys = Object.keys(F.databasescleaner); keys.wait(function(item, next) { - NOSQL(item).clean(next); + if (item[0] === '$') + TABLE(item).clean(next); + else + NOSQL(item).clean(next); }); } diff --git a/nosql.js b/nosql.js index 339fbe5e2..624d876fe 100755 --- a/nosql.js +++ b/nosql.js @@ -43,6 +43,7 @@ if (!global.framework_builders) global.framework_builders = require('./builders'); const EXTENSION = '.nosql'; +const EXTENSION_TABLE = '.table'; const EXTENSION_BINARY = '.nosql-binary'; const EXTENSION_TMP = '.nosql-tmp'; const EXTENSION_LOG = '.nosql-log'; @@ -68,6 +69,7 @@ const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' }; const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; const CLEANDBTICKS = 86400000 * 2; // 48 hours +const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); @@ -213,6 +215,8 @@ exports.worker = function() { obj.type = type; obj.name = instance.name; obj.time = Date.now(); + obj.t = instance instanceof Table; + if (arguments.length > 2) { obj.arg = []; for (var i = 2; i < arguments.length; i++) @@ -227,6 +231,8 @@ exports.worker = function() { obj.type = type; obj.name = instance.name; obj.time = Date.now(); + obj.t = instance instanceof Table; + if (arguments.length > 2) { obj.arg = []; for (var i = 2; i < arguments.length; i++) @@ -241,6 +247,7 @@ exports.worker = function() { CMD.arg = obj.arg; CMD.data = obj.builder ? obj.builder.stringify() : null; CMD.name = obj.name; + CMD.t = obj.t; if (callback !== false) { CMD.id = Math.random().toString(32).substring(2); FORKCALLBACKS[CMD.id] = obj; @@ -514,8 +521,113 @@ exports.worker = function() { return this; }; + Table.prototype.find = function(builder) { + if (builder) + builder.db = this; + else + builder = new DatabaseBuilder(this); + return send(this, 'find').builder = builder; + }; + + Table.prototype.top = function(max) { + var builder = new DatabaseBuilder(this); + builder.take(max); + return send(this, 'find').builder = builder; + }; + + Table.prototype.one = function() { + var builder = new DatabaseBuilder(this); + builder.first(); + return send(this, 'one').builder = builder; + }; + + Table.prototype.insert = function(doc, unique) { + + var self = this; + var builder; + + if (unique) { + builder = self.one(); + + var callback; + + builder.callback(function(err, d) { + if (d) + callback && callback(null, 0); + else { + var tmp = self.insert(doc); + tmp.callback(callback); + tmp.$options.log = builder.$options.log; + } + }); + + builder.callback = function(fn) { + callback = fn; + return builder; + }; + + return builder; + } + + return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); + }; + + Table.prototype.count = function() { + var builder = new DatabaseBuilder(this); + return send(this, 'count').builder = builder; + }; + + Table.prototype.update = function(doc, insert) { + return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + }; + + Table.prototype.modify = function(doc, insert) { + return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + }; + + Table.prototype.clear = function(callback) { + send(this, 'clear').callback = callback; + return this; + }; + + Table.prototype.clean = function(callback) { + send(this, 'clean').callback = callback; + return this; + }; }; +function Table(name, filename) { + var t = this; + t.filename = filename + EXTENSION_TABLE; + t.directory = Path.dirname(filename); + t.name = name; + t.$name = '$' + name; + t.pending_reader = []; + t.pending_update = []; + t.pending_append = []; + t.pending_reader = []; + t.pending_remove = []; + t.pending_streamer = []; + t.pending_clean = []; + t.pending_clear = []; + t.pending_locks = []; + t.$events = {}; + + t.step = 0; + t.ready = false; + t.$free = true; + t.$writting = false; + t.$reading = false; + + Fs.createReadStream(t.filename, { end: 2048 }).on('data', function(chunk) { + t.parseSchema(chunk.toString('utf8').split('\n')[0].split('|')); + t.ready = true; + t.next(0); + }).on('error', function() { + throw new Error('Table "{0}" doesn\'t contain schema'.format(name)); + }); +} + function Database(name, filename, readonly) { var self = this; @@ -566,7 +678,7 @@ function Database(name, filename, readonly) { self.$reading = false; } -Database.prototype.emit = function(name, a, b, c, d, e, f, g) { +Table.prototype.emit = Database.prototype.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -586,7 +698,7 @@ Database.prototype.emit = function(name, a, b, c, d, e, f, g) { return this; }; -Database.prototype.on = function(name, fn) { +Table.prototype.on = Database.prototype.on = function(name, fn) { if (!fn.$once) this.$free = false; @@ -598,12 +710,12 @@ Database.prototype.on = function(name, fn) { return this; }; -Database.prototype.once = function(name, fn) { +Table.prototype.once = Database.prototype.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -Database.prototype.removeListener = function(name, fn) { +Table.prototype.removeListener = Database.prototype.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -615,7 +727,7 @@ Database.prototype.removeListener = function(name, fn) { return this; }; -Database.prototype.removeAllListeners = function(name) { +Table.prototype.removeAllListeners = Database.prototype.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -632,11 +744,16 @@ exports.DatabaseCounter = Counter; exports.DatabaseBinary = Binary; exports.DatabaseIndexes = Indexes; exports.DatabaseStorage = Storage; +exports.DatabaseTable = Table; exports.load = function(name, filename) { return new Database(name, filename); }; +exports.table = function(name, filename) { + return new Table(name, filename); +}; + exports.memory = exports.inmemory = function(name, view) { if (view) name += '#' + view; @@ -2814,6 +2931,7 @@ function DatabaseBuilder(db) { this.$options = {}; this.$repository = {}; this.$counter = 0; + this.$keys = {}; } DatabaseBuilder.prototype.promise = promise; @@ -2861,7 +2979,8 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { unique.push(val); } - var db = NOSQL(join.name); + var isTable = self.db instanceof Table; + var db = isTable ? TABLE(join.name) : NOSQL(join.name); if (join.scalartype) { join.items = []; @@ -2901,7 +3020,10 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { join.items = docs; next(); }; - db.find(join.view, join.builder).in(join.a, unique); + if (isTable) + db.find(join.builder).in(join.a, unique); + else + db.find(join.view, join.builder).in(join.a, unique); } else { join.items = join.builder.$options.first ? null : []; next(); @@ -2999,6 +3121,10 @@ DatabaseBuilder.prototype.join = function(field, name, view) { self.$join[key].a = a; self.$join[key].b = b; + + if (self.$keys) + self.$keys[b] = 1; + return join; }; @@ -3120,6 +3246,9 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { if (self.$scope) code = 'if(!$is){' + code + '}'; + if (self.$keys) + self.$keys[name] = 1; + self.$code.push(code.format(name, key, operator)); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3133,6 +3262,9 @@ DatabaseBuilder.prototype.query = function(code) { if (self.$scope) code = 'if(!$is){' + code + '}'; + if (self.$keys) + self.$keys = null; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; @@ -3153,6 +3285,10 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { var code = compare_datetype('month', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; + + if (self.$keys) + self.$keys[name] = 1; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3172,6 +3308,10 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { var code = compare_datetype('day', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; + + if (self.$keys) + self.$keys[name] = 1; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3191,6 +3331,10 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { var code = compare_datetype('year', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; + + if (self.$keys) + self.$keys[name] = 1; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3225,6 +3369,9 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam if (self.$scope) code = 'if(!$is){' + code + '}'; + if (self.$keys) + self.$keys[name] = 1; + self.$code.push(code.format(name, key)); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3235,6 +3382,10 @@ DatabaseBuilder.prototype.regexp = function(name, value) { var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){$is=({1}).test(doc.{0})}'; if (self.$scope) code = 'if(!$is){' + code + '}'; + + if (self.$keys) + self.$keys[name] = 1; + self.$code.push(code.format(name, value.toString())); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3255,6 +3406,9 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { value = value.toLowerCase().split(' '); } + if (self.$keys) + self.$keys[name] = 1; + var count = 1; if (weight) @@ -3389,6 +3543,10 @@ DatabaseBuilder.prototype.compile = function() { DatabaseBuilder.prototype.in = function(name, value) { var self = this; + + if (self.$keys) + self.$keys[name] = 1; + var key = 'in' + (self.$counter++); self.$params[key] = value instanceof Array ? value : [value]; var code = 'if($is)$is=false;$tmp=doc.{0};if($tmp instanceof Array){for(var $i=0;$i<$tmp.length;$i++){if(arg.{1}.indexOf($tmp[$i])!==-1){$is=true;break}}}else{if(arg.{1}.indexOf($tmp)!==-1)$is=true}'.format(name, key); @@ -3401,6 +3559,10 @@ DatabaseBuilder.prototype.in = function(name, value) { DatabaseBuilder.prototype.notin = function(name, value) { var self = this; + + if (self.$keys) + self.$keys[name] = 1; + var key = 'in' + (self.$counter++); self.$params[key] = value instanceof Array ? value : [value]; var code = '$is=true;$tmp=doc.{0};if($tmp instanceof Array){for(var $i=0;$i<$tmp.length;$i++){if(arg.{1}.indexOf($tmp[$i])!==-1){$is=false;break}}}else{if(arg.{1}.indexOf($tmp)!==-1)$is=false}'.format(name, key); @@ -3413,6 +3575,10 @@ DatabaseBuilder.prototype.notin = function(name, value) { DatabaseBuilder.prototype.between = function(name, a, b) { var self = this; + + if (self.$keys) + self.$keys[name] = 1; + var keya = 'ba' + (self.$counter++); var keyb = 'bb' + (self.$counter++); var code = '$is=doc.{0}>=arg.{1}&&doc.{0}<=arg.{2};'.format(name, keya, keyb); @@ -3468,6 +3634,8 @@ DatabaseBuilder.prototype.fields = function() { !opt.fields && (opt.fields = []); opt.fields.push(name); } + if (self.$keys) + self.$keys[name] = 1; } return self; }; @@ -3488,6 +3656,9 @@ DatabaseBuilder.prototype.code = function(code) { self.$code.push(code + ';'); !self.$scope && self.$code.push('if(!$is)return;'); + if (self.$keys) + self.$keys = null; + return self; }; @@ -3503,6 +3674,9 @@ DatabaseBuilder.prototype.prepare = function(fn) { if (self.$scope) code = 'if(!$is){' + code + '}'; + if (self.$keys) + self.$keys = null; + self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return this; @@ -6227,6 +6401,952 @@ Backuper.prototype.flush = function() { return self; }; +Table.prototype.insert = function(doc, unique) { + + var self = this; + var builder; + + self.readonly && self.throwReadonly(); + + if (unique) { + + builder = self.one(); + var callback; + + builder.callback(function(err, d) { + if (d) + callback && callback(null, 0); + else + self.insert(doc).callback(callback); + }); + + builder.callback = function(fn) { + callback = fn; + return builder; + }; + + return builder; + } + + builder = new DatabaseBuilder2(self); + self.pending_append.push({ doc: doc, builder: builder }); + setImmediate(next_operation, self, 1); + self.$events.insert && self.emit('insert', doc); + return builder; +}; + +Table.prototype.update = function(doc, insert) { + var self = this; + self.readonly && self.throwReadonly(); + var builder = new DatabaseBuilder(self); + self.pending_update.push({ builder: builder, doc: doc, count: 0, insert: insert === true ? doc : insert }); + setImmediate(next_operation, self, 2); + return builder; +}; + +Table.prototype.modify = function(doc, insert) { + var self = this; + self.readonly && self.throwReadonly(); + var builder = new DatabaseBuilder(self); + var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; + var keys = Object.keys(data); + + if (!keys.length) + return builder; + + self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); + setImmediate(next_operation, self, 2); + return builder; +}; + +Table.prototype.remove = function() { + var self = this; + self.readonly && self.throwReadonly(); + var builder = new DatabaseBuilder(self); + self.pending_remove.push({ builder: builder, count: 0 }); + setImmediate(next_operation, self, 3); + return builder; +}; + +Table.prototype.find = function(builder) { + var self = this; + if (builder) + builder.db = self; + else + builder = new DatabaseBuilder(self); + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); + return builder; +}; + +Table.prototype.stream = function(fn, repository, callback) { + var self = this; + + if (typeof(repository) === 'function') { + callback = repository; + repository = null; + } + + self.pending_streamer.push({ fn: fn, callback: callback, repository: repository || {} }); + setImmediate(next_operation, self, 10); + return self; +}; + +Table.prototype.throwReadonly = function() { + throw new Error('Table "{0}" is readonly.'.format(this.name)); +}; + +Table.prototype.scalar = function(type, field) { + return this.find().scalar(type, field); +}; + +Table.prototype.count = function() { + var self = this; + var builder = new DatabaseBuilder(self); + self.pending_reader.push({ builder: builder, count: 0, type: 1 }); + setImmediate(next_operation, self, 4); + return builder; +}; + +Table.prototype.one = function() { + var self = this; + var builder = new DatabaseBuilder(self); + builder.first(); + self.pending_reader.push({ builder: builder, count: 0 }); + setImmediate(next_operation, self, 4); + return builder; +}; + +Table.prototype.top = function(max) { + var self = this; + var builder = new DatabaseBuilder(self); + builder.take(max); + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); + return builder; +}; + +Table.prototype.clean = function(callback) { + var self = this; + self.pending_clean.push(callback || NOOP); + setImmediate(next_operation, self, 13); + return self; +}; + +Table.prototype.clear = function(callback) { + var self = this; + self.pending_clear.push(callback || NOOP); + setImmediate(next_operation, self, 12); + return self; +}; + +Table.prototype.next = function(type) { + + if (!this.ready || (type && NEXTWAIT[this.step])) + return; + + if (!this.$writting && !this.$reading) { + + if (this.step !== 12 && this.pending_clear.length) { + this.$clear(); + return; + } + + if (this.step !== 13 && this.pending_clean.length) { + this.$clean(); + return; + } + + if (this.step !== 7 && !this.pending_reindex && this.pending_drops) { + this.$drop(); + return; + } + + if (this.step !== 14 && this.pending_locks.length) { + this.$lock(); + return; + } + } + + if (!this.$writting) { + + if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) { + this.$append(); + return; + } + + if (this.step !== 2 && !this.$writting && this.pending_update.length) { + this.$update(); + return; + } + + if (this.step !== 3 && !this.$writting && this.pending_remove.length) { + this.$remove(); + return; + } + } + + if (!this.$reading) { + + if (this.step !== 4 && this.pending_reader.length) { + this.$reader(); + return; + } + + if (this.step !== 10 && this.pending_streamer.length) { + this.$streamer(); + return; + } + } + + if (this.step !== type) { + this.step = 0; + setImmediate(next_operation, this, 0); + } +}; + +Table.prototype.$append = function() { + var self = this; + self.step = 1; + + if (!self.pending_append.length) { + self.next(0); + return; + } + + self.$writting = true; + + self.pending_append.splice(0).limit(JSONBUFFER, function(items, next) { + + var data = ''; + + for (var i = 0, length = items.length; i < length; i++) + data += self.stringify(items[i].doc) + NEWLINE; + + Fs.appendFile(self.filename, data, function(err) { + err && F.error(err, 'Table insert: ' + self.name); + for (var i = 0, length = items.length; i < length; i++) { + items[i].builder.$options.log && items[i].builder.log(); + var callback = items[i].builder.$callback; + callback && callback(err, 1); + } + next(); + }); + + }, () => setImmediate(next_append, self)); +}; + +Table.prototype.$reader = function() { + + var self = this; + + self.step = 4; + + if (!self.pending_reader.length) { + self.next(0); + return self; + } + + self.$reading = true; + + var filter = self.pending_reader.splice(0); + var length = filter.length; + var first = true; + var indexer = 0; + var keys = {}; + var keyscount = 0; + + for (var i = 0; i < length; i++) { + var fil = filter[i]; + if (!fil.builder.$options.first || fil.builder.$options.sort) + first = false; + + if (fil.builder.$keys == null) + keys = null; + else { + var tmp = Object.keys(fil.builder.$keys); + for (var j = 0; j < tmp.length; j++) { + keyscount++; + keys[tmp[j]] = 1; + } + } + + fil.scalarcount = 0; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + } + + if (first && length > 1) + first = false; + + var fs = new NoSQLStream(self.filename); + var data = {}; + + fs.divider = '\n'; + data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + + fs.ondocuments = function() { + + var lines = fs.docs.split(fs.divider); + var val; + + for (var j = indexer ? 0 : 1; j < lines.length; j++) { + + data.line = lines[j].split('|'); + data.index = indexer++; + + indexer++; + + var obj = self.parseData(data); + + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + item.filter.index = indexer; + var output = item.compare(obj, item.filter, indexer); + if (!output) + continue; + + item.count++; + + if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) + continue; + + item.counter++; + + if (item.type) + continue; + + switch (builder.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[builder.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[builder.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'max': + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'avg': + val = output[builder.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (builder.$inlinesort) + nosqlinlinesorter(item, builder, output); + else if (item.response) + item.response.push(output); + else + item.response = [output]; + break; + } + + if (first) + return false; + } + } + }; + + fs.$callback = function() { + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + var output; + + if (builder.$options.scalar || !builder.$options.sort) { + + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || []; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + continue; + } + + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else + output = item.response || []; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + builder.done(); + } + + fs = null; + self.$reading = false; + self.next(0); + }; + + fs.openread(); + return self; +}; + +Table.prototype.$update = function() { + + var self = this; + self.step = 2; + + if (!self.pending_update.length) { + self.next(0); + return self; + } + + self.$writting = true; + + var filter = self.pending_update.splice(0); + var length = filter.length; + var backup = false; + var filters = 0; + var change = false; + + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + if (fil.backup || fil.builder.$options.backup) + backup = true; + } + + var indexer = 0; + var fs = new NoSQLStream(self.filename); + fs.divider = '\n'; + + var data = {}; + data.keys = self.$keys; + + fs.ondocuments = function() { + + var lines = fs.docs.split(fs.divider); + var val; + + for (var a = indexer ? 0 : 1; a < lines.length; a++) { + + data.line = lines[a].split('|'); + data.index = indexer++; + + var doc = self.parseData(data); + + var is = false; + var rec = fs.docsbuffer[a]; + + for (var i = 0; i < length; i++) { + + var item = filter[i]; + if (item.skip) + continue; + + item.filter.index = indexer; + + var output = item.compare(doc, item.filter, indexer); + if (output) { + + if (item.filter.options.first) { + item.skip = true; + filters++; + } + + if (item.keys) { + for (var j = 0; j < item.keys.length; j++) { + var key = item.keys[j]; + var val = item.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + output[key] = val(output[key], output); + else + output[key] = val; + } + } + } else + output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; + + var e = item.keys ? 'modify' : 'update'; + self.$events[e] && self.emit(e, output); + item.count++; + doc = output; + is = true; + } + } + + if (is) { + + if (backup) { + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.backup && item.backup.write(rec.doc + NEWLINE); + item.builder.$options.backup && item.builder.$backupdoc(rec.doc); + } + } + + var upd = self.stringify(doc); + + if (upd === rec.doc) + continue; + + var was = true; + + if (!change) + change = true; + + if (rec.doc.length === upd.length) { + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + was = false; + } + } + + if (was) { + var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; + fs.write(tmp, rec.position); + fs.write2(upd + NEWLINE); + } + } + + if (filters === length) + return false; + } + }; + + fs.$callback = function() { + + if (self.indexes) + F.databasescleaner[self.$name] = (F.databasescleaner[self.$name] || 0) + 1; + + for (var i = 0; i < length; i++) { + var item = filter[i]; + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert); + var tmp = self.insert(item.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + } + } + + fs = null; + self.$writting = false; + self.next(0); + + change && self.$events.change && self.emit('change', 'update'); + }; + + fs.openupdate(); + return self; +}; + +Table.prototype.$remove = function() { + + var self = this; + self.step = 3; + + if (!self.pending_remove.length) { + self.next(0); + return; + } + + self.$writting = true; + + var fs = new NoSQLStream(self.filename); + var filter = self.pending_remove.splice(0); + var length = filter.length; + var change = false; + var indexer = 0; + var backup = false; + + fs.divider = '\n'; + + for (var i = 0; i < length; i++) { + var fil = filter[i]; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + if (fil.backup || fil.builder.$options.backup) + backup = true; + } + + var data = {}; + data.keys = self.$keys; + + fs.ondocuments = function() { + + var lines = fs.docs.split(fs.divider); + + for (var a = indexer ? 0 : 1; a < lines.length; a++) { + + data.line = lines[a].split('|'); + data.index = indexer++; + + indexer++; + var removed = false; + var doc = self.parseData(data); + var rec = fs.docsbuffer[a]; + + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.filter.index = indexer; + var output = item.compare(doc, item.filter, indexer); + if (output) { + removed = true; + doc = output; + break; + } + } + + if (removed) { + + if (backup) { + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.backup && item.backup.write(rec.doc + NEWLINE); + item.builder.$options.backup && item.builder.$backupdoc(rec.doc); + } + } + + if (!change) + change = true; + + item.count++; + self.$events.remove && self.emit('remove', doc); + fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); + } + } + }; + + fs.$callback = function() { + + if (self.indexes) + F.databasescleaner[self.$name] = (F.databasescleaner[self.$name] || 0) + 1; + + for (var i = 0; i < length; i++) { + var item = filter[i]; + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + } + + fs = null; + self.$writting = false; + self.next(0); + + if (change) { + self.views && setImmediate(views_refresh, self); + self.$events.change && self.emit('change', 'remove'); + } + }; + + fs.openupdate(); +}; + +Table.prototype.$clean = function() { + + var self = this; + self.step = 13; + + if (!self.pending_clean.length) { + self.next(0); + return; + } + + var filter = self.pending_clean.splice(0); + var length = filter.length; + var now = Date.now(); + + F.databasescleaner[self.name] = undefined; + F.config['nosql-logger'] && PRINTLN('NoSQL Table "{0}" cleaning (beg)'.format(self.name)); + + var fs = new NoSQLStream(self.filename); + var writer = Fs.createWriteStream(self.filename + '-tmp'); + + fs.divider = NEWLINE; + + fs.ondocuments = function() { + writer.write(fs.docs + NEWLINE); + }; + + fs.$callback = function() { + writer.end(); + }; + + writer.on('finish', function() { + Fs.rename(self.filename + '-tmp', self.filename, function() { + F.config['nosql-logger'] && PRINTLN('NoSQL Table "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); + for (var i = 0; i < length; i++) + filter[i](); + self.$events.clean && self.emit('clean'); + self.next(0); + fs = null; + }); + }); + + fs.openread(); +}; + +Table.prototype.$clear = function() { + + var self = this; + self.step = 12; + + if (!self.pending_clear.length) { + self.next(0); + return; + } + + var filter = self.pending_clear.splice(0); + Fs.unlink(self.filename, function() { + for (var i = 0; i < filter.length; i++) + filter[i](); + + Fs.appendFile(self.filename, self.stringifySchema() + NEWLINE, function() { + self.$events.change && self.emit('change', 'clear'); + self.next(0); + }); + }); +}; + +Table.prototype.$streamer = function() { + + var self = this; + self.step = 10; + + if (!self.pending_streamer.length) { + self.next(0); + return self; + } + + self.$reading = true; + + var filter = self.pending_streamer.splice(0); + var length = filter.length; + var count = 0; + var fs = new NoSQLStream(self.filename); + var data = {}; + + data.keys = self.$keys; + fs.divider = '\n'; + + fs.ondocuments = function() { + var lines = fs.docs.split(fs.divider); + for (var a = count ? 0 : 1; a < lines.length; a++) { + data.line = lines[a].split('|'); + data.index = count++; + var doc = self.parseData(data); + for (var i = 0; i < length; i++) + filter[i].fn(doc, filter[i].repository, count); + } + }; + + fs.$callback = function() { + for (var i = 0; i < length; i++) + filter[i].callback && filter[i].callback(null, filter[i].repository, count); + self.$reading = false; + self.next(0); + fs = null; + }; + + fs.openread(); + return self; +}; + +Table.prototype.parseSchema = function() { + var self = this; + var arr = arguments[0] instanceof Array ? arguments[0] : arguments; + + self.$schema = {}; + self.$keys = []; + + for (var i = 0; i < arr.length; i++) { + var arg = arr[i].split(':'); + var type = 0; + switch (arg[1].toLowerCase().trim()) { + case 'string': + type = 1; + break; + case 'number': + type = 2; + break; + case 'boolean': + type = 3; + break; + case 'date': + type = 4; + break; + case 'object': + type = 5; + break; + } + var name = arg[0].trim(); + self.$schema[name] = { type: type, pos: i }; + self.$keys.push(name); + } + + return self; +}; + +Table.prototype.stringifySchema = function() { + + var self = this; + var data = []; + + for (var i = 0; i < self.$keys.length; i++) { + + var key = self.$keys[i]; + var meta = self.$schema[key]; + var type = 'string'; + + switch (meta.type) { + case 2: + type = 'number'; + break; + case 3: + type = 'boolean'; + break; + case 4: + type = 'date'; + break; + case 5: + type = 'object'; + break; + } + + data.push(key + ':' + type); + } + + return data.join('|'); +}; + +Table.prototype.parseData = function(data) { + + var self = this; + var obj = {}; + var val; + + for (var i = 0; i < data.keys.length; i++) { + var key = data.keys[i]; + var meta = self.$schema[key]; + + if (meta == null) + continue; + + var pos = meta.pos + 1; + + switch (meta.type) { + case 1: // String + obj[key] = data.line[pos]; + break; + case 2: // Number + obj[key] = +data.line[pos]; + break; + case 3: // Boolean + val = data.line[pos]; + obj[key] = BOOLEAN[val]; + break; + case 4: // Date + val = data.line[pos]; + obj[key] = val ? new Date(val) : null; + break; + case 5: // Object + val = data.line[pos]; + obj[key] = val ? val.parseJSON(true) : null; + break; + } + } + return obj; +}; + +Table.prototype.stringify = function(doc) { + + var self = this; + var output = '+'; + + for (var i = 0; i < self.$keys.length; i++) { + var key = self.$keys[i]; + var meta = self.$schema[key]; + var val = doc[key]; + + switch (meta.type) { + case 1: // String + val = val ? val : ''; + break; + case 2: // Number + val = (val || 0); + break; + case 3: // Boolean + val = (val == true ? '1' : '0'); + break; + case 4: // Date + val = val ? val.toISOString() : ''; + break; + case 5: // Object + val = val ? JSON.stringify(val) : ''; + break; + } + + output += '|' + val; + } + + return output; +}; + +Table.prototype.free = function(force) { + var self = this; + if (!force && !self.$free) + return self; + self.removeAllListeners(true); + delete F.databases['$' + self.name]; + return self; +}; + // ====================================================== // Helper functions // ====================================================== diff --git a/nosqlstream.js b/nosqlstream.js index 620a72fc1..5d31a752f 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -43,6 +43,7 @@ function NoSQLStream(filename) { this.cache = [null, null]; this.buffer = null; this.divider = ','; + this.remchar = '-'; // this.canceled = false; // this.docs = ''; // this.docscount = 0; @@ -82,7 +83,7 @@ NoSQLStream.prototype.readhelpers = function() { while (index !== -1) { var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] === '-') { + if (tmp[0] === self.remchar) { self.buffer = self.buffer.slice(index + 1); index = self.buffer.indexOf(NEWLINEBUFFER); if (index === -1) @@ -146,7 +147,7 @@ NoSQLStream.prototype.readhelpers = function() { while (index !== -1) { var tmp = self.buffer.toString('utf8', index); - if (tmp[1] === '-') { + if (tmp[1] === self.remchar) { self.buffer = self.buffer.slice(0, index); index = self.buffer.lastIndexOf(NEWLINEBUFFER); if (index === -1) @@ -320,8 +321,8 @@ NoSQLStream.prototype.writehelpers = function() { while (index !== -1) { var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] !== '-') { - self.docs += (self.docs ? ',' : '') + tmp; + if (tmp[0] !== self.remchar) { + self.docs += (self.docs ? self.divider : '') + tmp; self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); self.docscount++; if (self.docsbuffer.length >= BUFFERDOCS) { diff --git a/nosqlworker.js b/nosqlworker.js index 632c6ee1b..d09f4ee5c 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -67,7 +67,8 @@ process.on('message', function(msg) { return; } - var db = NOSQL(msg.name); + var db = msg.t ? TABLE(msg.name) : NOSQL(msg.name); + switch (msg.TYPE) { case 'find': db.find(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, count, repository) { From 5c4fe349b97681491969e46fd5b1c852baec184e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 13:08:49 +0200 Subject: [PATCH 0412/1669] Removed NoSQL indexes. --- changes.txt | 2 - index.js | 7 +- nosql.js | 714 +++---------------------------------------------- nosqlworker.js | 43 --- 4 files changed, 36 insertions(+), 730 deletions(-) diff --git a/changes.txt b/changes.txt index c907c3586..931dafa90 100755 --- a/changes.txt +++ b/changes.txt @@ -5,7 +5,6 @@ - added: (IMPORTANT) Total.js components support nested public files encoded in base64 - added: (IMPORTANT) NoSQL worker - added: (IMPORTANT) NoSQL embedded storage for smaller big data / IoT -- added: (IMPORTANT) NoSQL indexes - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` @@ -55,7 +54,6 @@ - added: config `security.txt` for auto-generating security.txt content (more in docs) - added: config `default-proxy` for default web proxy server - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` -- added: `NOSQLINDEXES(name)` alias for `NOSQL(name).indexes` - added: `GUID()` a global alias for `U.GUID()` - added: `VIEW()` a global alias for `F.view()` - added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue diff --git a/index.js b/index.js index 114fb103b..c98c0f472 100755 --- a/index.js +++ b/index.js @@ -292,7 +292,6 @@ global.Mail = framework_mail; global.WTF = (message, name, uri) => F.problem(message, name, uri); global.NOBIN = global.NOSQLBINARY = (name) => F.nosql(name).binary; global.NOSQLSTORAGE = (name) => F.nosql(name).storage; -global.NOSQLINDEXES = (name) => F.nosql(name).indexes; global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); global.CONFIG = (name) => F.config[name]; @@ -312,6 +311,10 @@ global.ROUTING = (name) => F.routing(name); global.SCHEDULE = (date, each, fn, param) => F.schedule(date, each, fn, param); global.FINISHED = framework_internal.onFinished; global.DESTROY = framework_internal.destroyStream; +global.FILESTORAGE = function(name) { + var key = 'storage_' + name; + return F.databases[key] ? F.databases[key] : (F.databases[key] = new framework_nosql.DatabaseBinary({ name: name }, F.path.databases('fs-' + name + '/'), '.file')); +}; global.UID = function(type) { @@ -942,8 +945,8 @@ F.prototypes = function(fn) { proto.DatabaseBuilder = framework_nosql.DatabaseBuilder.prototype; proto.DatabaseBuilder2 = framework_nosql.DatabaseBuilder2.prototype; proto.DatabaseCounter = framework_nosql.DatabaseCounter.prototype; - proto.DatabaseIndexes = framework_nosql.DatabaseIndexes.prototype; proto.DatabaseStorage = framework_nosql.DatabaseStorage.prototype; + proto.DatabaseTable = framework_nosql.DatabaseTable.prototype; proto.ErrorBuilder = framework_builders.ErrorBuilder.prototype; proto.HttpFile = framework_internal.HttpFile.prototype; proto.HttpRequest = PROTOREQ; diff --git a/nosql.js b/nosql.js index 624d876fe..b401fe82d 100755 --- a/nosql.js +++ b/nosql.js @@ -51,7 +51,6 @@ const EXTENSION_MAPREDUCE = '.nosql-mapreduce'; const EXTENSION_BACKUP = '.nosql-backup'; const EXTENSION_META = '.meta'; const EXTENSION_COUNTER = '-counter2'; -const EXTENSION_INDEXES = '-indexes'; const BINARY_HEADER_LENGTH = 2000; const COUNTER_MMA = [0, 0]; const DIRECTORYLENGTH = 9; @@ -185,9 +184,6 @@ exports.worker = function() { case 'counter.clear': case 'storage.stats': case 'storage.clear': - case 'indexes.clear': - case 'indexes.reindex': - case 'indexes.get': var obj = FORKCALLBACKS[msg.id]; obj && obj.callback && obj.callback(msg.err, msg.response); break; @@ -492,35 +488,6 @@ exports.worker = function() { return this; }; - Indexes.prototype.create = function(name, properties, type) { - notify(this.db, 'indexes.create', name, properties, type); - return this; - }; - - Indexes.prototype.get = Indexes.prototype.read = function(name, value, callback) { - send(this.db, 'indexes.get', name, value).callback = callback; - return this; - }; - - Indexes.prototype.find = function(name, value) { - return send(this.db, 'indexes.find', name, value).builder = new DatabaseBuilder(this.db); - }; - - Indexes.prototype.clear = function(callback) { - send(this.db, 'indexes.clear').callback = callback; - return this; - }; - - Indexes.prototype.reindex = function(callback) { - send(this.db, 'indexes.reindex').callback = callback; - return this; - }; - - Indexes.prototype.noreindex = function() { - notify(this.db, 'indexes.noreindex'); - return this; - }; - Table.prototype.find = function(builder) { if (builder) builder.db = this; @@ -641,7 +608,6 @@ function Database(name, filename, readonly) { self.filenameLog = self.readonly || readonly === true ? '' : filename + EXTENSION_LOG; self.filenameBackup = self.readonly || readonly === true ? '' : filename + EXTENSION_BACKUP; self.filenameStorage = self.readonly || readonly === true ? '' : filename + '-storage/{0}' + EXTENSION; - self.filenameIndexes = self.readonly || readonly === true ? '' : filename + '-indexes/{0}' + EXTENSION; self.filenameMeta = filename + EXTENSION_META; self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + EXTENSION); self.inmemory = {}; @@ -669,7 +635,6 @@ function Database(name, filename, readonly) { self.pending_reindex = false; self.binary = self.readonly || readonly === true ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); self.storage = self.readonly || readonly === true ? null : new Storage(self, self.directory + '/' + self.name + '-storage/'); - self.indexes = self.readonly || readonly === true ? null : new Indexes(self, self.directory + '/' + self.name + '-indexes/'); self.counter = readonly === true ? null : new Counter(self); self.$timeoutmeta; self.$events = {}; @@ -742,7 +707,6 @@ exports.DatabaseBuilder = DatabaseBuilder; exports.DatabaseBuilder2 = DatabaseBuilder2; exports.DatabaseCounter = Counter; exports.DatabaseBinary = Binary; -exports.DatabaseIndexes = Indexes; exports.DatabaseStorage = Storage; exports.DatabaseTable = Table; @@ -812,14 +776,7 @@ function next_operation(self, type) { Database.prototype.ready = function(fn) { var self = this; - - if (!self.indexes) - fn.call(self); - else if (self.indexes.isreindex || self.indexes.reindexing) - setTimeout((self, fn) => self.ready(fn), 500, self, fn); - else - fn.call(self); - + fn.call(self); return self; }; @@ -901,15 +858,8 @@ Database.prototype.restore = function(filename, callback) { if (!err) { self.$meta(); self.binary.$refresh(); - if (self.indexes && self.indexes.indexes.length) { - self.indexes.reindex(function() { - self.refresh(); - self.storage && self.storage.refresh(); - }); - } else { - self.refresh(); - self.storage && self.storage.refresh(); - } + self.refresh(); + self.storage && self.storage.refresh(); } self.$events.change && self.emit('change', 'restore'); callback && callback(err, response); @@ -1042,7 +992,6 @@ Database.prototype.free = function(force) { self.counter.removeAllListeners(true); self.binary.removeAllListeners(true); self.removeAllListeners(true); - self.indexes = null; self.binary = null; self.counter = null; delete F.databases[self.name]; @@ -1422,9 +1371,6 @@ Database.prototype.$append = function() { var json = ''; for (var i = 0, length = items.length; i < length; i++) { json += items[i].doc + NEWLINE; - - if (self.indexes && self.indexes.indexes.length) - self.indexes.insert(items[i].raw); } Fs.appendFile(self.filename, json, function(err) { @@ -1514,7 +1460,6 @@ Database.prototype.$update = function() { var doc = docs[a]; var is = false; - var copy = self.indexes && self.indexes.indexes.length ? CLONE(doc) : null; var rec = fs.docsbuffer[a]; for (var i = 0; i < length; i++) { @@ -1557,9 +1502,6 @@ Database.prototype.$update = function() { if (is) { - if (self.indexes && self.indexes.indexes.length) - self.indexes.update(doc, copy); - if (backup) { for (var i = 0; i < length; i++) { var item = filter[i]; @@ -1598,10 +1540,6 @@ Database.prototype.$update = function() { }; fs.$callback = function() { - - if (self.indexes) - F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; - for (var i = 0; i < length; i++) { var item = filter[i]; if (item.insert && !item.count) { @@ -1693,9 +1631,6 @@ Database.prototype.$update_inmemory = function() { is = true; } } - - if (is && self.indexes && self.indexes.indexes.length) - self.indexes.update(doc); } self.$save('#'); @@ -2638,20 +2573,12 @@ Database.prototype.$remove = function() { item.count++; self.$events.remove && self.emit('remove', doc); - - if (self.indexes && self.indexes.indexes.length) - self.indexes.remove(doc); - fs.write('-' + rec.doc.substring(1) + NEWLINE, rec.position); } } }; fs.$callback = function() { - - if (self.indexes) - F.databasescleaner[self.name] = (F.databasescleaner[self.name] || 0) + 1; - for (var i = 0; i < length; i++) { var item = filter[i]; item.builder.$options.log && item.builder.log(); @@ -2683,20 +2610,11 @@ Database.prototype.$clear = function() { var filter = self.pending_clear.splice(0); Fs.unlink(self.filename, function() { - if (self.indexes.length) { - self.indexes.clear(function() { - for (var i = 0; i < filter.length; i++) - filter[i](); - self.views && setImmediate(views_refresh, self); - self.next(0); - }); - } else { - for (var i = 0; i < filter.length; i++) - filter[i](); - self.views && setImmediate(views_refresh, self); - self.$events.change && self.emit('change', 'clear'); - self.next(0); - } + for (var i = 0; i < filter.length; i++) + filter[i](); + self.views && setImmediate(views_refresh, self); + self.$events.change && self.emit('change', 'clear'); + self.next(0); }); }; @@ -4757,14 +4675,16 @@ Counter.prototype.clear = function(callback) { return self; }; -function Binary(db, directory) { - this.db = db; - this.directory = directory; - this.$events = {}; - this.metafile = directory + 'meta.json'; - this.meta = { $version: 1, updated: NOW }; - this.cachekey = 'nobin_' + db.name + '_'; - this.$refresh(); +function Binary(db, directory, ext) { + var t = this; + t.db = db; + t.ext = ext || EXTENSION_BINARY; + t.directory = directory; + t.$events = {}; + t.metafile = directory + 'meta.json'; + t.meta = { $version: 1, updated: NOW }; + t.cachekey = 'nobin_' + db.name + '_'; + t.$refresh(); } Binary.prototype.$refresh = function() { @@ -4950,7 +4870,7 @@ Binary.prototype.insert = function(name, buffer, custom, callback) { self.$save(); var filename = id.toString().padLeft(DIRECTORYLENGTH, '0'); - var stream = Fs.createWriteStream(Path.join(path, filename + EXTENSION_BINARY)); + var stream = Fs.createWriteStream(Path.join(path, filename + self.ext)); stream.write(header, 'binary'); stream.end(buffer); @@ -5007,9 +4927,9 @@ Binary.prototype.insertstream = function(id, name, type, stream, callback, custo var path = self.$directory(id); self.check(path); filename = id.toString().padLeft(DIRECTORYLENGTH, '0'); - filepath = Path.join(path, filename + EXTENSION_BINARY); + filepath = Path.join(path, filename + self.ext); } else - filepath = framework_utils.join(self.directory, self.db.name + '#' + id + EXTENSION_BINARY); + filepath = framework_utils.join(self.directory, self.db.name + '#' + id + self.ext); var writer = Fs.createWriteStream(filepath); writer.write(header, 'binary'); @@ -5126,10 +5046,10 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { var path = self.$directory(id); self.check(path); filename = id.toString().padLeft(DIRECTORYLENGTH, '0'); - filepath = Path.join(path, filename + EXTENSION_BINARY); + filepath = Path.join(path, filename + self.ext); } else { self.check(); - filepath = framework_utils.join(self.directory, self.db.name + '#' + id + EXTENSION_BINARY); + filepath = framework_utils.join(self.directory, self.db.name + '#' + id + self.ext); } switch (ext) { @@ -5197,9 +5117,9 @@ Binary.prototype.read = function(id, callback, count) { var filename; if (isnew) { - filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + self.ext); } else - filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); + filename = framework_utils.join(self.directory, id + self.ext); var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); @@ -5239,9 +5159,9 @@ Binary.prototype.readbase64 = function(id, callback, count) { var filename; if (isnew) { - filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + self.ext); } else - filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); + filename = framework_utils.join(self.directory, id + self.ext); var stream = Fs.createReadStream(filename, BINARYREADMETA); stream.on('error', err => callback(err)); @@ -5281,9 +5201,9 @@ Binary.prototype.remove = function(id, callback) { if (isnew) { var path = self.$directory(id); - filename = Path.join(path, id.toString().padLeft(DIRECTORYLENGTH, '0') + EXTENSION_BINARY); + filename = Path.join(path, id.toString().padLeft(DIRECTORYLENGTH, '0') + self.ext); } else - filename = framework_utils.join(self.directory, id + EXTENSION_BINARY); + filename = framework_utils.join(self.directory, id + self.ext); Fs.unlink(filename, function(err) { @@ -5376,7 +5296,7 @@ Binary.prototype.browse = function(directory, callback) { var target = framework_utils.join(self.directory, directory); var output = []; - var le = EXTENSION_BINARY.length; + var le = self.ext.length; response.wait(function(item, next) { Fs.stat(target + '/' + item, function(err, stat) { @@ -5429,7 +5349,7 @@ Binary.prototype.all = function(callback) { var target = framework_utils.join(self.directory); var output = []; - var le = EXTENSION_BINARY.length; + var le = self.ext.length; pending.wait(function(item, next) { Fs.stat(target + '/' + item, function(err, stat) { @@ -5471,572 +5391,6 @@ Binary.prototype.all = function(callback) { return self; }; -function Indexes(db, directory) { - this.db = db; - this.directory = directory; - this.indexes = []; - this.changes = []; - this.flushing = false; - this.instances = {}; - this.reindexing = false; - this.isreindex = false; - this.meta = { $version: 1 }; - try { - this.meta = Fs.readFileSync(this.db.filename + EXTENSION_INDEXES).toString('utf8').parseJSON(true) || {}; - } catch (e) {} -} - -Indexes.prototype.create = function(name, properties, type) { - - var self = this; - - if (type == null && typeof(properties) === 'string') { - type = properties; - properties = null; - } - - var prop = properties ? properties : [name]; - var key = prop.join(',') + (type ? ('=' + type) : ''); - - !self.indexes.findItem('name', name) && self.indexes.push({ name: name, properties: prop, type: type }); - - var meta = self.meta[name]; - var reindex = false; - - if (meta) { - if (meta.key !== key) { - reindex = true; - meta.key = key; - } - } else { - self.meta[name] = { key: key, documents: 0, changes: 0, cleaned: NOW.getTime() }; - reindex = true; - } - - if (!self.isreindex && reindex) - self.isreindex = reindex; - - reindex && setTimeout2(self.db.name + '_reindex', () => self.reindex(), 1000); - return self; -}; - -Indexes.prototype.noreindex = function() { - var self = this; - self.isreindex = false; - clearTimeout2(self.db.name + '_reindex'); - return self; -}; - -Indexes.prototype.$meta = function() { - var self = this; - self.$metachanged = false; - Fs.writeFile(self.db.filename + EXTENSION_INDEXES, JSON.stringify(this.meta), NOOP); - return this; -}; - -Indexes.prototype.$index = function(index, value) { - - var key = ''; - var number = false; - var num = 2; - - for (var i = 0; i < value.length; i++) { - var val = value[i]; - - switch (typeof(val)) { - case 'number': - if (index.type === 'first') - val = val.toString()[0]; - else if (index.type === 'reverse') { - val = val.toString(); - val = val.substring(val.length - num).padLeft(num, '0'); - } else - val = val.toString().substring(0, num).padLeft(num, '0'); - number = true; - break; - case 'boolean': - val = val ? '1' : '0'; - break; - case 'string': - if (REGNUMBER.test(val)) { - val = +val; - if (index.type === 'first') - val = val.toString()[0]; - else if (index.type === 'reverse') { - val = val.toString(); - val = val.substring(val.length - num).padLeft(num, '0'); - } else - val = val.toString().substring(0, num).padLeft(num, '0'); - number = true; - } else { - if (val.isUID()) { - val = val.substring(0, 2) + val.substring(2, 4); - number = true; - } else { - val = val.toLowerCase().removeDiacritics().match(REGINDEXCHAR); - if (val) { - val = val.toString(); - switch (index.type) { - case 'first': - val = val[0]; - break; - case 'reverse': - val = val.substring(val.length - 2); - break; - case 'soundex': - val = val.soundex(); - break; - default: - val = val.substring(0, 2); - } - } - } - } - break; - case 'object': - val = val instanceof Date ? (val.getFullYear().toString().substring(2) + val.format('MM')) : ''; - break; - } - - if (val == null) - continue; - - if (val) - key += val; - } - - return key ? (index.name + '_' + (number && value.length === 1 ? key : key)) : null; -}; - -Indexes.prototype.get = Indexes.prototype.read = function(name, value, callback) { - - var self = this; - var index = self.indexes.findItem('name', name); - - if (!index) { - callback(new Error('Index not found.')); - return self; - } - - if (!(value instanceof Array)) - value = [value]; - - var key = self.$index(index, value); - if (!key) { - callback(new Error('Bad value for generating index.')); - return self; - } - - if (self.changes.length) { - var change = self.findchanges(index, key, value); - if (change) { - callback(null, CLONE(change.doc)); - return self; - } - } - - if (self.instances[key]) { - self.instances[key].PENDING++; - } else { - self.instances[key] = new Database(key, self.directory + key, true); - self.instances[key].PENDING = 0; - } - - var builder = self.instances[key].one(); - - for (var i = 0; i < index.properties.length; i++) - builder.where(index.properties[i], value[i]); - - builder.callback(function(err, response) { - - if (self.instances[key].PENDING) - self.instances[key].PENDING--; - else - delete self.instances[key]; - - callback(err, response); - }); - - return self; -}; - -Indexes.prototype.find = function(name, value) { - - var self = this; - var index = self.indexes.findItem('name', name); - - if (!index) - throw new Error('Index not found.'); - - if (!(value instanceof Array)) - value = [value]; - - var key = self.$index(index, value); - if (!key) - throw new Error('Bad value for generating index.'); - - if (self.instances[key]) { - self.instances[key].PENDING++; - } else { - self.instances[key] = new Database(key, self.directory + key, true); - self.instances[key].PENDING = 0; - } - - var builder = self.instances[key].find(); - - builder.$done = function() { - if (self.instances[key].PENDING) - self.instances[key].PENDING--; - else - delete self.instances[key]; - }; - - return builder; -}; - -Indexes.prototype.clear = function(callback) { - var self = this; - Fs.readdir(self.directory, function(err, files) { - - if (err) { - callback(); - return; - } - - files.wait(function(item, next) { - Fs.unlink(Path.join(self.directory, item), next); - }, callback); - }); - return self; -}; - -Indexes.prototype.reindex = function(callback) { - var self = this; - - clearTimeout2(self.db.name + '_reindex'); - - if (self.reindexing) { - callback && callback(new Error('Re-indexing is running.')); - return self; - } - - if (self.db.step === 1 || self.db.step === 2 || self.db.step === 3 || self.db.step === 7) { - // We need to wait - setTimeout(function(self, callback) { - self.reindex(callback); - }, 500, self, callback); - return self; - } - - self.db.pending_reindex = true; - self.reindexing = true; - var now = Date.now(); - - F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" re-indexing (beg)'.format(self.db.name)); - - var keys = Object.keys(self.meta); - var ticks = NOW.getTime(); - - for (var i = 0; i < self.indexes.length; i++) { - var item = self.meta[self.indexes[i].name]; - item.documents = 0; - item.cleaned = ticks; - } - - // Clears non-exist indexes - for (var i = 0; i < keys.length; i++) { - if (self.indexes.findItem('name', keys[i]) == null) - delete self.meta[keys[i]]; - } - - self.clear(function() { - self.db.$events['indexing-begin'] && self.db.emit('indexing-begin'); - var chunker = U.chunker(self.db.name + '_reindex', 10000); - self.db.stream(function(doc) { - chunker.write(doc); - }, function(err, repository, count) { - chunker.end(); - chunker.each(function(docs, next) { - self.db.$events.indexing && self.db.emit('indexing', chunker.percentage, chunker.count); - for (var i = 0; i < docs.length; i++) - self.insert(docs[i], true); - self.$reindexingnext = next; - }, function() { - self.isreindex = false; - self.$meta(); - self.db.$events['indexing-end'] && self.db.emit('indexing-end'); - self.reindexing = false; - self.db.pending_reindex = false; - self.db.next(0); - callback && callback(null, count); - F.config['nosql-logger'] && PRINTLN('NoSQL embedded "{0}" re-indexing (end, {1}s)'.format(self.db.name, (((Date.now() - now) / 1000) >> 0))); - }); - }); - }); - - return self; -}; - -Indexes.prototype.check = function() { - - var self = this; - if (self.exists) - return self; - - self.exists = true; - - try { - Fs.mkdirSync(self.directory); - } catch (err) {} - - return self; -}; - -Indexes.prototype.makeindex = function(index, doc) { - - var arr = []; - - for (var i = 0; i < index.properties.length; i++) { - var val = doc[index.properties[i]]; - arr.push(val); - } - - return arr; -}; - -Indexes.prototype.findchanges = function(index, key, values) { - var self = this; - for (var i = 0, length = self.changes.length; i < length; i++) { - var item = self.changes[i]; - if (item.key !== key || item.value.length !== values.length) - continue; - - var is = true; - - for (var j = 0; j < values.length; j++) { - if (values[j] !== item.value[j]) { - is = false; - break; - } - } - - if (is) - return item; - } -}; - -Indexes.prototype.insert = function(doc, reindex) { - - var self = this; - - for (var i = 0; i < self.indexes.length; i++) { - - var index = self.indexes[i]; - var values = self.makeindex(index, doc); - - if (values.length) { - - var key = self.$index(index, values); - - if (!key) - continue; - - if (!reindex) { - var item = self.findchanges(index, key, values); - if (item) { - item.doc = doc; - return self; - } - } - - self.changes.push({ insert: true, key: key, doc: doc, name: index.name, value: values }); - } - } - - self.changes.length && self.flush(); - return self; -}; - -Indexes.prototype.update = function(doc, old) { - var self = this; - - for (var i = 0; i < self.indexes.length; i++) { - - var index = self.indexes[i]; - var values = self.makeindex(index, doc); - - if (values.length) { - var key = self.$index(index, values); - if (!key) - continue; - - var oldvalues = self.makeindex(index, old); - var oldkey = self.$index(index, oldvalues); - - // Because of cleaning - self.meta[index.name].changes++; - self.$metachanged = true; - - var item = self.findchanges(index, key, values); - if (item) - item.doc = doc; - else - self.changes.push({ update: true, key: key, doc: doc, name: index.name, properties: index.properties, value: values }); - - if (oldkey !== key && oldkey) - self.changes.push({ remove: true, key: oldkey, name: index.name, properties: index.properties, value: oldvalues }); - - } - } - - self.changes.length && self.flush(); - return self; -}; - -Indexes.prototype.remove = function(doc) { - var self = this; - - for (var i = 0; i < self.indexes.length; i++) { - - var index = self.indexes[i]; - var values = self.makeindex(index, doc); - - if (values.length) { - var key = self.$index(index, values); - if (!key) - continue; - - // Because of cleaning - self.meta[index.name].changes++; - self.$metachanged = true; - - var item = self.findchanges(index, key, values); - if (!item) - self.changes.push({ remove: true, key: key, name: index.name, properties: index.properties, value: values }); - } - } - - self.changes.length && self.flush(); - return self; -}; - -Indexes.prototype.flush = function() { - - var self = this; - - if (self.flushing) - return self; - - self.check(); - self.flushing = true; - - var count = 0; - - var fn = function() { - - if (count > 0) - return; - - self.flushing = false; - self.$metachanged && self.$meta(); - self.$free(); - - if (!self.changes.length) { - if (self.$reindexingnext) { - self.$reindexingnext(); - self.$reindexingnext = null; - } - } - - self.changes.length && setImmediate(() => self.flush()); - }; - - var arr = self.changes.splice(0, 50); - var ticks = NOW.getTime() - CLEANDBTICKS; - - for (var i = 0; i < arr.length; i++) { - - var item = arr[i]; - - if (self.instances[item.key]) { - self.instances[item.key].PENDING++; - } else { - self.instances[item.key] = new Database(item.key, self.directory + item.key, true); - self.instances[item.key].PENDING = 0; - self.instances[item.key].CLEANDB = self.meta[item.name].changes > 0 && self.meta[item.name].cleaned < ticks ? item.name : null; - } - - if (item.update) { - count++; - var builder = self.instances[item.key].update(item.doc, item.doc).callback(function() { - if (self.instances[item.key].PENDING) - self.instances[item.key].PENDING--; - count--; - fn(); - }); - - for (var j = 0; j < item.properties.length; j++) - builder.where(item.properties[j], item.value[j]); - - } else if (item.insert) { - - count++; - - if (self.meta[item.name]) - self.meta[item.name].documents++; - - self.instances[item.key].insert(item.doc).callback(function() { - if (self.instances[item.key].PENDING) - self.instances[item.key].PENDING--; - count--; - fn(); - }); - - } else { - - count++; - - if (self.meta[item.name]) - self.meta[item.name].documents--; - - var builder = self.instances[item.key].remove().callback(function() { - if (self.instances[item.key].PENDING) - self.instances[item.key].PENDING--; - count--; - fn(); - }); - - for (var j = 0; j < item.properties.length; j++) - builder.where(item.properties[j], item.value[j]); - } - } -}; - -Indexes.prototype.$free = function() { - var self = this; - var keys = Object.keys(self.instances); - for (var i = 0; i < keys.length; i++) { - var db = self.instances[keys[i]]; - if (!db || !db.PENDING) { - // Cleans removed/changed documents - if (db && db.CLEANDB) { - db.PENDING++; - db.clean(function() { - db.PENDING--; - var a = self.meta[db.CLEANDB]; - a.cleaned = NOW.getTime(); - a.changes = 0; - db.CLEANDB = null; - self.$meta(); - delete self.instances[keys[i]]; - }); - } else - delete self.instances[keys[i]]; - } - } - return self; -}; - function Storage(db, directory) { this.db = db; this.directory = directory; @@ -6960,9 +6314,6 @@ Table.prototype.$update = function() { fs.$callback = function() { - if (self.indexes) - F.databasescleaner[self.$name] = (F.databasescleaner[self.$name] || 0) + 1; - for (var i = 0; i < length; i++) { var item = filter[i]; if (item.insert && !item.count) { @@ -7066,9 +6417,6 @@ Table.prototype.$remove = function() { fs.$callback = function() { - if (self.indexes) - F.databasescleaner[self.$name] = (F.databasescleaner[self.$name] || 0) + 1; - for (var i = 0; i < length; i++) { var item = filter[i]; item.builder.$options.log && item.builder.log(); diff --git a/nosqlworker.js b/nosqlworker.js index d09f4ee5c..8b6c4d59d 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -40,9 +40,6 @@ const RESCOUNTERCLEAR = { TYPE: 'counter.clear' }; const RESTORAGESCAN = { TYPE: 'storage.scan' }; const RESTORAGESTATS = { TYPE: 'storage.stats' }; const RESSTORAGECLEAR = { TYPE: 'storage.clear' }; -const RESINDEXESGET = { TYPE: 'indexes.get' }; -const RESINDEXESCLEAR = { TYPE: 'indexes.clear' }; -const RESINDEXESREINDEX = { TYPE: 'indexes.reindex' }; const RESSTREAM = { TYPE: 'stream' }; function killprocess() { @@ -233,46 +230,6 @@ process.on('message', function(msg) { process.send(RESSTORAGECLEAR); }); break; - case 'indexes.create': - db.indexes.create(msg.arg[0], msg.arg[1], msg.arg[2]); - break; - case 'indexes.get': - db.indexes.get(msg.arg[0], msg.arg[1], function(err, response) { - RESINDEXESGET.id = msg.id; - RESINDEXESGET.response = response; - RESINDEXESGET.err = err; - process.send(RESINDEXESGET); - }); - break; - case 'indexes.find': - db.indexes.find(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, count, repository) { - RESFIND.err = err; - RESFIND.response = response; - RESFIND.count = count; - RESFIND.repository = repository; - RESFIND.id = msg.id; - process.send(RESFIND); - }); - break; - case 'indexes.clear': - db.indexes.clear(function(err, response) { - RESINDEXESCLEAR.id = msg.id; - RESINDEXESCLEAR.response = response; - RESINDEXESCLEAR.err = err; - process.send(RESINDEXESCLEAR); - }); - break; - case 'indexes.reindex': - db.indexes.reindex(function(err, response) { - RESINDEXESREINDEX.id = msg.id; - RESINDEXESREINDEX.response = response; - RESINDEXESREINDEX.err = err; - process.send(RESINDEXESREINDEX); - }); - break; - case 'indexes.noreindex': - db.indexes.noreindex(); - break; case 'stream': db[msg.TYPE](eval('(' + msg.arg[0] + ')'), msg.arg[1], function(err, repository, count) { RESSTREAM.id = msg.id; From 9a9775f1f8a92406bfaa1e703754acbfffe7b6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 13:18:29 +0200 Subject: [PATCH 0413/1669] Improved DB. --- nosql.js | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/nosql.js b/nosql.js index b401fe82d..016a8b38a 100755 --- a/nosql.js +++ b/nosql.js @@ -581,6 +581,7 @@ function Table(name, filename) { t.$events = {}; t.step = 0; + t.stopped = false; t.ready = false; t.$free = true; t.$writting = false; @@ -591,7 +592,16 @@ function Table(name, filename) { t.ready = true; t.next(0); }).on('error', function() { - throw new Error('Table "{0}" doesn\'t contain schema'.format(name)); + t.stopped = true; + t.pending_reader.length && (t.pending_reader = []); + t.pending_update.length && (t.pending_update = []); + t.pending_append.length && (t.pending_append = []); + t.pending_reader.length && (t.pending_reader = []); + t.pending_remove.length && (t.pending_remove = []); + t.pending_streamer.length && (t.pending_streamer = []); + t.pending_clean.length && (t.pending_clean = []); + t.pending_clear.length && (t.pending_clear = []); + t.throwStopped(); }); } @@ -5760,7 +5770,7 @@ Table.prototype.insert = function(doc, unique) { var self = this; var builder; - self.readonly && self.throwReadonly(); + self.stopped && self.throwStopped(); if (unique) { @@ -5791,7 +5801,7 @@ Table.prototype.insert = function(doc, unique) { Table.prototype.update = function(doc, insert) { var self = this; - self.readonly && self.throwReadonly(); + self.stopped && self.throwStopped(); var builder = new DatabaseBuilder(self); self.pending_update.push({ builder: builder, doc: doc, count: 0, insert: insert === true ? doc : insert }); setImmediate(next_operation, self, 2); @@ -5800,7 +5810,7 @@ Table.prototype.update = function(doc, insert) { Table.prototype.modify = function(doc, insert) { var self = this; - self.readonly && self.throwReadonly(); + self.stopped && self.throwStopped(); var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; var keys = Object.keys(data); @@ -5815,7 +5825,7 @@ Table.prototype.modify = function(doc, insert) { Table.prototype.remove = function() { var self = this; - self.readonly && self.throwReadonly(); + self.stopped && self.throwStopped(); var builder = new DatabaseBuilder(self); self.pending_remove.push({ builder: builder, count: 0 }); setImmediate(next_operation, self, 3); @@ -5824,6 +5834,7 @@ Table.prototype.remove = function() { Table.prototype.find = function(builder) { var self = this; + self.stopped && self.throwStopped(); if (builder) builder.db = self; else @@ -5835,6 +5846,7 @@ Table.prototype.find = function(builder) { Table.prototype.stream = function(fn, repository, callback) { var self = this; + self.stopped && self.throwStopped(); if (typeof(repository) === 'function') { callback = repository; @@ -5850,12 +5862,17 @@ Table.prototype.throwReadonly = function() { throw new Error('Table "{0}" is readonly.'.format(this.name)); }; +Table.prototype.throwStopped = function() { + throw new Error('Table "{0}" doesn\'t contain schema'.format(this.name)); +}; + Table.prototype.scalar = function(type, field) { return this.find().scalar(type, field); }; Table.prototype.count = function() { var self = this; + self.stopped && self.throwStopped(); var builder = new DatabaseBuilder(self); self.pending_reader.push({ builder: builder, count: 0, type: 1 }); setImmediate(next_operation, self, 4); @@ -5864,6 +5881,7 @@ Table.prototype.count = function() { Table.prototype.one = function() { var self = this; + self.stopped && self.throwStopped(); var builder = new DatabaseBuilder(self); builder.first(); self.pending_reader.push({ builder: builder, count: 0 }); @@ -5873,6 +5891,7 @@ Table.prototype.one = function() { Table.prototype.top = function(max) { var self = this; + self.stopped && self.throwStopped(); var builder = new DatabaseBuilder(self); builder.take(max); self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); @@ -5882,6 +5901,7 @@ Table.prototype.top = function(max) { Table.prototype.clean = function(callback) { var self = this; + self.stopped && self.throwStopped(); self.pending_clean.push(callback || NOOP); setImmediate(next_operation, self, 13); return self; @@ -5889,6 +5909,7 @@ Table.prototype.clean = function(callback) { Table.prototype.clear = function(callback) { var self = this; + self.stopped && self.throwStopped(); self.pending_clear.push(callback || NOOP); setImmediate(next_operation, self, 12); return self; From a4004951819281f7dee5da8799de381ba5106d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 13:58:28 +0200 Subject: [PATCH 0414/1669] Added escaping values for TABLE db. --- nosql.js | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index 016a8b38a..478d3376d 100755 --- a/nosql.js +++ b/nosql.js @@ -61,13 +61,12 @@ const NEWLINE = '\n'; const REGBOOL = /":true/g; // for updates of boolean types const REGCHINA = /[\u3400-\u9FBF]/; const REGCLEAN = /^[\s]+|[\s]+$/g; -const REGINDEXCHAR = /[a-z]{1,2}/; -const REGNUMBER = /^\d+$/; +const REGTESCAPE = /\|/g; +const REGTUNESCAPE = /%7C/g; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' }; const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; -const CLEANDBTICKS = 86400000 * 2; // 48 hours const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { @@ -6304,7 +6303,6 @@ Table.prototype.$update = function() { } var upd = self.stringify(doc); - if (upd === rec.doc) continue; @@ -6638,6 +6636,7 @@ Table.prototype.parseData = function(data) { var self = this; var obj = {}; + var esc = false; var val; for (var i = 0; i < data.keys.length; i++) { @@ -6647,11 +6646,16 @@ Table.prototype.parseData = function(data) { if (meta == null) continue; + if (i === 0) + esc = data.line[0] === '*'; + var pos = meta.pos + 1; switch (meta.type) { case 1: // String obj[key] = data.line[pos]; + if (esc && obj[key]) + obj[key] = obj[key].replace(REGTUNESCAPE, '|'); break; case 2: // Number obj[key] = +data.line[pos]; @@ -6666,6 +6670,8 @@ Table.prototype.parseData = function(data) { break; case 5: // Object val = data.line[pos]; + if (esc && val) + val = val.replace(REGTUNESCAPE, '|'); obj[key] = val ? val.parseJSON(true) : null; break; } @@ -6676,7 +6682,8 @@ Table.prototype.parseData = function(data) { Table.prototype.stringify = function(doc) { var self = this; - var output = '+'; + var output = ''; + var esc = false; for (var i = 0; i < self.$keys.length; i++) { var key = self.$keys[i]; @@ -6701,10 +6708,18 @@ Table.prototype.stringify = function(doc) { break; } + if (!esc && (meta.type === 1 || meta.type === 5)) { + val += ''; + if (val.indexOf('|') !== -1) { + esc = true; + val = val.replace(REGTESCAPE, '%7C'); + } + } + output += '|' + val; } - return output; + return (esc ? '*' : '+') + output; }; Table.prototype.free = function(force) { From 2c961fe72c8992fdb0da355eef9110dcee3f2c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 16:56:21 +0200 Subject: [PATCH 0415/1669] Improved TABLE updates. --- nosql.js | 107 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/nosql.js b/nosql.js index 478d3376d..7c195e5da 100755 --- a/nosql.js +++ b/nosql.js @@ -2858,7 +2858,7 @@ function DatabaseBuilder(db) { this.$options = {}; this.$repository = {}; this.$counter = 0; - this.$keys = {}; + this.$keys = []; } DatabaseBuilder.prototype.promise = promise; @@ -3049,8 +3049,7 @@ DatabaseBuilder.prototype.join = function(field, name, view) { self.$join[key].a = a; self.$join[key].b = b; - if (self.$keys) - self.$keys[b] = 1; + self.$keys && self.$keys.push(b); return join; }; @@ -3173,9 +3172,7 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { if (self.$scope) code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys[name] = 1; - + self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key, operator)); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3213,9 +3210,7 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { if (self.$scope) code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys[name] = 1; - + self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3236,9 +3231,7 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { if (self.$scope) code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys[name] = 1; - + self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3259,8 +3252,7 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { if (self.$scope) code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys[name] = 1; + self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); @@ -3296,9 +3288,7 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam if (self.$scope) code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys[name] = 1; - + self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key)); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3310,9 +3300,7 @@ DatabaseBuilder.prototype.regexp = function(name, value) { if (self.$scope) code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys[name] = 1; - + self.$keys && self.$keys.push(name); self.$code.push(code.format(name, value.toString())); !self.$scope && self.$code.push('if(!$is)return;'); return self; @@ -3333,11 +3321,9 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { value = value.toLowerCase().split(' '); } - if (self.$keys) - self.$keys[name] = 1; + self.$keys && self.$keys.push(name); var count = 1; - if (weight) count = ((value.length / 100) * weight) >> 0; @@ -3457,10 +3443,10 @@ DatabaseBuilder.prototype.repository = function(key, value) { return this; }; -DatabaseBuilder.prototype.compile = function() { +DatabaseBuilder.prototype.compile = function(noTrimmer) { var self = this; var raw = self.$code.join(''); - var code = 'var repository=$F.repository,options=$F.options,arg=$F.arg,fn=$F.fn,$is=false,$tmp;var R=repository;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + 'if(options.fields){var $doc={};for(var $i=0;$i Date: Mon, 25 Jun 2018 20:48:17 +0300 Subject: [PATCH 0416/1669] Add old document to modify & update events. Adding data before modification/update to modify&update event. --- nosql.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 158c9fc9d..64496f8f7 100755 --- a/nosql.js +++ b/nosql.js @@ -1410,7 +1410,7 @@ Database.prototype.$update = function() { var output = item.compare(doc, item.filter, indexer); if (output) { - + var oldDocument = Object.assign({}, output); if (item.filter.options.first) { item.skip = true; filters++; @@ -1431,7 +1431,7 @@ Database.prototype.$update = function() { output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; var e = item.keys ? 'modify' : 'update'; - self.$events[e] && self.emit(e, output); + self.$events[e] && self.emit(e, output, oldDocument); item.count++; doc = output; is = true; @@ -1552,7 +1552,7 @@ Database.prototype.$update_inmemory = function() { item.filter.index = j; var output = item.compare(doc, item.filter, j); if (output) { - + var oldDocument = Object.assign({}, output); builder.$options.backup && builder.$backupdoc(doc); if (item.keys) { @@ -1569,7 +1569,7 @@ Database.prototype.$update_inmemory = function() { doc = typeof(item.doc) === 'function' ? item.doc(doc) : item.doc; var e = item.keys ? 'modify' : 'update'; - self.$events[e] && self.emit(e, doc); + self.$events[e] && self.emit(e, doc, oldDocument); item.count++; if (!change) change = true; From 95086c573b8e96166984fa95c7360b82fa01bd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 20:34:04 +0200 Subject: [PATCH 0417/1669] Fixed `SUCCESS()` with schema model. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index c98c0f472..f4fa51d64 100755 --- a/index.js +++ b/index.js @@ -542,7 +542,7 @@ global.SUCCESS = function(success, value) { success = true; SUCCESSHELPER.success = !!success; - SUCCESSHELPER.value = value == null ? undefined : value; + SUCCESSHELPER.value = value == null ? undefined : (framework_builders.isSchema(value) ? value.$clean() : value); SUCCESSHELPER.error = err ? err : undefined; return SUCCESSHELPER; }; From 2944220971410273c203717c3888dc6f4398d887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 20:36:10 +0200 Subject: [PATCH 0418/1669] Improved code. --- nosql.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/nosql.js b/nosql.js index bb7d756ad..a2209b856 100755 --- a/nosql.js +++ b/nosql.js @@ -1481,7 +1481,10 @@ Database.prototype.$update = function() { var output = item.compare(doc, item.filter, indexer); if (output) { - var oldDocument = Object.assign({}, output); + + var e = item.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(output) : 0; + if (item.filter.options.first) { item.skip = true; filters++; @@ -1501,8 +1504,7 @@ Database.prototype.$update = function() { } else output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - var e = item.keys ? 'modify' : 'update'; - self.$events[e] && self.emit(e, output, oldDocument); + self.$events[e] && self.emit(e, output, old); item.count++; doc = output; is = true; @@ -1607,7 +1609,6 @@ Database.prototype.$update_inmemory = function() { for (var a = 0, al = data.length; a < al; a++) { var doc = data[a]; - var is = false; for (var i = 0; i < length; i++) { @@ -1616,7 +1617,10 @@ Database.prototype.$update_inmemory = function() { item.filter.index = j; var output = item.compare(doc, item.filter, j); if (output) { - var oldDocument = Object.assign({}, output); + + var e = item.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(output) : 0; + builder.$options.backup && builder.$backupdoc(doc); if (item.keys) { @@ -1632,12 +1636,10 @@ Database.prototype.$update_inmemory = function() { } else doc = typeof(item.doc) === 'function' ? item.doc(doc) : item.doc; - var e = item.keys ? 'modify' : 'update'; - self.$events[e] && self.emit(e, doc, oldDocument); + self.$events[e] && self.emit(e, doc, old); item.count++; if (!change) change = true; - is = true; } } } @@ -6264,6 +6266,9 @@ Table.prototype.$update = function() { data.keys = tmp; } + var e = item.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(output) : 0; + if (item.keys) { for (var j = 0; j < item.keys.length; j++) { var key = item.keys[j]; @@ -6278,8 +6283,7 @@ Table.prototype.$update = function() { } else output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - var e = item.keys ? 'modify' : 'update'; - self.$events[e] && self.emit(e, output); + self.$events[e] && self.emit(e, output, old); item.count++; doc = output; is = true; From 318e537bb2388bb5d60d80e00266771c7a110b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 20:43:05 +0200 Subject: [PATCH 0419/1669] Added new contributor. --- package.json | 3 +++ readme.md | 1 + 2 files changed, 4 insertions(+) diff --git a/package.json b/package.json index 83d701c7d..93286ff07 100755 --- a/package.json +++ b/package.json @@ -95,6 +95,9 @@ }, { "name": "Pedro Costa", "email": "pedro@pmcdigital.pt" + }, { + "name": "Sarp Aykent", + "email": "shackhers@gmail.com" }], "version": "3.0.0-80", "homepage": "http://www.totaljs.com", diff --git a/readme.md b/readme.md index e655b5c5b..6987e6735 100755 --- a/readme.md +++ b/readme.md @@ -114,6 +114,7 @@ $ npm install -g total.js | [Tema Smirnov](https://github.com/TemaSM) | contributor | | ! [Jeroen van Hilst](https://github.com/frunjik) | contributor | | ! [Pedro Maia Costa](https://github.com/pnmcosta) | contributor | | +! [Sarp Aykent](https://github.com/HACKHERS) | contributor | | ## Useful modules From dccc79f0f93afde84bef0a62418d227112afcd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 20:43:42 +0200 Subject: [PATCH 0420/1669] Fixed markdown syntax. --- readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 6987e6735..8f620d01e 100755 --- a/readme.md +++ b/readme.md @@ -112,9 +112,9 @@ $ npm install -g total.js | [luoage](https://github.com/luoage) | contributor | | | [Mato Holly](https://github.com/boardshepherd) | contributor | | | [Tema Smirnov](https://github.com/TemaSM) | contributor | | -! [Jeroen van Hilst](https://github.com/frunjik) | contributor | | -! [Pedro Maia Costa](https://github.com/pnmcosta) | contributor | | -! [Sarp Aykent](https://github.com/HACKHERS) | contributor | | +| [Jeroen van Hilst](https://github.com/frunjik) | contributor | | +| [Pedro Maia Costa](https://github.com/pnmcosta) | contributor | | +| [Sarp Aykent](https://github.com/HACKHERS) | contributor | | ## Useful modules From ac3724d6f3e6108cf35e814f657dfce033feef7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 23:44:25 +0200 Subject: [PATCH 0421/1669] Improved NoSQL embedded. --- nosql.js | 444 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 234 insertions(+), 210 deletions(-) diff --git a/nosql.js b/nosql.js index a2209b856..b42628e93 100755 --- a/nosql.js +++ b/nosql.js @@ -252,12 +252,13 @@ exports.worker = function() { var DP = Database.prototype; var CP = Counter.prototype; + var SP = Storage.prototype; DP.once = DP.on = DP.emit = DP.removeListener = DP.removeAllListeners = CP.on = CP.once = CP.emit = CP.removeListener = CP.removeAllListeners = function() { PRINTLN('ERROR --> NoSQL events are not supported in fork mode.'); }; - Database.prototype.find = function(view, builder) { + DP.find = function(view, builder) { if (builder) builder.db = this; else @@ -265,7 +266,7 @@ exports.worker = function() { return send(this, 'find', view).builder = builder; }; - Database.prototype.find2 = function(builder) { + DP.find2 = function(builder) { if (builder) builder.db = this; else @@ -273,19 +274,19 @@ exports.worker = function() { return send(this, 'find2').builder = builder; }; - Database.prototype.top = function(max, view) { + DP.top = function(max, view) { var builder = new DatabaseBuilder(this); builder.take(max); return send(this, 'find', view).builder = builder; }; - Database.prototype.one = function(view) { + DP.one = function(view) { var builder = new DatabaseBuilder(this); builder.first(); return send(this, 'one', view).builder = builder; }; - Database.prototype.insert = function(doc, unique) { + DP.insert = function(doc, unique) { var self = this; var builder; @@ -316,67 +317,67 @@ exports.worker = function() { return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); }; - Database.prototype.count = function(view) { + DP.count = function(view) { var builder = new DatabaseBuilder(this); return send(this, 'count', view).builder = builder; }; - Database.prototype.view = function(name) { + DP.view = function(name) { var builder = new DatabaseBuilder(this); builder.id('$view_' + name); return send(this, 'view', name).builder = builder; }; - Database.prototype.update = function(doc, insert) { + DP.update = function(doc, insert) { return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); }; - Database.prototype.modify = function(doc, insert) { + DP.modify = function(doc, insert) { return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); }; - Database.prototype.restore = function(filename, callback) { + DP.restore = function(filename, callback) { var obj = send(this, 'restore', filename); obj.callback = callback; return this; }; - Database.prototype.backup = function(filename, callback) { + DP.backup = function(filename, callback) { var obj = send(this, 'backup', filename); obj.callback = callback; return this; }; - Database.prototype.refresh = function() { + DP.refresh = function() { notify(this, 'refresh'); return this; }; - Database.prototype.drop = function() { + DP.drop = function() { notify(this, 'drop'); return this; }; - Database.prototype.clear = function(callback) { + DP.clear = function(callback) { send(this, 'clear').callback = callback; return this; }; - Database.prototype.clean = function(callback) { + DP.clean = function(callback) { send(this, 'clean').callback = callback; return this; }; - Database.prototype.ready = function(callback) { + DP.ready = function(callback) { send(this, 'ready').callback = callback; return this; }; - Database.prototype.remove = function(filename) { + DP.remove = function(filename) { return send(this, 'remove', filename).builder = new DatabaseBuilder(this); }; - Database.prototype.stream = function(fn, repository, callback) { + DP.stream = function(fn, repository, callback) { if (typeof(repository) === 'function') { callback = repository; @@ -387,32 +388,32 @@ exports.worker = function() { return this; }; - Counter.prototype.min = function(id, count) { + CP.min = function(id, count) { notify(this.db, 'counter.min', id, count); return this; }; - Counter.prototype.max = function(id, count) { + CP.max = function(id, count) { notify(this.db, 'counter.max', id, count); return this; }; - Counter.prototype.sum = Counter.prototype.inc = Counter.prototype.hit = function(id, count) { + CP.sum = CP.inc = CP.hit = function(id, count) { notify(this.db, 'counter.hit', id, count); return this; }; - Counter.prototype.remove = function(id) { + CP.remove = function(id) { notify(this.db, 'counter.remove', id); return this; }; - Counter.prototype.read = function(options, callback) { + CP.read = function(options, callback) { send(this.db, 'counter.read', options).callback = callback; return this; }; - Counter.prototype.stats = Counter.prototype.stats_sum = function(top, year, month, day, type, callback) { + CP.stats = CP.stats_sum = function(top, year, month, day, type, callback) { if (typeof(day) == 'function') { callback = day; @@ -429,17 +430,17 @@ exports.worker = function() { return this; }; - Counter.prototype.clear = function(callback) { + CP.clear = function(callback) { send(this.db, 'counter.clear').callback = callback; return this; }; - Storage.prototype.insert = function(doc) { + SP.insert = function(doc) { notify(this.db, 'storage.insert', doc); return this; }; - Storage.prototype.scan = function(beg, end, mapreduce, callback) { + SP.scan = function(beg, end, mapreduce, callback) { if (typeof(beg) === 'function') { mapreduce = beg; @@ -456,12 +457,12 @@ exports.worker = function() { return this; }; - Storage.prototype.mapreduce = function(name, fn) { + SP.mapreduce = function(name, fn) { send(this.db, 'storage.mapreduce', name, fn); return this; }; - Storage.prototype.stats = function(name, callback) { + SP.stats = function(name, callback) { if (typeof(name) === 'function') { callback = name; @@ -472,7 +473,7 @@ exports.worker = function() { return this; }; - Storage.prototype.clear = function(beg, end, callback) { + SP.clear = function(beg, end, callback) { if (typeof(beg) === 'function') { callback = end; @@ -580,27 +581,34 @@ function Table(name, filename) { t.$events = {}; t.step = 0; - t.stopped = false; t.ready = false; t.$free = true; t.$writting = false; t.$reading = false; + t.$meta(); + Fs.createReadStream(t.filename, { end: 2048 }).on('data', function(chunk) { t.parseSchema(chunk.toString('utf8').split('\n')[0].split('|')); t.ready = true; t.next(0); }).on('error', function() { - t.stopped = true; - t.pending_reader.length && (t.pending_reader = []); - t.pending_update.length && (t.pending_update = []); - t.pending_append.length && (t.pending_append = []); - t.pending_reader.length && (t.pending_reader = []); - t.pending_remove.length && (t.pending_remove = []); - t.pending_streamer.length && (t.pending_streamer = []); - t.pending_clean.length && (t.pending_clean = []); - t.pending_clear.length && (t.pending_clear = []); - t.throwStopped(); + var schema = F.config['table.' + name]; + if (schema) { + t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); + Fs.writeFileSync(t.filename, t.stringifySchema() + NEWLINE, 'utf8'); + } else { + t.readonly = true; + t.pending_reader.length && (t.pending_reader = []); + t.pending_update.length && (t.pending_update = []); + t.pending_append.length && (t.pending_append = []); + t.pending_reader.length && (t.pending_reader = []); + t.pending_remove.length && (t.pending_remove = []); + t.pending_streamer.length && (t.pending_streamer = []); + t.pending_clean.length && (t.pending_clean = []); + t.pending_clear.length && (t.pending_clear = []); + t.throwReadonly(); + } }); } @@ -652,7 +660,10 @@ function Database(name, filename, readonly) { self.$reading = false; } -Table.prototype.emit = Database.prototype.emit = function(name, a, b, c, d, e, f, g) { +const TP = Table.prototype; +const DP = Database.prototype; + +TP.emit = DP.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -672,7 +683,7 @@ Table.prototype.emit = Database.prototype.emit = function(name, a, b, c, d, e, f return this; }; -Table.prototype.on = Database.prototype.on = function(name, fn) { +TP.on = DP.on = function(name, fn) { if (!fn.$once) this.$free = false; @@ -684,12 +695,12 @@ Table.prototype.on = Database.prototype.on = function(name, fn) { return this; }; -Table.prototype.once = Database.prototype.once = function(name, fn) { +TP.once = DP.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -Table.prototype.removeListener = Database.prototype.removeListener = function(name, fn) { +TP.removeListener = DP.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -701,7 +712,7 @@ Table.prototype.removeListener = Database.prototype.removeListener = function(na return this; }; -Table.prototype.removeAllListeners = Database.prototype.removeAllListeners = function(name) { +TP.removeAllListeners = DP.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -733,15 +744,15 @@ exports.memory = exports.inmemory = function(name, view) { return INMEMORY[name] = true; }; -Database.prototype.get = function(name) { +TP.get = DP.get = function(name) { return this.meta(name); }; -Database.prototype.set = function(name, value) { +TP.set = DP.set = function(name, value) { return this.meta(name, value); }; -Database.prototype.meta = function(name, value) { +TP.meta = DP.meta = function(name, value) { var self = this; if (value === undefined) return self.metadata ? self.metadata[name] : undefined; @@ -753,7 +764,7 @@ Database.prototype.meta = function(name, value) { return self; }; -Database.prototype.backups = function(filter, callback) { +DP.backups = function(filter, callback) { if (callback === undefined) { callback = filter; @@ -783,13 +794,13 @@ function next_operation(self, type) { self.next(type); } -Database.prototype.ready = function(fn) { +DP.ready = function(fn) { var self = this; fn.call(self); return self; }; -Database.prototype.insert = function(doc, unique) { +DP.insert = function(doc, unique) { var self = this; var builder; @@ -823,11 +834,11 @@ Database.prototype.insert = function(doc, unique) { return builder; }; -Database.prototype.upsert = function(doc) { +DP.upsert = function(doc) { return this.insert(doc, true); }; -Database.prototype.update = function(doc, insert) { +DP.update = function(doc, insert) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); @@ -837,22 +848,22 @@ Database.prototype.update = function(doc, insert) { return builder; }; -Database.prototype.modify = function(doc, insert) { +DP.modify = function(doc, insert) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; var keys = Object.keys(data); - if (!keys.length) - return builder; + if (keys.length) { + self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); + setImmediate(next_operation, self, 2); + } - self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); - setImmediate(next_operation, self, 2); return builder; }; -Database.prototype.restore = function(filename, callback) { +DP.restore = function(filename, callback) { var self = this; self.readonly && self.throwReadonly(); U.wait(() => !self.type, function(err) { @@ -878,7 +889,7 @@ Database.prototype.restore = function(filename, callback) { return self; }; -Database.prototype.backup = function(filename, callback) { +DP.backup = function(filename, callback) { var self = this; self.readonly && self.throwReadonly(); @@ -953,7 +964,7 @@ Database.prototype.backup = function(filename, callback) { return self; }; -Database.prototype.backup2 = function(filename, remove) { +DP.backup2 = function(filename, remove) { if (typeof(filename) === 'boolean') { remove = filename; @@ -986,7 +997,7 @@ Database.prototype.backup2 = function(filename, remove) { return builder; }; -Database.prototype.drop = function() { +DP.drop = function() { var self = this; self.readonly && self.throwReadonly(); self.pending_drops = true; @@ -994,7 +1005,7 @@ Database.prototype.drop = function() { return self; }; -Database.prototype.free = function(force) { +DP.free = function(force) { var self = this; if (!force && !self.$free) return self; @@ -1007,35 +1018,37 @@ Database.prototype.free = function(force) { return self; }; -Database.prototype.release = function() { +DP.release = function() { var self = this; self.inmemory = {}; self.inmemorylastusage = undefined; return self; }; -Database.prototype.clear = function(callback) { +TP.clear = DP.clear = function(callback) { var self = this; + self.readonly && self.throwReadonly(); self.pending_clear.push(callback || NOOP); setImmediate(next_operation, self, 12); return self; }; -Database.prototype.clean = function(callback) { +TP.clean = DP.clean = function(callback) { var self = this; + self.readonly && self.throwReadonly(); self.pending_clean.push(callback || NOOP); setImmediate(next_operation, self, 13); return self; }; -Database.prototype.lock = function(callback) { +TP.lock = DP.lock = function(callback) { var self = this; self.pending_locks.push(callback || NOOP); setImmediate(next_operation, self, 14); return self; }; -Database.prototype.remove = function(filename) { +DP.remove = function(filename) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); @@ -1049,7 +1062,7 @@ Database.prototype.remove = function(filename) { return builder; }; -Database.prototype.find = function(view, builder) { +DP.find = function(view, builder) { var self = this; if (builder) @@ -1068,7 +1081,7 @@ Database.prototype.find = function(view, builder) { return builder; }; -Database.prototype.find2 = function(builder) { +DP.find2 = function(builder) { var self = this; if (builder) @@ -1084,7 +1097,7 @@ Database.prototype.find2 = function(builder) { return builder; }; -Database.prototype.stream = function(fn, repository, callback) { +DP.stream = function(fn, repository, callback) { var self = this; if (typeof(repository) === 'function') { @@ -1097,15 +1110,15 @@ Database.prototype.stream = function(fn, repository, callback) { return self; }; -Database.prototype.throwReadonly = function() { +DP.throwReadonly = function() { throw new Error('Database "{0}" is readonly.'.format(this.name)); }; -Database.prototype.scalar = function(type, field, view) { +DP.scalar = function(type, field, view) { return this.find(view).scalar(type, field); }; -Database.prototype.count = function(view) { +DP.count = function(view) { var self = this; var builder = new DatabaseBuilder(self); @@ -1120,7 +1133,7 @@ Database.prototype.count = function(view) { return builder; }; -Database.prototype.one = function(view) { +DP.one = function(view) { var self = this; var builder = new DatabaseBuilder(self); @@ -1137,7 +1150,7 @@ Database.prototype.one = function(view) { return builder; }; -Database.prototype.top = function(max, view) { +DP.top = function(max, view) { var self = this; var builder = new DatabaseBuilder(self); builder.take(max); @@ -1153,7 +1166,7 @@ Database.prototype.top = function(max, view) { return builder; }; -Database.prototype.view = function(name) { +DP.view = function(name) { var builder = new DatabaseBuilder(this); if (!this.views) this.views = {}; @@ -1181,7 +1194,7 @@ Database.prototype.view = function(name) { const NEXTWAIT = { 7: true, 8: true, 9: true, 12: true, 13: true, 14: true }; -Database.prototype.next = function(type) { +DP.next = function(type) { if (type && NEXTWAIT[this.step]) return; @@ -1277,7 +1290,7 @@ Database.prototype.next = function(type) { } }; -Database.prototype.refresh = function() { +DP.refresh = function() { if (this.views) { this.pending_views = true; setImmediate(next_operation, this, 5); @@ -1290,7 +1303,7 @@ Database.prototype.refresh = function() { // ====================================================================== // InMemory saving -Database.prototype.$save = function(view) { +DP.$save = function(view) { var self = this; setTimeout2('nosql.' + self.name + '.' + view, function() { var data = self.inmemory[view] || EMPTYARRAY; @@ -1307,7 +1320,7 @@ Database.prototype.$save = function(view) { return self; }; -Database.prototype.$inmemory = function(view, callback) { +DP.$inmemory = function(view, callback) { var self = this; self.readonly && self.throwReadonly(); @@ -1344,7 +1357,7 @@ Database.prototype.$inmemory = function(view, callback) { return self; }; -Database.prototype.$meta = function(write) { +TP.$meta = DP.$meta = function(write) { var self = this; @@ -1364,7 +1377,7 @@ Database.prototype.$meta = function(write) { return self; }; -Database.prototype.$append = function() { +DP.$append = function() { var self = this; self.step = 1; @@ -1405,7 +1418,7 @@ function next_append(self) { self.views && setImmediate(views_refresh, self); } -Database.prototype.$append_inmemory = function() { +DP.$append_inmemory = function() { var self = this; self.step = 1; @@ -1430,7 +1443,7 @@ Database.prototype.$append_inmemory = function() { }); }; -Database.prototype.$update = function() { +DP.$update = function() { var self = this; self.step = 2; @@ -1582,7 +1595,7 @@ function views_refresh(self) { self.refresh(); } -Database.prototype.$update_inmemory = function() { +DP.$update_inmemory = function() { var self = this; self.step = 2; @@ -1673,7 +1686,7 @@ Database.prototype.$update_inmemory = function() { }); }; -Database.prototype.$reader = function() { +DP.$reader = function() { var self = this; self.step = 4; @@ -1697,7 +1710,7 @@ Database.prototype.$reader = function() { return self; }; -Database.prototype.$readerview = function() { +DP.$readerview = function() { var self = this; self.step = 6; @@ -1739,7 +1752,7 @@ Database.prototype.$readerview = function() { return self; }; -Database.prototype.$reader2 = function(filename, items, callback, reader) { +DP.$reader2 = function(filename, items, callback, reader) { var self = this; @@ -1916,7 +1929,7 @@ Database.prototype.$reader2 = function(filename, items, callback, reader) { return self; }; -Database.prototype.$reader3 = function() { +DP.$reader3 = function() { var self = this; @@ -2107,7 +2120,7 @@ Database.prototype.$reader3 = function() { return self; }; -Database.prototype.$streamer = function() { +DP.$streamer = function() { var self = this; self.step = 10; @@ -2203,7 +2216,7 @@ function nosqlresort(arr, builder, doc) { } } -Database.prototype.$reader2_inmemory = function(name, items, callback) { +DP.$reader2_inmemory = function(name, items, callback) { var self = this; var filter = items; @@ -2346,7 +2359,7 @@ Database.prototype.$reader2_inmemory = function(name, items, callback) { }); }; -Database.prototype.$views = function() { +DP.$views = function() { var self = this; if (!self.views) { @@ -2445,7 +2458,7 @@ Database.prototype.$views = function() { fs.openread(); }; -Database.prototype.$views_inmemory = function() { +DP.$views_inmemory = function() { var self = this; self.step = 5; @@ -2520,7 +2533,7 @@ Database.prototype.$views_inmemory = function() { }); }; -Database.prototype.$remove = function() { +DP.$remove = function() { var self = this; self.step = 3; @@ -2609,7 +2622,7 @@ Database.prototype.$remove = function() { fs.openupdate(); }; -Database.prototype.$clear = function() { +DP.$clear = function() { var self = this; self.step = 12; @@ -2629,7 +2642,7 @@ Database.prototype.$clear = function() { }); }; -Database.prototype.$clean = function() { +DP.$clean = function() { var self = this; self.step = 13; @@ -2673,7 +2686,7 @@ Database.prototype.$clean = function() { fs.openread(); }; -Database.prototype.$lock = function() { +DP.$lock = function() { var self = this; self.step = 14; @@ -2691,7 +2704,7 @@ Database.prototype.$lock = function() { }); }; -Database.prototype.$remove_inmemory = function() { +DP.$remove_inmemory = function() { var self = this; self.step = 3; @@ -2760,7 +2773,7 @@ Database.prototype.$remove_inmemory = function() { }); }; -Database.prototype.$clear_inmemory = function() { +DP.$clear_inmemory = function() { var self = this; self.step = 12; @@ -2784,7 +2797,7 @@ Database.prototype.$clear_inmemory = function() { }); }; -Database.prototype.$drop = function() { +DP.$drop = function() { var self = this; self.step = 7; @@ -3588,16 +3601,18 @@ DatabaseBuilder.prototype.prepare = function(fn) { }; function Counter(db) { - var self = this; - self.TIMEOUT = 30000; - self.db = db; - self.cache; - self.key = 'nosql' + db.name.hash(); - self.type = 0; // 1 === saving, 2 === reading - self.$events = {}; + var t = this; + t.TIMEOUT = 30000; + t.db = db; + t.cache; + t.key = 'nosql' + db.name.hash(); + t.type = 0; // 1 === saving, 2 === reading + t.$events = {}; } -Counter.prototype.emit = function(name, a, b, c, d, e, f, g) { +const CP = Counter.prototype; + +CP.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -3617,7 +3632,7 @@ Counter.prototype.emit = function(name, a, b, c, d, e, f, g) { return this; }; -Counter.prototype.on = function(name, fn) { +CP.on = function(name, fn) { if (!fn.$once) this.db.$free = false; @@ -3630,12 +3645,12 @@ Counter.prototype.on = function(name, fn) { return this; }; -Counter.prototype.once = function(name, fn) { +CP.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -Counter.prototype.removeListener = function(name, fn) { +CP.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -3647,7 +3662,7 @@ Counter.prototype.removeListener = function(name, fn) { return this; }; -Counter.prototype.removeAllListeners = function(name) { +CP.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -3657,7 +3672,7 @@ Counter.prototype.removeAllListeners = function(name) { return this; }; -Counter.prototype.empty = function(key, value) { +CP.empty = function(key, value) { var self = this; !self.cache && (self.cache = {}); @@ -3671,7 +3686,7 @@ Counter.prototype.empty = function(key, value) { return self; }; -Counter.prototype.min = function(id, count) { +CP.min = function(id, count) { var self = this; @@ -3698,7 +3713,7 @@ Counter.prototype.min = function(id, count) { return self; }; -Counter.prototype.max = function(id, count) { +CP.max = function(id, count) { var self = this; @@ -3724,7 +3739,7 @@ Counter.prototype.max = function(id, count) { return self; }; -Counter.prototype.inc = Counter.prototype.hit = function(id, count) { +CP.inc = CP.hit = function(id, count) { var self = this; @@ -3746,7 +3761,7 @@ Counter.prototype.inc = Counter.prototype.hit = function(id, count) { return self; }; -Counter.prototype.remove = function(id) { +CP.remove = function(id) { var self = this; !self.cache && (self.cache = {}); @@ -3761,7 +3776,7 @@ Counter.prototype.remove = function(id) { return self; }; -Counter.prototype.count = function(id, callback) { +CP.count = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3775,7 +3790,7 @@ Counter.prototype.count = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.maximum = function(id, callback) { +CP.maximum = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3790,7 +3805,7 @@ Counter.prototype.maximum = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.minimum = function(id, callback) { +CP.minimum = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3805,7 +3820,7 @@ Counter.prototype.minimum = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.yearly = Counter.prototype.yearly_sum = function(id, callback) { +CP.yearly = CP.yearly_sum = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3819,7 +3834,7 @@ Counter.prototype.yearly = Counter.prototype.yearly_sum = function(id, callback) return this.read(options, callback); }; -Counter.prototype.monthly = Counter.prototype.monthly_sum = function(id, callback) { +CP.monthly = CP.monthly_sum = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3833,7 +3848,7 @@ Counter.prototype.monthly = Counter.prototype.monthly_sum = function(id, callbac return this.read(options, callback); }; -Counter.prototype.daily = Counter.prototype.daily_sum = function(id, callback) { +CP.daily = CP.daily_sum = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3847,7 +3862,7 @@ Counter.prototype.daily = Counter.prototype.daily_sum = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.yearly_max = function(id, callback) { +CP.yearly_max = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3862,7 +3877,7 @@ Counter.prototype.yearly_max = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.monthly_max = function(id, callback) { +CP.monthly_max = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3877,7 +3892,7 @@ Counter.prototype.monthly_max = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.daily_max = function(id, callback) { +CP.daily_max = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3892,7 +3907,7 @@ Counter.prototype.daily_max = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.yearly_min = function(id, callback) { +CP.yearly_min = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3907,7 +3922,7 @@ Counter.prototype.yearly_min = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.monthly_min = function(id, callback) { +CP.monthly_min = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3922,7 +3937,7 @@ Counter.prototype.monthly_min = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.daily_min = function(id, callback) { +CP.daily_min = function(id, callback) { if (typeof(id) === 'function') { callback = id; @@ -3937,7 +3952,7 @@ Counter.prototype.daily_min = function(id, callback) { return this.read(options, callback); }; -Counter.prototype.read = function(options, callback, reader) { +CP.read = function(options, callback, reader) { var self = this; @@ -4109,15 +4124,15 @@ Counter.prototype.read = function(options, callback, reader) { return self; }; -Counter.prototype.stats_max = function(top, year, month, day, callback) { +CP.stats_max = function(top, year, month, day, callback) { return this.stats(top, year, month, day, 'max', callback); }; -Counter.prototype.stats_min = function(top, year, month, day, callback) { +CP.stats_min = function(top, year, month, day, callback) { return this.stats(top, year, month, day, 'min', callback); }; -Counter.prototype.stats = Counter.prototype.stats_sum = function(top, year, month, day, type, callback, reader) { +CP.stats = CP.stats_sum = function(top, year, month, day, type, callback, reader) { var self = this; @@ -4517,7 +4532,7 @@ function counter_parse_days_all(output, value, year, opt) { } } -Counter.prototype.save = function() { +CP.save = function() { var self = this; self.db.readonly && self.db.throwReadonly(); @@ -4643,7 +4658,7 @@ Counter.prototype.save = function() { return self; }; -Counter.prototype.clear = function(callback) { +CP.clear = function(callback) { var self = this; if (self.type) { @@ -5379,27 +5394,30 @@ Binary.prototype.all = function(callback) { }; function Storage(db, directory) { - this.db = db; - this.directory = directory; - this.pending = []; - this.locked_writer = 0; - this.locked_reader = false; - this.exists = false; + var t = this; + t.db = db; + t.directory = directory; + t.pending = []; + t.locked_writer = 0; + t.locked_reader = false; + t.exists = false; if (!FORK) { - this.$mapreducefile = Path.join(db.directory, db.name + EXTENSION_MAPREDUCE); - this.$mapreduce = []; - this.refresh(); + t.$mapreducefile = Path.join(db.directory, db.name + EXTENSION_MAPREDUCE); + t.$mapreduce = []; + t.refresh(); } } -Storage.prototype.refresh = function() { +const SP = Storage.prototype; + +SP.refresh = function() { try { this.$mapreduce = Fs.readFileSync(this.$mapreducefile).toString('utf8').parseJSON(true); } catch (e) {} return this; }; -Storage.prototype.check = function() { +SP.check = function() { var self = this; if (self.exists) @@ -5414,7 +5432,7 @@ Storage.prototype.check = function() { return self; }; -Storage.prototype.insert = function(doc) { +SP.insert = function(doc) { var self = this; @@ -5459,7 +5477,7 @@ Storage.prototype.insert = function(doc) { return self; }; -Storage.prototype.stats = function(name, fn) { +SP.stats = function(name, fn) { if (fn == null) { var obj = {}; for (var i = 0; i < this.$mapreduce.length; i++) { @@ -5474,7 +5492,7 @@ Storage.prototype.stats = function(name, fn) { return this; }; -Storage.prototype.mapreduce = function(name, fn, def) { +SP.mapreduce = function(name, fn, def) { var self = this; @@ -5504,13 +5522,13 @@ Storage.prototype.mapreduce = function(name, fn, def) { return self; }; -Storage.prototype.$mapreducesave = function() { +SP.$mapreducesave = function() { var self = this; Fs.writeFile(self.$mapreducefile, JSON.stringify(self.$mapreduce, (k, v) => k !== 'reduce' ? v : undefined), F.errorcallback); return self; }; -Storage.prototype.listing = function(beg, end, callback) { +SP.listing = function(beg, end, callback) { var tmp; if (beg) { @@ -5567,7 +5585,7 @@ Storage.prototype.listing = function(beg, end, callback) { return self; }; -Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { +SP.scan = function(beg, end, mapreduce, callback, reverse) { var self = this; if (typeof(beg) === 'function') { @@ -5667,7 +5685,7 @@ Storage.prototype.scan = function(beg, end, mapreduce, callback, reverse) { return self; }; -Storage.prototype.clear = function(beg, end, callback) { +SP.clear = function(beg, end, callback) { var self = this; if (typeof(beg) === 'function') { @@ -5742,12 +5760,12 @@ Backuper.prototype.flush = function() { return self; }; -Table.prototype.insert = function(doc, unique) { +TP.insert = function(doc, unique) { var self = this; var builder; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); if (unique) { @@ -5776,42 +5794,42 @@ Table.prototype.insert = function(doc, unique) { return builder; }; -Table.prototype.update = function(doc, insert) { +TP.update = function(doc, insert) { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_update.push({ builder: builder, doc: doc, count: 0, insert: insert === true ? doc : insert }); setImmediate(next_operation, self, 2); return builder; }; -Table.prototype.modify = function(doc, insert) { +TP.modify = function(doc, insert) { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; var keys = Object.keys(data); - if (!keys.length) - return builder; + if (keys.length) { + self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); + setImmediate(next_operation, self, 2); + } - self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); - setImmediate(next_operation, self, 2); return builder; }; -Table.prototype.remove = function() { +TP.remove = function() { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_remove.push({ builder: builder, count: 0 }); setImmediate(next_operation, self, 3); return builder; }; -Table.prototype.find = function(builder) { +TP.find = function(builder) { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); if (builder) builder.db = self; else @@ -5821,9 +5839,9 @@ Table.prototype.find = function(builder) { return builder; }; -Table.prototype.stream = function(fn, repository, callback) { +TP.stream = function(fn, repository, callback) { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); if (typeof(repository) === 'function') { callback = repository; @@ -5835,30 +5853,26 @@ Table.prototype.stream = function(fn, repository, callback) { return self; }; -Table.prototype.throwReadonly = function() { - throw new Error('Table "{0}" is readonly.'.format(this.name)); -}; - -Table.prototype.throwStopped = function() { - throw new Error('Table "{0}" doesn\'t contain schema'.format(this.name)); +TP.throwReadonly = function() { + throw new Error('Table "{0}" doesn\'t contain any schema'.format(this.name)); }; -Table.prototype.scalar = function(type, field) { +TP.scalar = function(type, field) { return this.find().scalar(type, field); }; -Table.prototype.count = function() { +TP.count = function() { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_reader.push({ builder: builder, count: 0, type: 1 }); setImmediate(next_operation, self, 4); return builder; }; -Table.prototype.one = function() { +TP.one = function() { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.first(); self.pending_reader.push({ builder: builder, count: 0 }); @@ -5866,9 +5880,9 @@ Table.prototype.one = function() { return builder; }; -Table.prototype.top = function(max) { +TP.top = function(max) { var self = this; - self.stopped && self.throwStopped(); + self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.take(max); self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); @@ -5876,23 +5890,15 @@ Table.prototype.top = function(max) { return builder; }; -Table.prototype.clean = function(callback) { - var self = this; - self.stopped && self.throwStopped(); - self.pending_clean.push(callback || NOOP); - setImmediate(next_operation, self, 13); - return self; -}; - -Table.prototype.clear = function(callback) { +TP.lock = function(callback) { var self = this; - self.stopped && self.throwStopped(); - self.pending_clear.push(callback || NOOP); - setImmediate(next_operation, self, 12); + self.readonly && self.throwReadonly(); + self.pending_locks.push(callback || NOOP); + setImmediate(next_operation, self, 14); return self; }; -Table.prototype.next = function(type) { +TP.next = function(type) { if (!this.ready || (type && NEXTWAIT[this.step])) return; @@ -5957,7 +5963,7 @@ Table.prototype.next = function(type) { } }; -Table.prototype.$append = function() { +TP.$append = function() { var self = this; self.step = 1; @@ -5988,7 +5994,7 @@ Table.prototype.$append = function() { }, () => setImmediate(next_append, self)); }; -Table.prototype.$reader = function() { +TP.$reader = function() { var self = this; @@ -6183,7 +6189,7 @@ Table.prototype.$reader = function() { return self; }; -Table.prototype.$update = function() { +TP.$update = function() { var self = this; self.step = 2; @@ -6355,7 +6361,7 @@ Table.prototype.$update = function() { return self; }; -Table.prototype.$remove = function() { +TP.$remove = function() { var self = this; self.step = 3; @@ -6463,7 +6469,7 @@ Table.prototype.$remove = function() { fs.openupdate(); }; -Table.prototype.$clean = function() { +TP.$clean = function() { var self = this; self.step = 13; @@ -6507,7 +6513,7 @@ Table.prototype.$clean = function() { fs.openread(); }; -Table.prototype.$clear = function() { +TP.$clear = function() { var self = this; self.step = 12; @@ -6529,7 +6535,25 @@ Table.prototype.$clear = function() { }); }; -Table.prototype.$streamer = function() { +TP.$lock = function() { + + var self = this; + self.step = 14; + + if (!self.pending_locks.length) { + self.next(0); + return; + } + + var filter = self.pending_locks.splice(0); + filter.wait(function(fn, next) { + fn.call(self, next); + }, function() { + self.next(0); + }); +}; + +TP.$streamer = function() { var self = this; self.step = 10; @@ -6573,7 +6597,7 @@ Table.prototype.$streamer = function() { return self; }; -Table.prototype.parseSchema = function() { +TP.parseSchema = function() { var self = this; var arr = arguments[0] instanceof Array ? arguments[0] : arguments; @@ -6608,7 +6632,7 @@ Table.prototype.parseSchema = function() { return self; }; -Table.prototype.stringifySchema = function() { +TP.stringifySchema = function() { var self = this; var data = []; @@ -6640,7 +6664,7 @@ Table.prototype.stringifySchema = function() { return data.join('|'); }; -Table.prototype.parseData = function(data, cache) { +TP.parseData = function(data, cache) { var self = this; var obj = {}; @@ -6675,7 +6699,7 @@ Table.prototype.parseData = function(data, cache) { break; case 3: // Boolean val = data.line[pos]; - obj[key] = BOOLEAN[val]; + obj[key] = BOOLEAN[val] == 1; break; case 4: // Date val = data.line[pos]; @@ -6692,7 +6716,7 @@ Table.prototype.parseData = function(data, cache) { return obj; }; -Table.prototype.stringify = function(doc) { +TP.stringify = function(doc) { var self = this; var output = ''; @@ -6735,7 +6759,7 @@ Table.prototype.stringify = function(doc) { return (esc ? '*' : '+') + output; }; -Table.prototype.free = function(force) { +TP.free = function(force) { var self = this; if (!force && !self.$free) return self; From 10cd939b5810854a7d625a623ed548a1b380f62d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 23:47:27 +0200 Subject: [PATCH 0422/1669] Improved NoSQL. --- nosql.js | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/nosql.js b/nosql.js index b42628e93..ac9d48b1f 100755 --- a/nosql.js +++ b/nosql.js @@ -358,12 +358,12 @@ exports.worker = function() { return this; }; - DP.clear = function(callback) { + TP.clear = DP.clear = function(callback) { send(this, 'clear').callback = callback; return this; }; - DP.clean = function(callback) { + TP.clean = DP.clean = function(callback) { send(this, 'clean').callback = callback; return this; }; @@ -488,7 +488,7 @@ exports.worker = function() { return this; }; - Table.prototype.find = function(builder) { + TP.find = function(builder) { if (builder) builder.db = this; else @@ -496,19 +496,19 @@ exports.worker = function() { return send(this, 'find').builder = builder; }; - Table.prototype.top = function(max) { + TP.top = function(max) { var builder = new DatabaseBuilder(this); builder.take(max); return send(this, 'find').builder = builder; }; - Table.prototype.one = function() { + TP.one = function() { var builder = new DatabaseBuilder(this); builder.first(); return send(this, 'one').builder = builder; }; - Table.prototype.insert = function(doc, unique) { + TP.insert = function(doc, unique) { var self = this; var builder; @@ -539,28 +539,18 @@ exports.worker = function() { return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); }; - Table.prototype.count = function() { + TP.count = function() { var builder = new DatabaseBuilder(this); return send(this, 'count').builder = builder; }; - Table.prototype.update = function(doc, insert) { + TP.update = function(doc, insert) { return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); }; - Table.prototype.modify = function(doc, insert) { + TP.modify = function(doc, insert) { return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); }; - - Table.prototype.clear = function(callback) { - send(this, 'clear').callback = callback; - return this; - }; - - Table.prototype.clean = function(callback) { - send(this, 'clean').callback = callback; - return this; - }; }; function Table(name, filename) { From 861677dcf1b50aacf07ba5b03bc8b3b81c3fb2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 25 Jun 2018 23:47:49 +0200 Subject: [PATCH 0423/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 93286ff07..b4d6ad927 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-80", + "version": "3.0.0-81", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From c696bb4ca62d44cf95e619a5f012c7919bdb4ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 00:17:57 +0200 Subject: [PATCH 0424/1669] Added `Table.backups()`. --- nosql.js | 102 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/nosql.js b/nosql.js index ac9d48b1f..ecfb01465 100755 --- a/nosql.js +++ b/nosql.js @@ -44,6 +44,7 @@ if (!global.framework_builders) const EXTENSION = '.nosql'; const EXTENSION_TABLE = '.table'; +const EXTENSION_TABLE_BACKUP = '.table-backup'; const EXTENSION_BINARY = '.nosql-binary'; const EXTENSION_TMP = '.nosql-tmp'; const EXTENSION_LOG = '.nosql-log'; @@ -556,6 +557,7 @@ exports.worker = function() { function Table(name, filename) { var t = this; t.filename = filename + EXTENSION_TABLE; + t.filenameBackup = filename + EXTENSION_TABLE_BACKUP; t.directory = Path.dirname(filename); t.name = name; t.$name = '$' + name; @@ -595,6 +597,7 @@ function Table(name, filename) { t.pending_reader.length && (t.pending_reader = []); t.pending_remove.length && (t.pending_remove = []); t.pending_streamer.length && (t.pending_streamer = []); + t.pending_locks.length && (t.pending_locks = []); t.pending_clean.length && (t.pending_clean = []); t.pending_clear.length && (t.pending_clear = []); t.throwReadonly(); @@ -754,7 +757,7 @@ TP.meta = DP.meta = function(name, value) { return self; }; -DP.backups = function(filter, callback) { +TP.backups = DP.backups = function(filter, callback) { if (callback === undefined) { callback = filter; @@ -762,15 +765,28 @@ DP.backups = function(filter, callback) { } var self = this; + var isTable = self instanceof Table; + + if (isTable && !self.ready) { + setTimeout((self, filter, callback) => self.backups(filter, callback), 500, self, filter, callback); + return self; + } + var stream = Fs.createReadStream(self.filenameBackup); var output = []; + var tmp = {}; + + tmp.keys = self.$keys; stream.on('data', U.streamer(NEWLINEBUF, function(item, index) { var end = item.indexOf('|', item.indexOf('|') + 2); var meta = item.substring(0, end); var arr = meta.split('|'); var dv = arr[0].trim().replace(' ', 'T') + ':00.000Z'; - var obj = { id: index + 1, date: dv.parseDate(), user: arr[1].trim(), data: item.substring(end + 1).trim().parseJSON(true) }; + tmp.line = item.substring(end + 1).trim(); + if (isTable) + tmp.line = tmp.line.split('|'); + var obj = { id: index + 1, date: dv.parseDate(), user: arr[1].trim(), data: self instanceof Table ? self.parseData(tmp) : tmp.line.parseJSON(true) }; if (!filter || filter(obj)) output.push(obj); }), stream); @@ -1033,6 +1049,7 @@ TP.clean = DP.clean = function(callback) { TP.lock = DP.lock = function(callback) { var self = this; + self.readonly && self.throwReadonly(); self.pending_locks.push(callback || NOOP); setImmediate(next_operation, self, 14); return self; @@ -5843,6 +5860,79 @@ TP.stream = function(fn, repository, callback) { return self; }; +TP.extend = function(schema, callback) { + var self = this; + self.readonly && self.throwReadonly(); + self.lock(function(next) { + + var olds = self.$schema; + var oldk = self.$keys; + + self.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); + + var meta = self.stringifySchema() + NEWLINE; + var news = self.$schema; + var newk = self.$keys; + + self.$schema = olds; + self.$keys = oldk; + + var count = 0; + var fs = new NoSQLStream(self.filename); + var data = {}; + + var tmp = self.filename + '-tmp'; + var writer = Fs.createWriteStream(tmp); + + writer.write(meta, 'utf8'); + writer.on('finish', function() { + Fs.rename(tmp, self.filename, function() { + next(); + callback && callback(); + }); + }); + + data.keys = self.$keys; + fs.divider = '\n'; + + fs.ondocuments = function() { + + var lines = fs.docs.split(fs.divider); + var items = []; + + self.$schema = olds; + self.$keys = oldk; + + for (var a = count ? 0 : 1; a < lines.length; a++) { + data.line = lines[a].split('|'); + data.index = count++; + var doc = self.parseData(data); + items.push(doc); + } + + self.$schema = news; + self.$keys = newk; + + var buffer = ''; + for (var i = 0; i < items.length; i++) + buffer += self.stringify(items[i]) + NEWLINE; + buffer && writer.write(buffer, 'utf8'); + }; + + fs.$callback = function() { + self.$schema = news; + self.$keys = newk; + writer.end(); + fs = null; + }; + + fs.openread(); + }); + + return self; +}; + + TP.throwReadonly = function() { throw new Error('Table "{0}" doesn\'t contain any schema'.format(this.name)); }; @@ -5880,14 +5970,6 @@ TP.top = function(max) { return builder; }; -TP.lock = function(callback) { - var self = this; - self.readonly && self.throwReadonly(); - self.pending_locks.push(callback || NOOP); - setImmediate(next_operation, self, 14); - return self; -}; - TP.next = function(type) { if (!this.ready || (type && NEXTWAIT[this.step])) From 03e2e9c42122d3a78649e9b7d0c988341b49e95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 11:29:22 +0200 Subject: [PATCH 0425/1669] Impoved `TABLE()` schemas. --- nosql.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index ecfb01465..266644a4b 100755 --- a/nosql.js +++ b/nosql.js @@ -580,15 +580,29 @@ function Table(name, filename) { t.$meta(); + var schema = F.config['table.' + name]; + Fs.createReadStream(t.filename, { end: 2048 }).on('data', function(chunk) { + + if (schema) { + t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); + schema = t.stringifySchema(); + } + t.parseSchema(chunk.toString('utf8').split('\n')[0].split('|')); + t.ready = true; t.next(0); + + if (t.stringifySchema() !== schema) + t.extend(schema); + }).on('error', function() { - var schema = F.config['table.' + name]; if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); Fs.writeFileSync(t.filename, t.stringifySchema() + NEWLINE, 'utf8'); + t.ready = true; + t.next(0); } else { t.readonly = true; t.pending_reader.length && (t.pending_reader = []); From 4621e7f4970cc39611d8b0e67ae6a54f8ef0ab56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 11:31:58 +0200 Subject: [PATCH 0426/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b4d6ad927..d1ec79d06 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-81", + "version": "3.0.0-82", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0ef54fad049a2c238abbfd4d46490aa018e36862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 12:44:02 +0200 Subject: [PATCH 0427/1669] Fixed extending of table. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 266644a4b..4f6ebceea 100755 --- a/nosql.js +++ b/nosql.js @@ -594,7 +594,7 @@ function Table(name, filename) { t.ready = true; t.next(0); - if (t.stringifySchema() !== schema) + if (schema && t.stringifySchema() !== schema) t.extend(schema); }).on('error', function() { From b9aabca48304032df149d62f3c2c7646db5801d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 14:11:15 +0200 Subject: [PATCH 0428/1669] Fixed initialization read. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 4f6ebceea..85c32a928 100755 --- a/nosql.js +++ b/nosql.js @@ -582,7 +582,7 @@ function Table(name, filename) { var schema = F.config['table.' + name]; - Fs.createReadStream(t.filename, { end: 2048 }).on('data', function(chunk) { + Fs.createReadStream(t.filename, { end: 10 }).once('data', function(chunk) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); From 6fd5dba98385fddc4ba676cd862fbe123acc5cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 15:12:46 +0200 Subject: [PATCH 0429/1669] Improved schemas. --- builders.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/builders.js b/builders.js index 8aab44136..4dc9e870c 100755 --- a/builders.js +++ b/builders.js @@ -1137,10 +1137,14 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, return; } + if (controller instanceof SchemaOptions) + controller = controller.controller; + if (model && !controller && model.$$controller) controller = model.$$controller; var builder = new ErrorBuilder(); + self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -1219,6 +1223,9 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); + if (controller instanceof SchemaOptions) + controller = controller.controller; + var output = self.default(); var $type = 'get'; @@ -1289,6 +1296,9 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { if (!self.onRemove) return callback(new Error('Operation "{0}/{1}" not found'.format(self.name, $type))); + if (controller instanceof SchemaOptions) + controller = controller.controller; + self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -1353,6 +1363,9 @@ SchemaBuilderEntity.prototype.query = function(options, callback, controller) { options = undefined; } + if (controller instanceof SchemaOptions) + controller = controller.controller; + var self = this; var builder = new ErrorBuilder(); var $type = 'query'; @@ -2100,6 +2113,9 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk return self; } + if (controller instanceof SchemaOptions) + controller = controller.controller; + if (model && !controller && model.$$controller) controller = model.$$controller; @@ -2233,6 +2249,9 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca return self; } + if (controller instanceof SchemaOptions) + controller = controller.controller; + if (model && !controller && model.$$controller) controller = model.$$controller; @@ -2368,6 +2387,9 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); + if (controller instanceof SchemaOptions) + controller = controller.controller; + if (model && !controller && model.$$controller) controller = model.$$controller; From 09141d8c38916b0923724c92ad91f206f0f336b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 26 Jun 2018 23:39:10 +0200 Subject: [PATCH 0430/1669] Improved escaping values. --- nosql.js | 115 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 101 insertions(+), 14 deletions(-) diff --git a/nosql.js b/nosql.js index 85c32a928..7321f9544 100755 --- a/nosql.js +++ b/nosql.js @@ -62,8 +62,9 @@ const NEWLINE = '\n'; const REGBOOL = /":true/g; // for updates of boolean types const REGCHINA = /[\u3400-\u9FBF]/; const REGCLEAN = /^[\s]+|[\s]+$/g; -const REGTESCAPE = /\|/g; -const REGTUNESCAPE = /%7C/g; +const REGTESCAPE = /\||\n|\r/g; +const REGTUNESCAPE = /%7C|%0D|%0A/g; +const REGTESCAPETEST = /\||\n|\r/; const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 }; const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' }; @@ -582,14 +583,14 @@ function Table(name, filename) { var schema = F.config['table.' + name]; - Fs.createReadStream(t.filename, { end: 10 }).once('data', function(chunk) { + Fs.createReadStream(t.filename, { end: 1000 }).once('data', function(chunk) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); schema = t.stringifySchema(); } - t.parseSchema(chunk.toString('utf8').split('\n')[0].split('|')); + t.parseSchema(chunk.toString('utf8').split('\n', 1)[0].split('|')); t.ready = true; t.next(0); @@ -876,7 +877,24 @@ DP.modify = function(doc, insert) { var keys = Object.keys(data); if (keys.length) { - self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); + var inc = null; + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + switch (key[0]) { + case '+': + case '-': + case '*': + case '/': + !inc && (inc = {}); + var tmp = key.substring(1); + inc[tmp] = key[0]; + doc[tmp] = doc[key]; + doc[key] = undefined; + keys[i] = tmp; + break; + } + } + self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, inc: inc, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); } @@ -1531,7 +1549,22 @@ DP.$update = function() { if (val !== undefined) { if (typeof(val) === 'function') output[key] = val(output[key], output); - else + else if (item.inc && item.inc[key]) { + switch (item.inc[key]) { + case '+': + output[key] = (output[key] || 0) + val; + break; + case '-': + output[key] = (output[key] || 0) - val; + break; + case '*': + output[key] = (output[key] || 0) + val; + break; + case '/': + output[key] = (output[key] || 0) / val; + break; + } + } else output[key] = val; } } @@ -5830,12 +5863,27 @@ TP.modify = function(doc, insert) { var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; var keys = Object.keys(data); - if (keys.length) { - self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, insert: insert === true ? data : insert }); + var inc = null; + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + switch (key[0]) { + case '+': + case '-': + case '*': + case '/': + !inc && (inc = {}); + var tmp = key.substring(1); + inc[tmp] = key[0]; + doc[tmp] = doc[key]; + doc[key] = undefined; + keys[i] = tmp; + break; + } + } + self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, inc: inc, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); } - return builder; }; @@ -6368,7 +6416,22 @@ TP.$update = function() { if (val !== undefined) { if (typeof(val) === 'function') output[key] = val(output[key], output); - else + else if (item.inc && item.inc[key]) { + switch (item.inc[key]) { + case '+': + output[key] = (output[key] || 0) + val; + break; + case '-': + output[key] = (output[key] || 0) - val; + break; + case '*': + output[key] = (output[key] || 0) + val; + break; + case '/': + output[key] = (output[key] || 0) / val; + break; + } + } else output[key] = val; } } @@ -6778,7 +6841,7 @@ TP.parseData = function(data, cache) { case 1: // String obj[key] = data.line[pos]; if (esc && obj[key]) - obj[key] = obj[key].replace(REGTUNESCAPE, '|'); + obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); break; case 2: // Number obj[key] = +data.line[pos]; @@ -6794,7 +6857,7 @@ TP.parseData = function(data, cache) { case 5: // Object val = data.line[pos]; if (esc && val) - val = val.replace(REGTUNESCAPE, '|'); + val = val.replace(REGTUNESCAPE, regtescapereverse); obj[key] = val ? val.parseJSON(true) : null; break; } @@ -6833,9 +6896,9 @@ TP.stringify = function(doc) { if (!esc && (meta.type === 1 || meta.type === 5)) { val += ''; - if (val.indexOf('|') !== -1) { + if (REGTESCAPETEST.test(val)) { esc = true; - val = val.replace(REGTESCAPE, '%7C'); + val = val.replace(REGTESCAPE, regtescape); } } @@ -6845,6 +6908,30 @@ TP.stringify = function(doc) { return (esc ? '*' : '+') + output; }; +function regtescapereverse(c) { + switch (c) { + case '%0A': + return '\n'; + case '%0D': + return '\r'; + case '%7C': + return '|'; + } + return c; +} + +function regtescape(c) { + switch (c) { + case '\n': + return '%0A'; + case '\r': + return '%0D'; + case '|': + return '%7C'; + } + return c; +} + TP.free = function(force) { var self = this; if (!force && !self.$free) From 7860e662a5f0824f0f8d8726a074ef487f64b13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 08:23:20 +0200 Subject: [PATCH 0431/1669] Added `database.listing()` method. --- changes.txt | 1 + nosql.js | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index 931dafa90..01baf3997 100755 --- a/changes.txt +++ b/changes.txt @@ -35,6 +35,7 @@ - added: `SchemaOptions.stop()` alias to `$.model.$stop()` - added: a new route flag type `&group` something like `roles` but groups aren't evaluated - added: `route.groups` with defined groups +- added: NoSQL `database.listing([view])` which generates a listing response - added: `DatabaseBuilder.insert(fn(doc))` can modify a document after `update` or `modify` has `insert` mode - added: `DatabaseBuilder.query(code)` can contain a raw JS condition in the form e.g. `doc.age > 18 && doc.age < 33` - added: `DatabaseBuilder.regexp(name, regexp)` RegExp search in strings diff --git a/nosql.js b/nosql.js index 7321f9544..5634612ca 100755 --- a/nosql.js +++ b/nosql.js @@ -260,6 +260,16 @@ exports.worker = function() { PRINTLN('ERROR --> NoSQL events are not supported in fork mode.'); }; + DP.listing = function(view, builder) { + if (builder) + builder.db = this; + else + builder = new DatabaseBuilder(this); + builder.$options.listing = true; + builder.$take = builder.$options.take = 100; + return send(this, 'find', view).builder = builder; + }; + DP.find = function(view, builder) { if (builder) builder.db = this; @@ -498,6 +508,16 @@ exports.worker = function() { return send(this, 'find').builder = builder; }; + TP.listing = function(view, builder) { + if (builder) + builder.db = this; + else + builder = new DatabaseBuilder(this); + builder.$options.listing = true; + builder.$take = builder.$options.take = 100; + return send(this, 'find', view).builder = builder; + }; + TP.top = function(max) { var builder = new DatabaseBuilder(this); builder.take(max); @@ -1101,6 +1121,28 @@ DP.remove = function(filename) { return builder; }; +DP.listing = function(view, builder) { + var self = this; + + if (builder) + builder.db = self; + else + builder = new DatabaseBuilder(self); + + builder.$options.listing = true; + builder.$take = builder.$options.take = 100; + + if (view) { + self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); + setImmediate(next_operation, self, 6); + } else { + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); + } + + return builder; +}; + DP.find = function(view, builder) { var self = this; @@ -1806,6 +1848,14 @@ DP.$readerview = function() { return self; }; +function listing(builder, item) { + + var skip = builder.$options.skip || 0; + var take = builder.$options.take || 0; + + return { page: ((skip / take) + 1), pages: item.count ? Math.ceil(item.count / take) : 0, limit: take, count: item.count, items: item.response || [] }; +} + DP.$reader2 = function(filename, items, callback, reader) { var self = this; @@ -1927,6 +1977,7 @@ DP.$reader2 = function(filename, items, callback, reader) { }; fs.$callback = function() { + for (var i = 0; i < length; i++) { var item = filter[i]; var builder = item.builder; @@ -1938,6 +1989,8 @@ DP.$reader2 = function(filename, items, callback, reader) { output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; else if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -1964,8 +2017,12 @@ DP.$reader2 = function(filename, items, callback, reader) { if (builder.$options.first) output = item.response ? item.response[0] : undefined; - else - output = item.response || []; + else { + if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + } builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); builder.done(); @@ -2132,6 +2189,8 @@ DP.$reader3 = function() { output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; else if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -2158,6 +2217,8 @@ DP.$reader3 = function() { if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -2377,6 +2438,8 @@ DP.$reader2_inmemory = function(name, items, callback) { output = builder.$options.scalar === 'avg' ? item.scalar / item.counter : item.scalar; else if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -2402,6 +2465,8 @@ DP.$reader2_inmemory = function(name, items, callback) { if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -3451,8 +3516,9 @@ DatabaseBuilder.prototype.limit = function(count) { }; DatabaseBuilder.prototype.page = function(page, limit) { - this.$take = this.$options.take = limit; - this.$skip = this.$options.skip = page * limit; + if (limit) + this.$take = this.$options.take = limit; + this.$skip = this.$options.skip = page * this.$take; return this; }; @@ -5896,6 +5962,20 @@ TP.remove = function() { return builder; }; +TP.listing = function(builder) { + var self = this; + self.readonly && self.throwReadonly(); + if (builder) + builder.db = self; + else + builder = new DatabaseBuilder(self); + builder.$options.listing = true; + builder.$take = builder.$options.take = 100; + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); + return builder; +}; + TP.find = function(builder) { var self = this; self.readonly && self.throwReadonly(); @@ -6281,6 +6361,8 @@ TP.$reader = function() { output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; else if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -6307,6 +6389,8 @@ TP.$reader = function() { if (builder.$options.first) output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); else output = item.response || []; @@ -6764,6 +6848,7 @@ TP.parseSchema = function() { type = 2; break; case 'boolean': + case 'bool': type = 3; break; case 'date': @@ -6797,7 +6882,7 @@ TP.stringifySchema = function() { type = 'number'; break; case 3: - type = 'boolean'; + type = 'bool'; break; case 4: type = 'date'; From f4d207062028b1deeb03cc6784ddcb56ac6913af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 08:26:33 +0200 Subject: [PATCH 0432/1669] Replaced `bool` to `boolean`. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 5634612ca..deb37ef09 100755 --- a/nosql.js +++ b/nosql.js @@ -6882,7 +6882,7 @@ TP.stringifySchema = function() { type = 'number'; break; case 3: - type = 'bool'; + type = 'boolean'; break; case 4: type = 'date'; From d2850c0857eecfeef7e44ef0bdba13f7ab0bfe43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 08:49:44 +0200 Subject: [PATCH 0433/1669] Updated TABLE `date` serialization/deserialization. --- nosql.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index deb37ef09..239f14c6d 100755 --- a/nosql.js +++ b/nosql.js @@ -6937,7 +6937,7 @@ TP.parseData = function(data, cache) { break; case 4: // Date val = data.line[pos]; - obj[key] = val ? new Date(val) : null; + obj[key] = val ? new Date(val[10] === 'T' ? val : +val) : null; break; case 5: // Object val = data.line[pos]; @@ -6972,7 +6972,8 @@ TP.stringify = function(doc) { val = (val == true ? '1' : '0'); break; case 4: // Date - val = val ? val.toISOString() : ''; + // val = val ? val.toISOString() : ''; + val = val ? val.getTime() : ''; break; case 5: // Object val = val ? JSON.stringify(val) : ''; From 20da99c0a88889dc880988fda4d8349dcf28bbe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 08:55:16 +0200 Subject: [PATCH 0434/1669] Fixed TABLE `fields` parsing. --- nosql.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nosql.js b/nosql.js index 239f14c6d..9ece55999 100755 --- a/nosql.js +++ b/nosql.js @@ -6232,14 +6232,16 @@ TP.$reader = function() { var fil = filter[i]; if (!fil.builder.$options.first || fil.builder.$options.sort) first = false; - - if (fil.builder.$keys == null) + if (fil.builder.$keys == null && keys) keys = null; - else { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } + else if (keys) { + if (fil.builder.$options.fields) { + for (var j = 0; j < fil.builder.$keys.length; j++) { + keyscount++; + keys[fil.builder.$keys[j]] = 1; + } + } else + keys = null; } fil.scalarcount = 0; @@ -6274,6 +6276,7 @@ TP.$reader = function() { var item = filter[i]; var builder = item.builder; item.filter.index = indexer; + var output = item.compare(obj, item.filter, indexer); if (!output) continue; From 1f9a390ee8d0b2d1bf1abbb9b777216890f50531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 10:01:55 +0200 Subject: [PATCH 0435/1669] Improved `TABLE()` instance. --- nosql.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nosql.js b/nosql.js index 9ece55999..631fefff2 100755 --- a/nosql.js +++ b/nosql.js @@ -579,6 +579,7 @@ function Table(name, filename) { var t = this; t.filename = filename + EXTENSION_TABLE; t.filenameBackup = filename + EXTENSION_TABLE_BACKUP; + t.filenameCounter = filename + EXTENSION_TABLE + EXTENSION_COUNTER; t.directory = Path.dirname(filename); t.name = name; t.$name = '$' + name; @@ -599,6 +600,7 @@ function Table(name, filename) { t.$writting = false; t.$reading = false; + t.counter = new Counter(t); t.$meta(); var schema = F.config['table.' + name]; @@ -3725,7 +3727,7 @@ function Counter(db) { t.TIMEOUT = 30000; t.db = db; t.cache; - t.key = 'nosql' + db.name.hash(); + t.key = (db instanceof Table ? 'table' : 'nosql') + db.name.hash(); t.type = 0; // 1 === saving, 2 === reading t.$events = {}; } @@ -6843,10 +6845,7 @@ TP.parseSchema = function() { for (var i = 0; i < arr.length; i++) { var arg = arr[i].split(':'); var type = 0; - switch (arg[1].toLowerCase().trim()) { - case 'string': - type = 1; - break; + switch ((arg[1] || '').toLowerCase().trim()) { case 'number': type = 2; break; @@ -6860,6 +6859,10 @@ TP.parseSchema = function() { case 'object': type = 5; break; + case 'string': + default: + type = 1; + break; } var name = arg[0].trim(); self.$schema[name] = { type: type, pos: i }; From 02caa599d1578408fac205089a94c99301bf0133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 10:29:26 +0200 Subject: [PATCH 0436/1669] Fixed TABLE meta. --- nosql.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 631fefff2..a858283c2 100755 --- a/nosql.js +++ b/nosql.js @@ -580,6 +580,7 @@ function Table(name, filename) { t.filename = filename + EXTENSION_TABLE; t.filenameBackup = filename + EXTENSION_TABLE_BACKUP; t.filenameCounter = filename + EXTENSION_TABLE + EXTENSION_COUNTER; + t.filenameMeta = filename + EXTENSION_TABLE + '-meta'; t.directory = Path.dirname(filename); t.name = name; t.$name = '$' + name; @@ -6545,6 +6546,7 @@ TP.$update = function() { } var upd = self.stringify(doc); + console.log(upd, doc); if (upd === rec.doc) continue; @@ -6908,7 +6910,7 @@ TP.parseData = function(data, cache) { var self = this; var obj = {}; - var esc = false; + var esc = data.line[0] === '*'; var val; for (var i = 0; i < data.keys.length; i++) { @@ -6923,9 +6925,6 @@ TP.parseData = function(data, cache) { if (meta == null) continue; - if (i === 0) - esc = data.line[0] === '*'; - var pos = meta.pos + 1; switch (meta.type) { From c637487a9cbec28f2097649f4f1806617a1df4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 10:31:48 +0200 Subject: [PATCH 0437/1669] Removed useless logging. --- nosql.js | 1 - 1 file changed, 1 deletion(-) diff --git a/nosql.js b/nosql.js index a858283c2..1dbc413b0 100755 --- a/nosql.js +++ b/nosql.js @@ -6546,7 +6546,6 @@ TP.$update = function() { } var upd = self.stringify(doc); - console.log(upd, doc); if (upd === rec.doc) continue; From e88c6263adeb49b21d462ac38002e5270938a10c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 12:26:45 +0200 Subject: [PATCH 0438/1669] Added missing error handling in global schemas. --- index.js | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index f4fa51d64..912c8413c 100755 --- a/index.js +++ b/index.js @@ -361,28 +361,40 @@ global.$MAKE = function(schema, model, filter, callback, novalidate, argument) { global.$QUERY = function(schema, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); - o && o.query(options, callback, controller); + if (o) + o.query(options, callback, controller); + else + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); return !!o; }; global.$GET = function(schema, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); - o && o.get(options, callback, controller); + if (o) + o.get(options, callback, controller); + else + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); return !!o; }; global.$WORKFLOW = function(schema, name, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); - o && o.workflow2(name, options, callback, controller); + if (o) + o.workflow2(name, options, callback, controller); + else + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); return !!o; }; global.$TRANSFORM = function(schema, name, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); - o && o.transform2(name, options, callback, controller); + if (o) + o.transform2(name, options, callback, controller); + else + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); return !!o; }; @@ -396,7 +408,10 @@ global.$REMOVE = function(schema, options, callback, controller) { options = EMPTYOBJECT; } - o && o.remove(options, callback, controller); + if (o) + o.remove(options, callback, controller); + else + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); return !!o; }; @@ -423,10 +438,16 @@ function performschema(type, schema, model, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); + + if (!o) { + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + return false; + } + o.make(model, function(err, model) { - if (err) - callback(err); - else { + if (err) { + callback && callback(err); + } else { model.$$controller = controller; model[type](options, callback); } @@ -444,6 +465,12 @@ global.$ASYNC = function(schema, callback, index, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]).default(); + + if (!o) { + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + return EMPTYOBJECT; + } + controller && (o.$$controller = controller); return o.$async(callback, index); }; @@ -451,7 +478,10 @@ global.$ASYNC = function(schema, callback, index, controller) { global.$OPERATION = function(schema, name, options, callback, controller) { schema = parseSchema(schema); var o = framework_builders.getschema(schema[0], schema[1]); - o && o.operation2(name, options, callback, controller); + if (o) + o.operation2(name, options, callback, controller); + else + callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); return !!o; }; From d01f98c4c8e6544e9093c1ea1ae6d872fbe067e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 12:36:14 +0200 Subject: [PATCH 0439/1669] Fixed critical bug with `binary.insert()`. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 1dbc413b0..6d264e3b1 100755 --- a/nosql.js +++ b/nosql.js @@ -4920,7 +4920,7 @@ Binary.prototype.insert = function(name, buffer, custom, callback) { if (typeof(buffer) === 'function') { callback = buffer; buffer = custom = null; - } else { + } else if (!buffer.resume) { callback = custom; custom = buffer; buffer = null; @@ -5122,7 +5122,7 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { if (typeof(buffer) === 'function') { callback = buffer; buffer = custom = null; - } else { + } else if (!buffer.resume) { callback = custom; custom = buffer; buffer = null; From adf17cfe7613f2bd725c5d1f11ad14fccf5f1c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 12:36:39 +0200 Subject: [PATCH 0440/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d1ec79d06..f51e5d9a6 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-82", + "version": "3.0.0-83", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 6a47c74fb8a24fc8c0616b1d0ccb3fe7b4a746a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 13:11:30 +0200 Subject: [PATCH 0441/1669] Improved error messages. --- index.js | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 912c8413c..afab3084a 100755 --- a/index.js +++ b/index.js @@ -364,7 +364,7 @@ global.$QUERY = function(schema, options, callback, controller) { if (o) o.query(options, callback, controller); else - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return !!o; }; @@ -374,7 +374,7 @@ global.$GET = function(schema, options, callback, controller) { if (o) o.get(options, callback, controller); else - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return !!o; }; @@ -384,7 +384,7 @@ global.$WORKFLOW = function(schema, name, options, callback, controller) { if (o) o.workflow2(name, options, callback, controller); else - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return !!o; }; @@ -394,7 +394,7 @@ global.$TRANSFORM = function(schema, name, options, callback, controller) { if (o) o.transform2(name, options, callback, controller); else - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return !!o; }; @@ -411,7 +411,7 @@ global.$REMOVE = function(schema, options, callback, controller) { if (o) o.remove(options, callback, controller); else - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return !!o; }; @@ -440,7 +440,7 @@ function performschema(type, schema, model, options, callback, controller) { var o = framework_builders.getschema(schema[0], schema[1]); if (!o) { - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return false; } @@ -467,7 +467,7 @@ global.$ASYNC = function(schema, callback, index, controller) { var o = framework_builders.getschema(schema[0], schema[1]).default(); if (!o) { - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return EMPTYOBJECT; } @@ -481,7 +481,7 @@ global.$OPERATION = function(schema, name, options, callback, controller) { if (o) o.operation2(name, options, callback, controller); else - callback && callback(new Error('Schema {0}/{1} not found.'.format(schema[0], schema[1]))); + callback && callback(new Error('Schema "{0}" not found.'.format(getSchemaName(schema)))); return !!o; }; @@ -16844,6 +16844,10 @@ function parseComponent(body, filename) { return response; } +function getSchemaName(schema) { + return schema[0] === 'default' ? schema[1] : schema[0] + '/' + schema[1]; +} + // Default action for workflow routing function controller_json_workflow(id) { var self = this; @@ -16852,6 +16856,12 @@ function controller_json_workflow(id) { if (w instanceof Object) { if (!w.type) { var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); + + if (!schema) { + self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema))); + return; + } + if (schema.meta[w.id] !== undefined) { w.type = '$' + w.id; } else if (schema.meta['workflow#' + w.id] !== undefined) { @@ -16888,6 +16898,12 @@ function controller_json_workflow_multiple(id) { if (w instanceof Object) { if (!w.type) { var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); + + if (!schema) { + self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema))); + return; + } + var op = []; for (var i = 0; i < w.id.length; i++) { var id = w.id[i]; From d2318f904fd8987675c0712bd892640f1b7ee8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 27 Jun 2018 13:15:29 +0200 Subject: [PATCH 0442/1669] Improved `controller.callback()`. --- index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/index.js b/index.js index afab3084a..bb33e5bb7 100755 --- a/index.js +++ b/index.js @@ -12319,6 +12319,9 @@ Controller.prototype.callback = function(view) { var self = this; return function(err, data) { + if (self.res && self.res.success) + return; + var is = err instanceof framework_builders.ErrorBuilder; // NoSQL embedded database From 54031248baa3a12737e033a74205d4b4cd2a426e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 28 Jun 2018 22:22:08 +0200 Subject: [PATCH 0443/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f51e5d9a6..cacf8b0fc 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-83", + "version": "3.0.0-84", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 7a9e09445c26e50b51fbed3f1411e9990c2bc086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 08:58:13 +0200 Subject: [PATCH 0444/1669] Added `TABLE.find2()`. --- nosql.js | 228 ++++++++++++++++++++++++++++++++++++++++++++++++- nosqlstream.js | 2 +- 2 files changed, 227 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index 6d264e3b1..91a5708b1 100755 --- a/nosql.js +++ b/nosql.js @@ -70,6 +70,7 @@ const BINARYREADDATA = { start: BINARY_HEADER_LENGTH }; const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' }; const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; +const TABLERECORD = { '+': 1, '-': 1, '*': 1 }; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); @@ -278,7 +279,7 @@ exports.worker = function() { return send(this, 'find', view).builder = builder; }; - DP.find2 = function(builder) { + TP.find2 = DP.find2 = function(builder) { if (builder) builder.db = this; else @@ -508,7 +509,7 @@ exports.worker = function() { return send(this, 'find').builder = builder; }; - TP.listing = function(view, builder) { + TP.listing = function(builder) { if (builder) builder.db = this; else @@ -585,6 +586,7 @@ function Table(name, filename) { t.name = name; t.$name = '$' + name; t.pending_reader = []; + t.pending_reader2 = []; t.pending_update = []; t.pending_append = []; t.pending_reader = []; @@ -5991,6 +5993,17 @@ TP.find = function(builder) { return builder; }; +TP.find2 = function(builder) { + var self = this; + if (builder) + builder.db = self; + else + builder = new DatabaseBuilder(self); + self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 11); + return builder; +}; + TP.stream = function(fn, repository, callback) { var self = this; self.readonly && self.throwReadonly(); @@ -6168,6 +6181,11 @@ TP.next = function(type) { return; } + if (this.step !== 11 && this.pending_reader2.length) { + this.$reader3(); + return; + } + if (this.step !== 10 && this.pending_streamer.length) { this.$streamer(); return; @@ -6413,6 +6431,212 @@ TP.$reader = function() { return self; }; +TP.$reader3 = function() { + + var self = this; + + self.step = 11; + + if (!self.pending_reader2.length) { + self.next(0); + return self; + } + + self.$reading = true; + + var filter = self.pending_reader2.splice(0); + var length = filter.length; + var first = true; + var indexer = 0; + var keys = {}; + var keyscount = 0; + + for (var i = 0; i < length; i++) { + var fil = filter[i]; + if (!fil.builder.$options.first || fil.builder.$options.sort) + first = false; + if (fil.builder.$keys == null && keys) + keys = null; + else if (keys) { + if (fil.builder.$options.fields) { + for (var j = 0; j < fil.builder.$keys.length; j++) { + keyscount++; + keys[fil.builder.$keys[j]] = 1; + } + } else + keys = null; + } + + fil.scalarcount = 0; + fil.compare = fil.builder.compile(); + fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + } + + if (first && length > 1) + first = false; + + var fs = new NoSQLStream(self.filename); + var data = {}; + + fs.divider = '\n'; + data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + + fs.ondocuments = function() { + + var lines = fs.docs.split(fs.divider); + var val; + + for (var j = 0; j < lines.length; j++) { + + data.line = lines[j].split('|'); + + if (!TABLERECORD[data.line[0]]) + continue; + + data.index = indexer++; + indexer++; + + var obj = self.parseData(data); + + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + item.filter.index = indexer; + + var output = item.compare(obj, item.filter, indexer); + if (!output) + continue; + + item.count++; + + if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) + continue; + + item.counter++; + + if (item.type) + continue; + + switch (builder.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[builder.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[builder.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'max': + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'avg': + val = output[builder.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (builder.$inlinesort) + nosqlinlinesorter(item, builder, output); + else if (item.response) + item.response.push(output); + else + item.response = [output]; + break; + } + + if (first) + return false; + } + } + }; + + fs.$callback = function() { + + for (var i = 0; i < length; i++) { + var item = filter[i]; + var builder = item.builder; + var output; + + if (builder.$options.scalar || !builder.$options.sort) { + + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + continue; + } + + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + + builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); + builder.done(); + } + + self.$reading = false; + fs = null; + self.next(0); + }; + + fs.openreadreverse(); + return self; +}; + TP.$update = function() { var self = this; diff --git a/nosqlstream.js b/nosqlstream.js index 5d31a752f..1762f0ddd 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -155,7 +155,7 @@ NoSQLStream.prototype.readhelpers = function() { continue; } - self.docs += (self.docs ? self.divider : '') + tmp; + self.docs += (self.docs ? self.divider : '') + tmp.trim(); self.docscount++; if (self.docscount >= BUFFERDOCS) { From 00289df93115179207890092a52665e976b63caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 10:58:33 +0200 Subject: [PATCH 0445/1669] Removed NoSQL `views`. --- nosql.js | 554 ++++++++----------------------------------------- nosqlworker.js | 19 +- 2 files changed, 94 insertions(+), 479 deletions(-) diff --git a/nosql.js b/nosql.js index 91a5708b1..a1e8b723f 100755 --- a/nosql.js +++ b/nosql.js @@ -257,26 +257,26 @@ exports.worker = function() { var CP = Counter.prototype; var SP = Storage.prototype; - DP.once = DP.on = DP.emit = DP.removeListener = DP.removeAllListeners = CP.on = CP.once = CP.emit = CP.removeListener = CP.removeAllListeners = function() { + TP.once = TP.on = TP.emit = TP.removeListener = TP.removeAllListeners = DP.once = DP.on = DP.emit = DP.removeListener = DP.removeAllListeners = CP.on = CP.once = CP.emit = CP.removeListener = CP.removeAllListeners = function() { PRINTLN('ERROR --> NoSQL events are not supported in fork mode.'); }; - DP.listing = function(view, builder) { + TP.listing = DP.listing = function(builder) { if (builder) builder.db = this; else builder = new DatabaseBuilder(this); builder.$options.listing = true; builder.$take = builder.$options.take = 100; - return send(this, 'find', view).builder = builder; + return send(this, 'find').builder = builder; }; - DP.find = function(view, builder) { + TP.find = DP.find = function(builder) { if (builder) builder.db = this; else builder = new DatabaseBuilder(this); - return send(this, 'find', view).builder = builder; + return send(this, 'find').builder = builder; }; TP.find2 = DP.find2 = function(builder) { @@ -287,19 +287,19 @@ exports.worker = function() { return send(this, 'find2').builder = builder; }; - DP.top = function(max, view) { + TP.top = DP.top = function(max) { var builder = new DatabaseBuilder(this); builder.take(max); - return send(this, 'find', view).builder = builder; + return send(this, 'find').builder = builder; }; - DP.one = function(view) { + TP.one = DP.one = function() { var builder = new DatabaseBuilder(this); builder.first(); - return send(this, 'one', view).builder = builder; + return send(this, 'one').builder = builder; }; - DP.insert = function(doc, unique) { + TP.insert = DP.insert = function(doc, unique) { var self = this; var builder; @@ -330,22 +330,20 @@ exports.worker = function() { return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); }; - DP.count = function(view) { + TP.count = DP.count = function() { var builder = new DatabaseBuilder(this); - return send(this, 'count', view).builder = builder; + return send(this, 'count').builder = builder; }; - DP.view = function(name) { - var builder = new DatabaseBuilder(this); - builder.id('$view_' + name); - return send(this, 'view', name).builder = builder; + DP.view = function() { + throw new Error('NoSQL Views are not supported.'); }; - DP.update = function(doc, insert) { + TP.update = DP.update = function(doc, insert) { return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); }; - DP.modify = function(doc, insert) { + TP.modify = DP.modify = function(doc, insert) { return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); }; @@ -362,7 +360,6 @@ exports.worker = function() { }; DP.refresh = function() { - notify(this, 'refresh'); return this; }; @@ -381,16 +378,16 @@ exports.worker = function() { return this; }; - DP.ready = function(callback) { - send(this, 'ready').callback = callback; + TP.ready = DP.ready = function(callback) { + callback && callback(); return this; }; - DP.remove = function(filename) { + TP.remove = DP.remove = function(filename) { return send(this, 'remove', filename).builder = new DatabaseBuilder(this); }; - DP.stream = function(fn, repository, callback) { + TP.stream = DP.stream = function(fn, repository, callback) { if (typeof(repository) === 'function') { callback = repository; @@ -500,80 +497,6 @@ exports.worker = function() { send(this.db, 'storage.clear', beg, end).callback = callback; return this; }; - - TP.find = function(builder) { - if (builder) - builder.db = this; - else - builder = new DatabaseBuilder(this); - return send(this, 'find').builder = builder; - }; - - TP.listing = function(builder) { - if (builder) - builder.db = this; - else - builder = new DatabaseBuilder(this); - builder.$options.listing = true; - builder.$take = builder.$options.take = 100; - return send(this, 'find', view).builder = builder; - }; - - TP.top = function(max) { - var builder = new DatabaseBuilder(this); - builder.take(max); - return send(this, 'find').builder = builder; - }; - - TP.one = function() { - var builder = new DatabaseBuilder(this); - builder.first(); - return send(this, 'one').builder = builder; - }; - - TP.insert = function(doc, unique) { - - var self = this; - var builder; - - if (unique) { - builder = self.one(); - - var callback; - - builder.callback(function(err, d) { - if (d) - callback && callback(null, 0); - else { - var tmp = self.insert(doc); - tmp.callback(callback); - tmp.$options.log = builder.$options.log; - } - }); - - builder.callback = function(fn) { - callback = fn; - return builder; - }; - - return builder; - } - - return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); - }; - - TP.count = function() { - var builder = new DatabaseBuilder(this); - return send(this, 'count').builder = builder; - }; - - TP.update = function(doc, insert) { - return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); - }; - - TP.modify = function(doc, insert) { - return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); - }; }; function Table(name, filename) { @@ -672,16 +595,13 @@ function Database(name, filename, readonly) { self.pending_append = []; self.pending_reader = []; self.pending_remove = []; - self.pending_reader_view = readonly ? EMPTYARRAY : []; self.pending_reader2 = []; self.pending_streamer = []; self.pending_clean = []; self.pending_clear = []; self.pending_locks = []; - self.views = null; self.step = 0; self.pending_drops = false; - self.pending_views = false; self.pending_reindex = false; self.binary = self.readonly || readonly === true ? null : new Binary(self, self.directory + '/' + self.name + '-binary/'); self.storage = self.readonly || readonly === true ? null : new Storage(self, self.directory + '/' + self.name + '-storage/'); @@ -771,9 +691,7 @@ exports.table = function(name, filename) { return new Table(name, filename); }; -exports.memory = exports.inmemory = function(name, view) { - if (view) - name += '#' + view; +exports.memory = exports.inmemory = function(name) { return INMEMORY[name] = true; }; @@ -1126,58 +1044,38 @@ DP.remove = function(filename) { return builder; }; -DP.listing = function(view, builder) { +DP.listing = function(builder) { var self = this; - if (builder) builder.db = self; else builder = new DatabaseBuilder(self); - builder.$options.listing = true; builder.$take = builder.$options.take = 100; - - if (view) { - self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); - setImmediate(next_operation, self, 6); - } else { - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); - setImmediate(next_operation, self, 4); - } - + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); return builder; }; -DP.find = function(view, builder) { +DP.find = function(builder) { var self = this; - if (builder) builder.db = self; else builder = new DatabaseBuilder(self); - - if (view) { - self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); - setImmediate(next_operation, self, 6); - } else { - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); - setImmediate(next_operation, self, 4); - } - + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); return builder; }; DP.find2 = function(builder) { var self = this; - if (builder) builder.db = self; else builder = new DatabaseBuilder(self); - if (self.readonly) - return self.find(null, builder); - + return self.find(builder); self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); setImmediate(next_operation, self, 11); return builder; @@ -1200,67 +1098,38 @@ DP.throwReadonly = function() { throw new Error('Database "{0}" is readonly.'.format(this.name)); }; -DP.scalar = function(type, field, view) { - return this.find(view).scalar(type, field); +DP.scalar = function(type, field) { + return this.find().scalar(type, field); }; -DP.count = function(view) { +DP.count = function() { var self = this; var builder = new DatabaseBuilder(self); - - if (view) { - self.pending_reader_view.push({ builder: builder, count: 0, view: view, type: 1 }); - setImmediate(next_operation, self, 6); - } else { - self.pending_reader.push({ builder: builder, count: 0, type: 1 }); - setImmediate(next_operation, self, 4); - } - + self.pending_reader.push({ builder: builder, count: 0, type: 1 }); + setImmediate(next_operation, self, 4); return builder; }; -DP.one = function(view) { +DP.one = function() { var self = this; var builder = new DatabaseBuilder(self); - builder.first(); - - if (view) { - self.pending_reader_view.push({ builder: builder, count: 0, view: view }); - setImmediate(next_operation, self, 6); - } else { - self.pending_reader.push({ builder: builder, count: 0 }); - setImmediate(next_operation, self, 4); - } - + self.pending_reader.push({ builder: builder, count: 0 }); + setImmediate(next_operation, self, 4); return builder; }; -DP.top = function(max, view) { +DP.top = function(max) { var self = this; var builder = new DatabaseBuilder(self); builder.take(max); - - if (view) { - self.pending_reader_view.push({ builder: builder, count: 0, counter: 0, view: view }); - setImmediate(next_operation, self, 6); - } else { - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); - setImmediate(next_operation, self, 4); - } - + self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + setImmediate(next_operation, self, 4); return builder; }; -DP.view = function(name) { - var builder = new DatabaseBuilder(this); - if (!this.views) - this.views = {}; - this.views[name] = {}; - this.views[name] = builder; - this.views[name].$filename = this.filename.replace(/\.nosql/, '#' + name + '.nosql'); - builder.id('$view_' + name); - return builder; +DP.view = function() { + throw new Error('NoSQL Views are not supported.'); }; // 1 append @@ -1355,19 +1224,6 @@ DP.next = function(type) { this.$streamer(); return; } - - if (this.step !== 6 && this.pending_reader_view.length) { - this.$readerview(); - return; - } - - if (this.step !== 5 && this.pending_views) { - if (INMEMORY[this.name]) - this.$views_inmemory(); - else - this.$views(); - return; - } } if (this.step !== type) { @@ -1376,39 +1232,28 @@ DP.next = function(type) { } }; -DP.refresh = function() { - if (this.views) { - this.pending_views = true; - setImmediate(next_operation, this, 5); - } - return this; -}; - // ====================================================================== // FILE OPERATIONS // ====================================================================== // InMemory saving -DP.$save = function(view) { +DP.$save = function() { var self = this; - setTimeout2('nosql.' + self.name + '.' + view, function() { - var data = self.inmemory[view] || EMPTYARRAY; + setTimeout2('nosql.' + self.name, function() { + var data = self.inmemory['#'] || EMPTYARRAY; var builder = []; for (var i = 0, length = data.length; i < length; i++) builder.push(JSON.stringify(data[i]).replace(REGBOOL, JSONBOOL)); - - var filename = self.filename; - if (view !== '#') - filename = filename.replace(/\.nosql/, '#' + view + '.nosql'); - - Fs.writeFile(filename, builder.join(NEWLINE) + NEWLINE, F.errorcallback); + Fs.writeFile(self.filename, builder.join(NEWLINE) + NEWLINE, F.errorcallback); }, 50, 100); return self; }; -DP.$inmemory = function(view, callback) { +DP.$inmemory = function(callback) { var self = this; + var view = '#'; + self.readonly && self.throwReadonly(); // Last usage @@ -1501,7 +1346,6 @@ function next_append(self) { self.$writting = false; self.next(0); self.$events.change && self.emit('change', 'insert'); - self.views && setImmediate(views_refresh, self); } DP.$append_inmemory = function() { @@ -1515,7 +1359,7 @@ DP.$append_inmemory = function() { var items = self.pending_append.splice(0); - return self.$inmemory('#', function() { + return self.$inmemory(function() { for (var i = 0, length = items.length; i < length; i++) { self.inmemory['#'].push(JSON.parse(items[i].doc, jsonparser)); @@ -1524,7 +1368,7 @@ DP.$append_inmemory = function() { callback && callback(null, 1); } - self.$save('#'); + self.$save(); setImmediate(next_append, self); }); }; @@ -1681,21 +1525,13 @@ DP.$update = function() { fs = null; self.$writting = false; self.next(0); - - if (change) { - self.$events.change && self.emit('change', 'update'); - self.views && setImmediate(views_refresh, self); - } + change && self.$events.change && self.emit('change', 'update'); }; fs.openupdate(); return self; }; -function views_refresh(self) { - self.refresh(); -} - DP.$update_inmemory = function() { var self = this; @@ -1716,7 +1552,7 @@ DP.$update_inmemory = function() { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - return self.$inmemory('#', function() { + return self.$inmemory(function() { var data = self.inmemory['#']; @@ -1758,7 +1594,7 @@ DP.$update_inmemory = function() { } } - self.$save('#'); + self.$save(); for (var i = 0; i < length; i++) { var item = filter[i]; @@ -1777,12 +1613,7 @@ DP.$update_inmemory = function() { setImmediate(function() { self.next(0); - - if (change) { - self.$events.change && self.emit('change', 'update'); - self.views && setImmediate(views_refresh, self); - } - + change && self.$events.change && self.emit('change', 'update'); }); }); }; @@ -1799,7 +1630,7 @@ DP.$reader = function() { var list = self.pending_reader.splice(0); if (INMEMORY[self.name]) { - self.$reader2_inmemory('#', list, () => self.next(0)); + self.$reader2_inmemory(list, () => self.next(0)); } else { self.$reading = true; self.$reader2(self.filename, list, function() { @@ -1811,53 +1642,9 @@ DP.$reader = function() { return self; }; -DP.$readerview = function() { - - var self = this; - self.step = 6; - - if (!self.pending_reader_view.length) { - self.next(0); - return self; - } - - var group = {}; - var skip = true; - - for (var i = 0, length = self.pending_reader_view.length; i < length; i++) { - var item = self.pending_reader_view[i]; - var name = INMEMORY[self.name] || INMEMORY[self.name + '#' + item.view] ? item.view : self.views[item.view].$filename; - - skip = false; - - if (group[name]) - group[name].push(item); - else - group[name] = [item]; - } - - self.pending_reader_view = []; - - if (skip) { - self.next(0); - return self; - } - - Object.keys(group).wait(function(item, next) { - if (INMEMORY[self.name] || INMEMORY[self.name + '#' + item]) - self.$reader2_inmemory(item, group[item], next); - else - self.$reader2(item, group[item], next); - }, () => self.next(0), 5); - - return self; -}; - function listing(builder, item) { - var skip = builder.$options.skip || 0; var take = builder.$options.take || 0; - return { page: ((skip / take) + 1), pages: item.count ? Math.ceil(item.count / take) : 0, limit: take, count: item.count, items: item.response || [] }; } @@ -2336,11 +2123,12 @@ function nosqlresort(arr, builder, doc) { } } -DP.$reader2_inmemory = function(name, items, callback) { +DP.$reader2_inmemory = function(items, callback) { var self = this; var filter = items; var length = filter.length; + var name = '#'; for (var i = 0; i < length; i++) { var fil = filter[i]; @@ -2348,7 +2136,7 @@ DP.$reader2_inmemory = function(name, items, callback) { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - return self.$inmemory(name, function() { + return self.$inmemory(function() { var data = self.inmemory[name]; var val; @@ -2483,180 +2271,6 @@ DP.$reader2_inmemory = function(name, items, callback) { }); }; -DP.$views = function() { - - var self = this; - if (!self.views) { - self.next(0); - return; - } - - self.step = 5; - - if (!self.pending_views) { - self.next(0); - return; - } - - self.pending_views = false; - - var views = Object.keys(self.views); - var length = views.length; - - if (!length) { - self.next(0); - return self; - } - - var response = []; - - for (var i = 0; i < length; i++) { - var builder = self.views[views[i]]; - response.push({ response: [], name: views[i], compare: builder.compile(), filter: { repository: builder.$repository, options: builder.$options, arg: builder.$params, fn: builder.$functions }, builder: builder, count: 0, counter: 0, repository: {} }); - } - - var fs = new NoSQLStream(self.filename); - var indexer = 0; - - self.$reading = true; - - fs.ondocuments = function() { - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - for (var i = 0; i < docs.length; i++) { - var json = docs[i]; - for (var j = 0; j < length; j++) { - var item = self.views[views[j]]; - var res = response[j]; - res.filter.index = indexer; - var output = res.compare(json, res.filter, indexer); - if (!output) - continue; - res.count++; - if (!res.filter.options.sort && ((res.filter.options.skip && res.filter.options.skip >= res.count) || (res.filter.options.take && res.filter.options.take <= res.counter))) - continue; - res.counter++; - !item.type && res.response.push(output); - } - } - }; - - fs.$callback = function() { - response.wait(function(item, next) { - - var builder = item.builder; - - if (builder.$options.sort) { - if (builder.$options.sort.name) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - else if (builder.$options.sort === EMPTYOBJECT) - item.response.random(); - else - item.response.sort(builder.$options.sort); - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - var filename = builder.$filename; - Fs.unlink(filename, function() { - item.response.limit(20, function(items, next) { - var builder = []; - for (var i = 0, length = items.length; i < length; i++) - builder.push(JSON.stringify(items[i]).replace(REGBOOL, JSONBOOL)); - Fs.appendFile(filename, builder.join(NEWLINE) + NEWLINE, next); - }, function() { - // clears in-memory - self.inmemory[item.name] = undefined; - next(); - }); - }); - }, function() { - self.$reading = false; - self.next(0); - }, 5); - }; - - fs.openread(); -}; - -DP.$views_inmemory = function() { - - var self = this; - self.step = 5; - - if (!self.pending_views) { - self.next(0); - return self; - } - - self.pending_views = false; - - var views = Object.keys(self.views); - var length = views.length; - - if (!length) { - self.next(0); - return self; - } - - var response = []; - - for (var i = 0; i < length; i++) { - var builder = self.views[views[i]]; - response.push({ response: [], name: views[i], compare: builder.compile(), filter: { repository: builder.$repository, options: builder.$options, arg: builder.$params, fn: builder.$functions }, builder: builder, count: 0, counter: 0, repository: {} }); - } - - return self.$inmemory('#', function() { - var data = self.inmemory['#']; - - for (var j = 0, jl = data.length; j < jl; j++) { - var json = data[j]; - for (var i = 0; i < length; i++) { - var item = self.views[views[i]]; - var res = response[i]; - res.filter.index = j; - var output = res.compare(json, res.filter, j); - if (!output) - continue; - res.count++; - if (!item.$options.sort && ((item.$options.skip && item.$options.skip >= res.count) || (item.$options.take && item.$options.take <= res.counter))) - continue; - res.counter++; - !item.type && res.response.push(output); - } - } - - for (var j = 0, jl = response.length; j < jl; j++) { - - var item = response[j]; - var builder = item.builder; - - if (builder.$options.sort) { - if (builder.$options.sort.name) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - else if (builder.$options.sort === EMPTYOBJECT) - item.response.random(); - else - item.response.sort(builder.$options.sort); - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - self.inmemory[item.name] = item.response; - self.$save(item.name); - } - - self.next(0); - }); -}; - DP.$remove = function() { var self = this; @@ -2736,11 +2350,7 @@ DP.$remove = function() { fs = null; self.$writting = false; self.next(0); - - if (change) { - self.views && setImmediate(views_refresh, self); - self.$events.change && self.emit('change', 'remove'); - } + change && self.$events.change && self.emit('change', 'remove'); }; fs.openupdate(); @@ -2760,7 +2370,6 @@ DP.$clear = function() { Fs.unlink(self.filename, function() { for (var i = 0; i < filter.length; i++) filter[i](); - self.views && setImmediate(views_refresh, self); self.$events.change && self.emit('change', 'clear'); self.next(0); }); @@ -2848,7 +2457,7 @@ DP.$remove_inmemory = function() { fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; } - return self.$inmemory('#', function() { + return self.$inmemory(function() { var data = self.inmemory['#']; var arr = []; @@ -2880,7 +2489,7 @@ DP.$remove_inmemory = function() { if (change) { self.inmemory['#'] = arr; - self.$save('#'); + self.$save(); } for (var i = 0; i < length; i++) { @@ -2890,10 +2499,7 @@ DP.$remove_inmemory = function() { } self.next(0); - if (change) { - self.views && setImmediate(views_refresh, self); - self.$events.change && self.emit('change', 'remove'); - } + change && self.$events.change && self.emit('change', 'remove'); }); }; @@ -2908,16 +2514,12 @@ DP.$clear_inmemory = function() { } var filter = self.pending_clear.splice(0); - return self.$inmemory('#', function() { - + return self.$inmemory(function() { self.inmemory['#'] = []; - self.$save('#'); - + self.$save(); for (var i = 0; i < length; i++) filter[i](null); - self.next(0); - self.views && setImmediate(views_refresh, self); }); }; @@ -2932,7 +2534,6 @@ DP.$drop = function() { self.pending_drops = false; var remove = [self.filename, self.filenameTemp]; - self.views && Object.keys(self.views).forEach(key => remove.push(self.views[key].$filename)); try { Fs.readdirSync(self.binary.directory).forEach(function(filename) { @@ -3053,7 +2654,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { join.count = unique.length; for (var i = 0; i < unique.length; i++) { (function(val) { - var builder = db.scalar(join.scalartype, join.scalarfield, join.view).callback(function(err, response) { + var builder = db.scalar(join.scalartype, join.scalarfield).callback(function(err, response) { join.items.push({ id: val, response: response }); join.count--; if (join.count === 0) { @@ -3089,7 +2690,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { if (isTable) db.find(join.builder).in(join.a, unique); else - db.find(join.view, join.builder).in(join.a, unique); + db.find(join.builder).in(join.a, unique); } else { join.items = join.builder.$options.first ? null : []; next(); @@ -3163,13 +2764,13 @@ function findItems(items, field, value) { return arr; } -DatabaseBuilder.prototype.join = function(field, name, view) { +DatabaseBuilder.prototype.join = function(field, name) { var self = this; if (!self.$join) self.$join = {}; - var key = name + '.' + (view || '') + '.' + field; + var key = name + '.' + field; var join = self.$join[key]; if (join) return join; @@ -3177,7 +2778,6 @@ DatabaseBuilder.prototype.join = function(field, name, view) { var item = self.$join[key] = {}; item.field = field; item.name = name; - item.view = view; item.builder = join = new DatabaseBuilder(self.db); join.on = function(a, b) { @@ -5885,6 +5485,18 @@ Backuper.prototype.flush = function() { return self; }; +TP.ready = function(fn) { + var self = this; + if (self.ready) + fn.call(self); + else { + setTimeout(function(self, fn) { + self.ready(fn); + }, 100, self, fn); + } + return self; +}; + TP.insert = function(doc, unique) { var self = this; @@ -6922,11 +6534,7 @@ TP.$remove = function() { fs = null; self.$writting = false; self.next(0); - - if (change) { - self.views && setImmediate(views_refresh, self); - self.$events.change && self.emit('change', 'remove'); - } + change && self.$events.change && self.emit('change', 'remove'); }; fs.openupdate(); diff --git a/nosqlworker.js b/nosqlworker.js index 8b6c4d59d..38e75af36 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -68,7 +68,7 @@ process.on('message', function(msg) { switch (msg.TYPE) { case 'find': - db.find(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, count, repository) { + db.find().parse(msg.data).callback(function(err, response, count, repository) { RESFIND.err = err; RESFIND.response = response; RESFIND.count = count; @@ -87,8 +87,18 @@ process.on('message', function(msg) { process.send(RESFIND); }); break; + case 'top': + db.top().parse(msg.data).callback(function(err, response, count, repository) { + RESFIND.err = err; + RESFIND.response = response; + RESFIND.count = count; + RESFIND.repository = repository; + RESFIND.id = msg.id; + process.send(RESFIND); + }); + break; case 'one': - db.one(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, count, repository) { + db.one().parse(msg.data).callback(function(err, response, count, repository) { RESFIND.err = err; RESFIND.response = response; RESFIND.count = count; @@ -124,7 +134,7 @@ process.on('message', function(msg) { }); break; case 'count': - db.count(msg.arg ? msg.arg[0] : undefined).parse(msg.data).callback(function(err, response, count, repository) { + db.count().parse(msg.data).callback(function(err, response, count, repository) { RESCOUNT.err = err; RESCOUNT.response = response; RESCOUNT.count = count; @@ -142,9 +152,6 @@ process.on('message', function(msg) { process.send(RESREMOVE); }); break; - case 'view': - db.view(msg.arg[0]).parse(msg.data); - break; case 'backup': db.backup(msg.arg[0], function(err, response) { RESBACKUP.id = msg.id; From c9bd9ff006b333a554745899767bf85cddd4721b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 10:58:47 +0200 Subject: [PATCH 0446/1669] Updated `crypto` algorithm. --- index.js | 9 +++++-- utils.js | 71 +++++++++++++++++--------------------------------------- 2 files changed, 28 insertions(+), 52 deletions(-) diff --git a/index.js b/index.js index bb33e5bb7..57b9a308d 100755 --- a/index.js +++ b/index.js @@ -645,7 +645,9 @@ function Framework() { name: 'Total.js', version: '1.0.0', author: '', - secret: Os.hostname() + '-' + Os.platform() + '-' + Os.arch(), + secret: (Os.hostname() + '-' + Os.platform() + '-' + Os.arch()).crc32(true), + 'default-crypto': 'aes-128-ofb', + 'default-crypto-key': 16, 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -995,6 +997,7 @@ F.prototypes = function(fn) { proto.UrlBuilder = framework_builders.UrlBuilder.prototype; proto.WebSocket = WebSocket.prototype; proto.WebSocketClient = WebSocketClient.prototype; + proto.Session = framework_session.prototype; fn.call(proto, proto); return F; }; @@ -16964,4 +16967,6 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file +global.EMPTYCONTROLLER = EMPTYCONTROLLER; + +const framework_session = require('./session'); diff --git a/utils.js b/utils.js index 11dda857d..eb747e169 100755 --- a/utils.js +++ b/utils.js @@ -62,8 +62,6 @@ const regexpINTEGER = /(^-|\s-)?[0-9]+/g; const regexpFLOAT = /(^-|\s-)?[0-9.,]+/g; const regexpALPHA = /^[A-Za-z0-9]+$/; const regexpSEARCH = /[^a-zA-Zá-žÁ-Ž\d\s:]/g; -const regexpDECRYPT = /-|_/g; -const regexpENCRYPT = /\/|\+/g; const regexpUNICODE = /\\u([\d\w]{4})/gi; const regexpTERMINAL = /[\w\S]+/g; const regexpY = /y/g; @@ -3764,75 +3762,48 @@ String.prototype.toKeywords = String.prototype.keywords = function(forSearch, al return exports.keywords(this, forSearch, alternative, max_count, max_length, min_length); }; -String.prototype.encrypt = function(key, isUnique) { - var str = '0' + this; - var data_count = str.length; - var key_count = key.length; - var random = isUnique ? exports.random(120) + 40 : 65; - var count = data_count + (random % key_count); - var values = []; - var index = 0; +const CRYPTO = {}; - values[0] = String.fromCharCode(random); - var counter = this.length + key.length; +String.prototype.encrypt = function(key, isUnique) { - for (var i = count - 1; i > 0; i--) { - index = str.charCodeAt(i % data_count); - values[i] = String.fromCharCode(index ^ (key.charCodeAt(i % key_count) ^ random)); + if (!CRYPTO[key]) { + var size = F.config['default-crypto-key']; + CRYPTO[key] = Buffer.alloc(size, key.length > size ? key.substring(0, size) : key, 'utf8'); } - var hash = exports.createBuffer(counter + '=' + values.join(''), ENCODING).toString('base64').replace(regexpENCRYPT, text => text === '+' ? '_' : '-'); - index = hash.indexOf('='); - return index > 0 ? hash.substring(0, index) : hash; + var cipher = Crypto.createCipheriv(F.config['default-crypto'], CRYPTO[key], CRYPTO[key]); + var str = this.toString(); + var plus = isUnique ? '_' + GUID(10) : ''; + return cipher.update(plus + str.hash() + '#' + str, 'utf8', 'hex') + cipher.final('hex'); }; String.prototype.decrypt = function(key) { - var values = this.replace(regexpDECRYPT, text => text === '-' ? '/' : '+'); - var mod = values.length % 4; - - if (mod) { - for (var i = 0; i < mod; i++) - values += '='; + if (!CRYPTO[key]) { + var size = F.config['default-crypto-key']; + CRYPTO[key] = Buffer.alloc(size, key.length > size ? key.substring(0, size) : key, 'utf8'); } - values = exports.createBuffer(values, 'base64').toString(ENCODING); + var decipher = Crypto.createDecipheriv(F.config['default-crypto'], CRYPTO[key], CRYPTO[key]); + var result = decipher.update(this, 'hex') + decipher.final(); - var index = values.indexOf('='); - if (index === -1) - return null; - - var counter = +values.substring(0, index); - if (isNaN(counter)) + var beg = result.indexOf('#'); + if (beg === -1) return null; - - values = values.substring(index + 1); - - var count = values.length; - var random = values.charCodeAt(0); - - var key_count = key.length; - var data_count = count - (random % key_count); - var decrypt_data = []; - - for (var i = data_count - 1; i > 0; i--) { - index = values.charCodeAt(i) ^ (random ^ key.charCodeAt(i % key_count)); - decrypt_data[i] = String.fromCharCode(index); - } - - var val = decrypt_data.join(''); - return counter !== val.length + key.length ? null : val; + var o = result.substring(beg + 1); + var sum = result.substring(0, beg); + if (sum[0] === '_') + sum = sum.substring(10); + return +sum == o.hash() ? o : null; }; String.prototype.base64ToFile = function(filename, callback) { var self = this; - var index = self.indexOf(','); if (index === -1) index = 0; else index++; - Fs.writeFile(filename, self.substring(index), 'base64', callback || exports.noop); return this; }; From 2e88d5029e8a3a92079f1ce0826ca25822df58a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 10:58:51 +0200 Subject: [PATCH 0447/1669] New changes. --- changes.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes.txt b/changes.txt index 01baf3997..e10f79081 100755 --- a/changes.txt +++ b/changes.txt @@ -54,6 +54,8 @@ - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: config `security.txt` for auto-generating security.txt content (more in docs) - added: config `default-proxy` for default web proxy server +- added: config `default-crypto` for default crypto algorithm +- added: config `default-crypto-key` for default length of key crypto algorithm - added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `GUID()` a global alias for `U.GUID()` - added: `VIEW()` a global alias for `F.view()` @@ -106,6 +108,7 @@ - updated: `CORS()` tries to join multiple same preferences to one - updated: `U.keywords()` for Chinese/Japan characters - updated: `@{import()}` by adding `manifest` value linked to `/manifest.json` +- updated: internal crypto algorithm to `aes-128-ofb` - fixed: mail attachments - fixed: mail `message.manually()` From 541c2ef3fc46244389e9990da800129139ede89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 10:58:58 +0200 Subject: [PATCH 0448/1669] Updated unit-test. --- test/controllers/default.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/controllers/default.js b/test/controllers/default.js index 847a63522..06b91921a 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -109,11 +109,11 @@ exports.install = function() { self.plain('408'); }); - assert.ok(F.encrypt('123456', 'key', false) === 'MjM9QR8HExlaHQJQBxcGAEoaFQoGGgAW', 'F.encrypt(string)'); - assert.ok(F.decrypt('MjM9QR8HExlaHQJQBxcGAEoaFQoGGgAW', 'key', false) === '123456', 'F.decrypt(string)'); + assert.ok(F.encrypt('123456', 'key', false) === 'a79aad6f9b92c0d3f1beb3812179863d8e', 'F.encrypt(string)'); + assert.ok(F.decrypt('a79aad6f9b92c0d3f1beb3812179863d8e', 'key', false) === '123456', 'F.decrypt(string)'); - assert.ok(F.encrypt({ name: 'Peter' }, 'key', false) === 'MzM9QVUXTkwCThBbF3RXQRlYBkUFVRdOTAJOEFsXdFdBGQ', 'F.encrypt(object)'); - assert.ok(F.decrypt('MzM9QVUXTkwCThBbF3RXQRlYBkUFVRdOTAJOEFsXdFdBGQ', 'key').name === 'Peter', 'F.decrypt(object)'); + assert.ok(F.encrypt({ name: 'Peter' }, 'key', false) === 'bb99af6f9b93c1dff0beb3cb3124d365dd06c70c0a81ec1a3dce8a', 'F.encrypt(object)'); + assert.ok(F.decrypt('bb99af6f9b93c1dff0beb3cb3124d365dd06c70c0a81ec1a3dce8a', 'key').name === 'Peter', 'F.decrypt(object)'); assert.ok(SOURCE('main').hello() === 'world', 'source'); assert.ok(INCLUDE('main').hello() === 'world', 'source'); From f0db3f244b88813ddbf76206c79e9d93678cb126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 10:59:10 +0200 Subject: [PATCH 0449/1669] Added new part called `Session`. --- session.js | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 session.js diff --git a/session.js b/session.js new file mode 100644 index 000000000..eb309cc76 --- /dev/null +++ b/session.js @@ -0,0 +1,115 @@ +const Sessions = {}; +const Fs = require('fs'); + +global.SESSION = function(group) { + return Sessions[group] ? Sessions[group] : (Sessions[group] = new Session(group)); +}; + +function Session(group) { + var t = this; + t.data = {}; + t.group = group.crc32(true); + t.filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'framework_sessions_' + group.crc32(true) + '.jsoncache'); + t.load(); +} + +var SP = Session.prototype; + +SP.load = function() { + var self = this; + try { + self.data = Fs.readFileSync(self.filename).toString('utf8').parseJSON(true); + } catch (e) {} + return self; +}; + +SP.save = function() { + var self = this; + setTimeout2('session_' + self.group, function() { + Fs.writeFile(self.filename, JSON.stringify(self.data), F.error()); + }, 1000, 10); + return self; +}; + +SP.key = function() { + return this.group + 'X' + GUID(10); +}; + +SP.set = function(data, expire) { + + var self = this; + var id = self.key(); + var session = self.data[id]; + + if (!session) + session = self.data[id] = {}; + + session.data = data; + session.id = id; + session.expire = (expire || NOW.add('5 minutes')).getTime(); + + self.$events.set && self.emit('set', session); + self.save(); + return id; +}; + +SP.list = function() { + var self = this; + var arr = Object.keys(self.data); + var online = []; + for (var i = 0; i < arr.length; i++) + online.push(self.data[arr[i]]); + return online; +}; + +SP.count = function() { + return Object.keys(this.data).length; +}; + +SP.get = function(key) { + return this.data[key] ? this.data[key].data : null; +}; + +SP.meta = function(key) { + return this.data[key] ? this.data[key] : null; +}; + +SP.rem = function(key) { + var self = this; + if (self.data[key]) { + self.$events.remove && self.emit('remove', self.data[key]); + delete self.data[key]; + self.save(); + } + return self; +}; + +SP.clear = function() { + var self = this; + var arr = Object.keys(self.data); + var count = 0; + var time = NOW.getTime(); + for (var i = 0; i < arr.length; i++) { + var key = arr[key]; + var obj = self.data[key]; + if (obj.expire < time) { + self.$events.expire && self.emit('expire', obj); + delete self.data[key]; + count++; + } + } + count && self.save(); + return self; +}; + +ON('service', function(interval) { + if (interval % 5 === 0) { + var arr = Object.keys(Sessions); + for (var i = 0; i < arr.length; i++) + Sessions[arr[i]].clear(); + } +}); + +F.session = global.SESSION; +global.Session = Session; +exports.Session = Session; \ No newline at end of file From 6c0e4d1c075e47076677588c5708c85cae1d9925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 12:11:43 +0200 Subject: [PATCH 0450/1669] Fixed decrypt. --- utils.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils.js b/utils.js index eb747e169..5aa49ec0f 100755 --- a/utils.js +++ b/utils.js @@ -3784,8 +3784,14 @@ String.prototype.decrypt = function(key) { CRYPTO[key] = Buffer.alloc(size, key.length > size ? key.substring(0, size) : key, 'utf8'); } - var decipher = Crypto.createDecipheriv(F.config['default-crypto'], CRYPTO[key], CRYPTO[key]); - var result = decipher.update(this, 'hex') + decipher.final(); + var result; + + try { + var decipher = Crypto.createDecipheriv(F.config['default-crypto'], CRYPTO[key], CRYPTO[key]); + result = decipher.update(this, 'hex') + decipher.final(); + } catch (e) { + return null; + } var beg = result.indexOf('#'); if (beg === -1) @@ -3793,7 +3799,7 @@ String.prototype.decrypt = function(key) { var o = result.substring(beg + 1); var sum = result.substring(0, beg); if (sum[0] === '_') - sum = sum.substring(10); + sum = sum.substring(11); return +sum == o.hash() ? o : null; }; From a52b48b029241dd29dfc3fc058cad70ae0dfc188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 14:10:56 +0200 Subject: [PATCH 0451/1669] Improved NoSQL embedded. --- changes.txt | 1 + nosql.js | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/changes.txt b/changes.txt index e10f79081..adbd846c6 100755 --- a/changes.txt +++ b/changes.txt @@ -5,6 +5,7 @@ - added: (IMPORTANT) Total.js components support nested public files encoded in base64 - added: (IMPORTANT) NoSQL worker - added: (IMPORTANT) NoSQL embedded storage for smaller big data / IoT +- added: (IMPORTANT) Session helper object - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` diff --git a/nosql.js b/nosql.js index a1e8b723f..e7ed0d01b 100755 --- a/nosql.js +++ b/nosql.js @@ -262,7 +262,7 @@ exports.worker = function() { }; TP.listing = DP.listing = function(builder) { - if (builder) + if (builder instanceof DatabaseBuilder) builder.db = this; else builder = new DatabaseBuilder(this); @@ -272,7 +272,7 @@ exports.worker = function() { }; TP.find = DP.find = function(builder) { - if (builder) + if (builder instanceof DatabaseBuilder) builder.db = this; else builder = new DatabaseBuilder(this); @@ -280,7 +280,7 @@ exports.worker = function() { }; TP.find2 = DP.find2 = function(builder) { - if (builder) + if (builder instanceof DatabaseBuilder) builder.db = this; else builder = new DatabaseBuilder(this); @@ -616,6 +616,10 @@ function Database(name, filename, readonly) { const TP = Table.prototype; const DP = Database.prototype; +TP.view = DP.view = function(name) { + throw new Error('NoSQL Views are not supported in this version.'); +}; + TP.emit = DP.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { @@ -1059,7 +1063,7 @@ DP.listing = function(builder) { DP.find = function(builder) { var self = this; - if (builder) + if (builder instanceof DatabaseBuilder) builder.db = self; else builder = new DatabaseBuilder(self); @@ -1070,7 +1074,7 @@ DP.find = function(builder) { DP.find2 = function(builder) { var self = this; - if (builder) + if (builder instanceof DatabaseBuilder) builder.db = self; else builder = new DatabaseBuilder(self); @@ -1128,10 +1132,6 @@ DP.top = function(max) { return builder; }; -DP.view = function() { - throw new Error('NoSQL Views are not supported.'); -}; - // 1 append // 2 update // 3 remove @@ -3075,7 +3075,7 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { return self; }; -DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function() { +DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function(raw) { var obj = {}; obj.options = this.$options; obj.code = this.$code; @@ -3089,11 +3089,11 @@ DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = fun if (this.$repository) obj.repository = this.$repository; - return JSON.stringify(obj); + return raw ? CLONE(obj) : JSON.stringify(obj); }; DatabaseBuilder2.prototype.parse = DatabaseBuilder.prototype.parse = function(data) { - data = JSON.parse(data, jsonparser); + data = data instanceof String ? JSON.parse(data, jsonparser) : data; this.$options = data.options; this.$code = data.code; this.$params = data.params; From 557a00a4c840918eb8f241d5b93db82d00398f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 21:54:11 +0200 Subject: [PATCH 0452/1669] Updated encryption. --- utils.js | 90 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/utils.js b/utils.js index 5aa49ec0f..eef6f4fd1 100755 --- a/utils.js +++ b/utils.js @@ -3762,45 +3762,82 @@ String.prototype.toKeywords = String.prototype.keywords = function(forSearch, al return exports.keywords(this, forSearch, alternative, max_count, max_length, min_length); }; -const CRYPTO = {}; +function checksum(val) { + var sum = 0; + for (var i = 0; i < val.length; i++) + sum += val.charCodeAt(i); + return sum; +} String.prototype.encrypt = function(key, isUnique) { + var str = '0' + this; + var data_count = str.length; + var key_count = key.length; + var random = isUnique ? exports.random(120) + 40 : 65; + var count = data_count + (random % key_count); + var values = []; + var index = 0; + + values[0] = String.fromCharCode(random); + + var counter = this.length + key.length; - if (!CRYPTO[key]) { - var size = F.config['default-crypto-key']; - CRYPTO[key] = Buffer.alloc(size, key.length > size ? key.substring(0, size) : key, 'utf8'); + for (var i = count - 1; i > 0; i--) { + index = str.charCodeAt(i % data_count); + values[i] = String.fromCharCode(index ^ (key.charCodeAt(i % key_count) ^ random)); } - var cipher = Crypto.createCipheriv(F.config['default-crypto'], CRYPTO[key], CRYPTO[key]); - var str = this.toString(); - var plus = isUnique ? '_' + GUID(10) : ''; - return cipher.update(plus + str.hash() + '#' + str, 'utf8', 'hex') + cipher.final('hex'); + str = exports.createBuffer(counter + '=' + values.join(''), ENCODING).toString('hex'); + var sum = 0; + + for (var i = 0; i < str.length; i++) + sum += str.charCodeAt(i); + + return (sum + checksum(F.config.secret + key)) + '-' + str; }; String.prototype.decrypt = function(key) { - if (!CRYPTO[key]) { - var size = F.config['default-crypto-key']; - CRYPTO[key] = Buffer.alloc(size, key.length > size ? key.substring(0, size) : key, 'utf8'); - } + var index = this.indexOf('-'); + if (index === -1) + return null; - var result; + var cs = +this.substring(0, index); + if (!cs || isNaN(cs)) + return null; - try { - var decipher = Crypto.createDecipheriv(F.config['default-crypto'], CRYPTO[key], CRYPTO[key]); - result = decipher.update(this, 'hex') + decipher.final(); - } catch (e) { + var hash = this.substring(index + 1); + var sum = checksum(F.config.secret + key); + for (var i = 0; i < hash.length; i++) + sum += hash.charCodeAt(i); + + if (sum !== cs) return null; - } - var beg = result.indexOf('#'); - if (beg === -1) + var values = exports.createBuffer(hash, 'hex').toString(ENCODING); + var index = values.indexOf('='); + if (index === -1) + return null; + + var counter = +values.substring(0, index); + if (isNaN(counter)) return null; - var o = result.substring(beg + 1); - var sum = result.substring(0, beg); - if (sum[0] === '_') - sum = sum.substring(11); - return +sum == o.hash() ? o : null; + + values = values.substring(index + 1); + + var count = values.length; + var random = values.charCodeAt(0); + var key_count = key.length; + var data_count = count - (random % key_count); + var decrypt_data = []; + + for (var i = data_count - 1; i > 0; i--) { + index = values.charCodeAt(i) ^ (random ^ key.charCodeAt(i % key_count)); + decrypt_data[i] = String.fromCharCode(index); + } + + var val = decrypt_data.join(''); + return counter !== (val.length + key.length) ? null : val; }; String.prototype.base64ToFile = function(filename, callback) { @@ -5993,4 +6030,5 @@ exports.Callback = function(count, callback) { }; global.WAIT = exports.wait; -!global.F && require('./index'); \ No newline at end of file +!global.F && require('./index'); + From 417a3eb1ce7c8b98f86c9017e5b6a6a903fce412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 21:54:28 +0200 Subject: [PATCH 0453/1669] Removed useless code. --- index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 57b9a308d..860a82bf3 100755 --- a/index.js +++ b/index.js @@ -645,9 +645,7 @@ function Framework() { name: 'Total.js', version: '1.0.0', author: '', - secret: (Os.hostname() + '-' + Os.platform() + '-' + Os.arch()).crc32(true), - 'default-crypto': 'aes-128-ofb', - 'default-crypto-key': 16, + secret: (Os.hostname() + '-' + Os.platform() + '-' + Os.arch()).crc32(true).toString(), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -705,6 +703,7 @@ function Framework() { 'default-root': '', 'default-response-maxage': '11111111', 'default-errorbuilder-status': 200, + 'default-session': '10 minutes', // Seconds (2 minutes) 'default-cors-maxage': 120, From 3402f19fb4391b8c17cfe911a8f1a2dc8a44d5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 21:54:52 +0200 Subject: [PATCH 0454/1669] Updated code. --- session.js | 156 +++++++++++++++++++++++------------- test/controllers/default.js | 8 +- 2 files changed, 103 insertions(+), 61 deletions(-) diff --git a/session.js b/session.js index eb309cc76..97cb6b868 100644 --- a/session.js +++ b/session.js @@ -1,5 +1,4 @@ const Sessions = {}; -const Fs = require('fs'); global.SESSION = function(group) { return Sessions[group] ? Sessions[group] : (Sessions[group] = new Session(group)); @@ -7,49 +6,21 @@ global.SESSION = function(group) { function Session(group) { var t = this; + t.group = group; t.data = {}; - t.group = group.crc32(true); - t.filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'framework_sessions_' + group.crc32(true) + '.jsoncache'); - t.load(); + t.$events = {}; + t.expire = F.config['default-session']; } var SP = Session.prototype; -SP.load = function() { +SP.set = function(id, data) { var self = this; - try { - self.data = Fs.readFileSync(self.filename).toString('utf8').parseJSON(true); - } catch (e) {} - return self; -}; - -SP.save = function() { - var self = this; - setTimeout2('session_' + self.group, function() { - Fs.writeFile(self.filename, JSON.stringify(self.data), F.error()); - }, 1000, 10); - return self; -}; - -SP.key = function() { - return this.group + 'X' + GUID(10); -}; - -SP.set = function(data, expire) { - - var self = this; - var id = self.key(); - var session = self.data[id]; - - if (!session) - session = self.data[id] = {}; - - session.data = data; - session.id = id; - session.expire = (expire || NOW.add('5 minutes')).getTime(); - + var session = data; + session.$id = id; + session.$expire = NOW.add(self.expire).getTime(); + self.data[id] = session; self.$events.set && self.emit('set', session); - self.save(); return id; }; @@ -57,59 +28,130 @@ SP.list = function() { var self = this; var arr = Object.keys(self.data); var online = []; - for (var i = 0; i < arr.length; i++) - online.push(self.data[arr[i]]); + for (var i = 0; i < arr.length; i++) { + var item = self.data[arr[i]]; + item.$expire && online.push(item); + } return online; }; SP.count = function() { - return Object.keys(this.data).length; + return this.list().length; }; -SP.get = function(key) { - return this.data[key] ? this.data[key].data : null; +SP.get = function(key, noextend) { + var item = this.data[key]; + if (item && item.$expire) { + !noextend && (item.$expire = NOW.add(this.expire)); + return item; + } }; -SP.meta = function(key) { - return this.data[key] ? this.data[key] : null; -}; +SP.remove = function(key) { -SP.rem = function(key) { var self = this; - if (self.data[key]) { - self.$events.remove && self.emit('remove', self.data[key]); + var item; + + if (key instanceof Object) { + key.$expire = 0; + item = key; + } else if (self.data[key]) { + item = self.data[key]; delete self.data[key]; - self.save(); } + + item && self.$events.remove && self.emit('remove', key); return self; }; SP.clear = function() { + var self = this; + self.$events.clear && self.emit('clear'); + self.data = {}; + return self; +}; + +SP.clean = function() { var self = this; var arr = Object.keys(self.data); - var count = 0; var time = NOW.getTime(); for (var i = 0; i < arr.length; i++) { - var key = arr[key]; + var key = arr[i]; var obj = self.data[key]; - if (obj.expire < time) { - self.$events.expire && self.emit('expire', obj); + if (!obj.$expire || obj.$expire < time) { + obj.$expire && self.$events.expire && self.emit('expire', obj); delete self.data[key]; - count++; } } - count && self.save(); return self; }; +SP.emit = function(name, a, b, c, d, e, f, g) { + var evt = this.$events[name]; + if (evt) { + var clean = false; + for (var i = 0, length = evt.length; i < length; i++) { + if (evt[i].$once) + clean = true; + evt[i].call(this, a, b, c, d, e, f, g); + } + if (clean) { + evt = evt.remove(n => n.$once); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + } + return this; +}; + +SP.on = function(name, fn) { + + if (!fn.$once) + this.$free = false; + + if (this.$events[name]) + this.$events[name].push(fn); + else + this.$events[name] = [fn]; + return this; +}; + +SP.once = function(name, fn) { + fn.$once = true; + return this.on(name, fn); +}; + +SP.removeListener = function(name, fn) { + var evt = this.$events[name]; + if (evt) { + evt = evt.remove(n => n === fn); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + return this; +}; + +SP.removeAllListeners = function(name) { + if (name === true) + this.$events = EMPTYOBJECT; + else if (name) + this.$events[name] = undefined; + else + this.$events[name] = {}; + return this; +}; + ON('service', function(interval) { if (interval % 5 === 0) { var arr = Object.keys(Sessions); for (var i = 0; i < arr.length; i++) - Sessions[arr[i]].clear(); + Sessions[arr[i]].clean(); } }); F.session = global.SESSION; -global.Session = Session; exports.Session = Session; \ No newline at end of file diff --git a/test/controllers/default.js b/test/controllers/default.js index 06b91921a..d73e692df 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -109,11 +109,11 @@ exports.install = function() { self.plain('408'); }); - assert.ok(F.encrypt('123456', 'key', false) === 'a79aad6f9b92c0d3f1beb3812179863d8e', 'F.encrypt(string)'); - assert.ok(F.decrypt('a79aad6f9b92c0d3f1beb3812179863d8e', 'key', false) === '123456', 'F.decrypt(string)'); + assert.ok(F.encrypt('123456', 'key', false) === '5787-32333d411f0713195a1d0250071706004a1a150a061a0016', 'F.encrypt(string)'); + assert.ok(F.decrypt('5787-32333d411f0713195a1d0250071706004a1a150a061a0016', 'key', false) === '123456', 'F.decrypt(string)'); - assert.ok(F.encrypt({ name: 'Peter' }, 'key', false) === 'bb99af6f9b93c1dff0beb3cb3124d365dd06c70c0a81ec1a3dce8a', 'F.encrypt(object)'); - assert.ok(F.decrypt('bb99af6f9b93c1dff0beb3cb3124d365dd06c70c0a81ec1a3dce8a', 'key').name === 'Peter', 'F.decrypt(object)'); + assert.ok(F.encrypt({ name: 'Peter' }, 'key', false) === '6931-33333d4155174e4c024e105b17745741195806450555174e4c024e105b1774574119', 'F.encrypt(object)'); + assert.ok(F.decrypt('6931-33333d4155174e4c024e105b17745741195806450555174e4c024e105b1774574119', 'key').name === 'Peter', 'F.decrypt(object)'); assert.ok(SOURCE('main').hello() === 'world', 'source'); assert.ok(INCLUDE('main').hello() === 'world', 'source'); From 2097848fc5791ad33342e2bb5ed5ed17d48a0816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 22:25:10 +0200 Subject: [PATCH 0455/1669] Improved code. --- changes.txt | 5 +++++ index.js | 10 ++++++++++ internal.js | 4 ++++ 3 files changed, 19 insertions(+) diff --git a/changes.txt b/changes.txt index adbd846c6..dc42fea35 100755 --- a/changes.txt +++ b/changes.txt @@ -87,6 +87,10 @@ - added: `TESTUSER([user])` for faking of `F.onAuthorize` delegate, targeted for unit-testing only - added: `G` as a global alias for `F.global` - added: a simple support for `.heic` and `.heif` image format +- added: `controller.sitemap_url2()` +- added: `controller.sitemap_name2()` +- added: `@{sitemap_url2()}` +- added: `@{sitemap_name2()}` - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` @@ -143,6 +147,7 @@ - improved: reading performance (+5%) in `U.streamer()` - improved: CSS compressor - improved: CORS processing +- improved: internal encryption/decryption mechanism ======= 2.9.4 (HOTFIX) diff --git a/index.js b/index.js index 860a82bf3..4470e902c 100755 --- a/index.js +++ b/index.js @@ -10911,6 +10911,16 @@ Controller.prototype.sitemap_name = function(name, a, b, c, d, e, f) { return item ? item.name.format(a, b, c, d, e, f) : ''; }; +Controller.prototype.sitemap_url2 = function(language, name, a, b, c, d, e, f) { + var item = F.sitemap(name || this.sitemapid, true, language); + return item ? item.url.format(a, b, c, d, e, f) : ''; +}; + +Controller.prototype.sitemap_name2 = function(language, name, a, b, c, d, e, f) { + var item = F.sitemap(name || this.sitemapid, true, language); + return item ? item.name.format(a, b, c, d, e, f) : ''; +}; + Controller.prototype.sitemap_add = function(parent, name, url) { var self = this; diff --git a/internal.js b/internal.js index 7e904103e..224ef9215 100755 --- a/internal.js +++ b/internal.js @@ -2289,9 +2289,13 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'sitemap_url': case 'sitemap_name': case 'sitemap_navigation': + case 'sitemap_url2': + case 'sitemap_name2': return 'self.' + command; case 'breadcrumb_url': case 'breadcrumb_name': + case 'breadcrumb_url2': + case 'breadcrumb_name2': case 'breadcrumb_navigation': return 'self.sitemap_' + command.substring(10); From fbcffc27a098ed2cb185d6d75bac60ea14213b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 22:48:22 +0200 Subject: [PATCH 0456/1669] Removed useless part. --- session.js | 157 ----------------------------------------------------- 1 file changed, 157 deletions(-) delete mode 100644 session.js diff --git a/session.js b/session.js deleted file mode 100644 index 97cb6b868..000000000 --- a/session.js +++ /dev/null @@ -1,157 +0,0 @@ -const Sessions = {}; - -global.SESSION = function(group) { - return Sessions[group] ? Sessions[group] : (Sessions[group] = new Session(group)); -}; - -function Session(group) { - var t = this; - t.group = group; - t.data = {}; - t.$events = {}; - t.expire = F.config['default-session']; -} - -var SP = Session.prototype; - -SP.set = function(id, data) { - var self = this; - var session = data; - session.$id = id; - session.$expire = NOW.add(self.expire).getTime(); - self.data[id] = session; - self.$events.set && self.emit('set', session); - return id; -}; - -SP.list = function() { - var self = this; - var arr = Object.keys(self.data); - var online = []; - for (var i = 0; i < arr.length; i++) { - var item = self.data[arr[i]]; - item.$expire && online.push(item); - } - return online; -}; - -SP.count = function() { - return this.list().length; -}; - -SP.get = function(key, noextend) { - var item = this.data[key]; - if (item && item.$expire) { - !noextend && (item.$expire = NOW.add(this.expire)); - return item; - } -}; - -SP.remove = function(key) { - - var self = this; - var item; - - if (key instanceof Object) { - key.$expire = 0; - item = key; - } else if (self.data[key]) { - item = self.data[key]; - delete self.data[key]; - } - - item && self.$events.remove && self.emit('remove', key); - return self; -}; - -SP.clear = function() { - var self = this; - self.$events.clear && self.emit('clear'); - self.data = {}; - return self; -}; - -SP.clean = function() { - var self = this; - var arr = Object.keys(self.data); - var time = NOW.getTime(); - for (var i = 0; i < arr.length; i++) { - var key = arr[i]; - var obj = self.data[key]; - if (!obj.$expire || obj.$expire < time) { - obj.$expire && self.$events.expire && self.emit('expire', obj); - delete self.data[key]; - } - } - return self; -}; - -SP.emit = function(name, a, b, c, d, e, f, g) { - var evt = this.$events[name]; - if (evt) { - var clean = false; - for (var i = 0, length = evt.length; i < length; i++) { - if (evt[i].$once) - clean = true; - evt[i].call(this, a, b, c, d, e, f, g); - } - if (clean) { - evt = evt.remove(n => n.$once); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - } - return this; -}; - -SP.on = function(name, fn) { - - if (!fn.$once) - this.$free = false; - - if (this.$events[name]) - this.$events[name].push(fn); - else - this.$events[name] = [fn]; - return this; -}; - -SP.once = function(name, fn) { - fn.$once = true; - return this.on(name, fn); -}; - -SP.removeListener = function(name, fn) { - var evt = this.$events[name]; - if (evt) { - evt = evt.remove(n => n === fn); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - return this; -}; - -SP.removeAllListeners = function(name) { - if (name === true) - this.$events = EMPTYOBJECT; - else if (name) - this.$events[name] = undefined; - else - this.$events[name] = {}; - return this; -}; - -ON('service', function(interval) { - if (interval % 5 === 0) { - var arr = Object.keys(Sessions); - for (var i = 0; i < arr.length; i++) - Sessions[arr[i]].clean(); - } -}); - -F.session = global.SESSION; -exports.Session = Session; \ No newline at end of file From 16e66daff4517daa0a41ca62fac2f36e06889d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 23:08:08 +0200 Subject: [PATCH 0457/1669] Removed session. --- index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/index.js b/index.js index 4470e902c..548bce2fe 100755 --- a/index.js +++ b/index.js @@ -703,7 +703,6 @@ function Framework() { 'default-root': '', 'default-response-maxage': '11111111', 'default-errorbuilder-status': 200, - 'default-session': '10 minutes', // Seconds (2 minutes) 'default-cors-maxage': 120, @@ -996,7 +995,6 @@ F.prototypes = function(fn) { proto.UrlBuilder = framework_builders.UrlBuilder.prototype; proto.WebSocket = WebSocket.prototype; proto.WebSocketClient = WebSocketClient.prototype; - proto.Session = framework_session.prototype; fn.call(proto, proto); return F; }; @@ -16977,5 +16975,3 @@ EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; global.EMPTYCONTROLLER = EMPTYCONTROLLER; - -const framework_session = require('./session'); From eb1c3271cd0e7b2b3f76d7d5480e6256a608ed0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 29 Jun 2018 23:08:20 +0200 Subject: [PATCH 0458/1669] Fixed DatabaseBuilder serialization. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index e7ed0d01b..369eb095d 100755 --- a/nosql.js +++ b/nosql.js @@ -3075,7 +3075,7 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { return self; }; -DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function(raw) { +DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function() { var obj = {}; obj.options = this.$options; obj.code = this.$code; @@ -3089,11 +3089,11 @@ DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = fun if (this.$repository) obj.repository = this.$repository; - return raw ? CLONE(obj) : JSON.stringify(obj); + return JSON.stringify(obj); }; DatabaseBuilder2.prototype.parse = DatabaseBuilder.prototype.parse = function(data) { - data = data instanceof String ? JSON.parse(data, jsonparser) : data; + data = JSON.parse(data, jsonparser); this.$options = data.options; this.$code = data.code; this.$params = data.params; From ab591022825c9f40dfe053a8d6f281e4fba82f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 30 Jun 2018 00:34:27 +0200 Subject: [PATCH 0459/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cacf8b0fc..a87f9a9d7 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-84", + "version": "3.0.0-85", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From e5b77c57ad9ba6c96f300a317f56750d8fd080cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 30 Jun 2018 08:50:28 +0200 Subject: [PATCH 0460/1669] Added `F.syshash` and generating default `secret`. --- changes.txt | 1 + index.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index dc42fea35..046b9d87f 100755 --- a/changes.txt +++ b/changes.txt @@ -91,6 +91,7 @@ - added: `controller.sitemap_name2()` - added: `@{sitemap_url2()}` - added: `@{sitemap_name2()}` +- added: `F.syshash` contains a simple MD5 hash with OS info - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index 548bce2fe..6dd80f1a7 100755 --- a/index.js +++ b/index.js @@ -635,6 +635,7 @@ function Framework() { this.version = 3000; this.version_header = '3.0.0'; this.version_node = process.version.toString(); + this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); this.config = { @@ -645,7 +646,7 @@ function Framework() { name: 'Total.js', version: '1.0.0', author: '', - secret: (Os.hostname() + '-' + Os.platform() + '-' + Os.arch()).crc32(true).toString(), + secret: this.syshash, 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -8859,6 +8860,9 @@ F.$configure_configs = function(arr, rewrite) { subtype = ''; switch (name) { + case 'secret': + obj[name] = value; + break; case 'default-request-length': OBSOLETE(name, 'You need to use "default-request-maxlength"'); obj['default-request-maxlength'] = U.parseInt(value); From 3434f6050f95ba2a191eae0777ab3916e22517c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 30 Jun 2018 10:12:21 +0200 Subject: [PATCH 0461/1669] Added `SchemaEntity.clear()`. --- builders.js | 16 ++++++++++++++++ changes.txt | 1 + 2 files changed, 17 insertions(+) diff --git a/builders.js b/builders.js index 4dc9e870c..5920d46be 100755 --- a/builders.js +++ b/builders.js @@ -330,6 +330,22 @@ SchemaBuilderEntity.prototype.required = function(name, fn) { return self; }; +SchemaBuilderEntity.prototype.clear = function() { + var self = this; + + self.schema = {}; + self.properties = []; + self.fields = []; + + if (self.dependencies) + self.dependencies = null; + + if (self.fields_allow) + self.fields_allow = null; + + return self; +}; + /** * Define type in schema * @param {String|String[]} name diff --git a/changes.txt b/changes.txt index 046b9d87f..e265ff81f 100755 --- a/changes.txt +++ b/changes.txt @@ -92,6 +92,7 @@ - added: `@{sitemap_url2()}` - added: `@{sitemap_name2()}` - added: `F.syshash` contains a simple MD5 hash with OS info +- added: `SchemaEntity.clear()` for removing all current definition - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` From b61773a4f5fc5c8ead8abb0c9913c4df6cb336cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 1 Jul 2018 09:21:14 +0200 Subject: [PATCH 0462/1669] Improved middleware --- index.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 6dd80f1a7..636159a1f 100755 --- a/index.js +++ b/index.js @@ -16673,7 +16673,7 @@ function existsSync(filename, file) { function async_middleware(index, req, res, middleware, callback, options, controller) { - if (res.success || res.headersSent) { + if (res.success || res.headersSent || res.finished) { req.$total_route && req.$total_success(); callback = null; return; @@ -16693,6 +16693,7 @@ function async_middleware(index, req, res, middleware, callback, options, contro if (item.$newversion) { var opt = req.$total_middleware; + if (!index || !opt) { opt = req.$total_middleware = new MiddlewareOptions(); opt.req = req; @@ -16732,7 +16733,11 @@ function async_middleware(index, req, res, middleware, callback, options, contro }, options, controller); } - if (output !== false) + if (res.headersSent || res.finished) { + req.$total_route && req.$total_success(); + callback = null; + return; + } else if (output !== false) return; req.$total_route && req.$total_success(); From 91e072ea7f1573c8a9757976d66826b717c8ef27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 1 Jul 2018 09:26:23 +0200 Subject: [PATCH 0463/1669] Updated `$.options` in Middleware by adding EMPTYOBJECT. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 636159a1f..2d18d2bbd 100755 --- a/index.js +++ b/index.js @@ -16699,7 +16699,7 @@ function async_middleware(index, req, res, middleware, callback, options, contro opt.req = req; opt.res = res; opt.middleware = middleware; - opt.options = options; + opt.options = options || EMPTYOBJECT; opt.controller = controller; opt.callback2 = callback; opt.next = function(err) { From 65d239e66e03297ab6abb5e670e7a62f9bf4572c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 2 Jul 2018 15:06:12 +0200 Subject: [PATCH 0464/1669] Improved `ROUTE()`. --- index.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 2d18d2bbd..7f8afcc40 100755 --- a/index.js +++ b/index.js @@ -1807,6 +1807,14 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { flags = null; } + var type = typeof(funcExecute); + + if (funcExecute instanceof Array) { + tmp = funcExecute; + funcExecute = flags; + flags = tmp; + } + var method = ''; var CUSTOM = typeof(url) === 'function' ? url : null; if (CUSTOM) @@ -1882,18 +1890,11 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { url = framework_internal.encodeUnicodeURL(url); - var type = typeof(funcExecute); var urlcache = url; if (!name) name = url; - if (type === 'object' || funcExecute instanceof Array) { - tmp = funcExecute; - funcExecute = flags; - flags = tmp; - } - if (method) { !flags && (flags = []); flags.push(method); @@ -2177,7 +2178,6 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { } else if (typeof(funcExecute) !== 'function') { viewname = (sitemap && sitemap.url !== '/' ? sitemap.id : workflow ? '' : url) || ''; - if (!workflow || (!viewname && !workflow)) { if (viewname.endsWith('/')) viewname = viewname.substring(0, viewname.length - 1); From ad2dceae69802a26ef91c3583b84bc466ddd8572 Mon Sep 17 00:00:00 2001 From: Martin Smola Date: Wed, 4 Jul 2018 12:26:26 +0200 Subject: [PATCH 0465/1669] Fixed U.get It now supports a path like `[0].prop` in case when the `obj` parameter is an array. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index eef6f4fd1..b01a34828 100755 --- a/utils.js +++ b/utils.js @@ -5704,7 +5704,7 @@ exports.get = function(obj, path) { var builder = []; for (var i = 0, length = arr.length - 1; i < length; i++) - builder.push('if(!w' + (arr[i][0] === '[' ? '' : '.') + arr[i] + ')return'); + builder.push('if(!w' + (!arr[i] || arr[i][0] === '[' ? '' : '.') + arr[i] + ')return'); var v = arr[arr.length - 1]; var fn = (new Function('w', builder.join(';') + ';return w' + (v[0] === '[' ? '' : '.') + v)); From 65a2e1474f0ffcd344d53217b11ec0006aaac541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 4 Jul 2018 22:19:20 +0200 Subject: [PATCH 0466/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a87f9a9d7..bd499271a 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-85", + "version": "3.0.0-86", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 3b3c6eae0a03a53e9c44e8d6a2ca0dbecd74ee2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 8 Jul 2018 19:33:08 +0200 Subject: [PATCH 0467/1669] Fixed `Zip` parsing. --- builders.js | 1 + 1 file changed, 1 insertion(+) diff --git a/builders.js b/builders.js index 5920d46be..83fdfbc6c 100755 --- a/builders.js +++ b/builders.js @@ -1763,6 +1763,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { tmp = ''; break; case 'zip': + tmp = tmp.replace(REGEXP_CLEAN_EMAIL, ''); if (tmp && !type.required && !tmp.isZIP()) tmp = ''; break; From f1fea29c649bc1551a0e7250367beed35720ca8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 10:10:59 +0200 Subject: [PATCH 0468/1669] Added a support for `TABLE()` update allocations. --- nosql.js | 71 ++++++-- recorder.js | 477 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 530 insertions(+), 18 deletions(-) create mode 100644 recorder.js diff --git a/nosql.js b/nosql.js index 369eb095d..64450f581 100755 --- a/nosql.js +++ b/nosql.js @@ -5685,7 +5685,7 @@ TP.extend = function(schema, callback) { var buffer = ''; for (var i = 0; i < items.length; i++) - buffer += self.stringify(items[i]) + NEWLINE; + buffer += self.stringify(items[i], true) + NEWLINE; buffer && writer.write(buffer, 'utf8'); }; @@ -5826,7 +5826,7 @@ TP.$append = function() { var data = ''; for (var i = 0, length = items.length; i < length; i++) - data += self.stringify(items[i].doc) + NEWLINE; + data += self.stringify(items[i].doc, true) + NEWLINE; Fs.appendFile(self.filename, data, function(err) { err && F.error(err, 'Table insert: ' + self.name); @@ -6303,11 +6303,12 @@ TP.$update = function() { for (var a = indexer ? 0 : 1; a < lines.length; a++) { data.line = lines[a].split('|'); + data.length = lines[a].length; data.index = indexer++; - var doc = self.parseData(data); var is = false; var rec = fs.docsbuffer[a]; + var doc = self.parseData(data, data.keys === self.$keys ? EMPTYOBJECT : null); for (var i = 0; i < length; i++) { @@ -6381,24 +6382,17 @@ TP.$update = function() { } } - var upd = self.stringify(doc); + var upd = self.stringify(doc, null, rec.length); if (upd === rec.doc) continue; - var was = true; - if (!change) change = true; - if (rec.doc.length === upd.length) { - var b = Buffer.byteLength(upd); - if (rec.length === b) { - fs.write(upd + NEWLINE, rec.position); - was = false; - } - } - - if (was) { + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + } else { var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; fs.write(tmp, rec.position); fs.write2(upd + NEWLINE); @@ -6742,12 +6736,15 @@ TP.parseData = function(data, cache) { var self = this; var obj = {}; var esc = data.line[0] === '*'; - var val; + var val, alloc; + + if (cache && data.keys.length === data.line.length - 2) + alloc = data.line[data.line.length - 1].length - 1; for (var i = 0; i < data.keys.length; i++) { var key = data.keys[i]; - if (cache && cache[key] != null) { + if (cache && cache !== EMPTYOBJECT && cache[key] != null) { obj[key] = cache[key]; continue; } @@ -6783,14 +6780,17 @@ TP.parseData = function(data, cache) { break; } } + + alloc && (obj.$$alloc = { size: alloc, length: data.length }); return obj; }; -TP.stringify = function(doc) { +TP.stringify = function(doc, insert, byteslen) { var self = this; var output = ''; var esc = false; + var size = 0; for (var i = 0; i < self.$keys.length; i++) { var key = self.$keys[i]; @@ -6800,9 +6800,11 @@ TP.stringify = function(doc) { switch (meta.type) { case 1: // String val = val ? val : ''; + size += 4; break; case 2: // Number val = (val || 0); + size += 2; break; case 3: // Boolean val = (val == true ? '1' : '0'); @@ -6810,9 +6812,11 @@ TP.stringify = function(doc) { case 4: // Date // val = val ? val.toISOString() : ''; val = val ? val.getTime() : ''; + !val && (size += 13); break; case 5: // Object val = val ? JSON.stringify(val) : ''; + size += 4; break; } @@ -6827,6 +6831,37 @@ TP.stringify = function(doc) { output += '|' + val; } + if (doc.$$alloc) { + var l = output.length; + var a = doc.$$alloc; + if (l < a.length) { + var s = (a.length - l) - 1; + if (s > 0) { + output += '|'.padRight(s, '.'); + if (byteslen) { + var b = byteslen - Buffer.byteLength(output); + if (b > 0) { + b--; + for (var i = 0; i < b; i++) + output += '.'; + } else { + var c = s - b; + if (c > 0) + output = output.substring(0, (output.length + b) - 1); + } + } + } else if (s === 0) + output += '|'; + else + insert = true; + } else + insert = true; + } else + insert = true; + + if (insert && size) + output += '|'.padRight(size, '.'); + return (esc ? '*' : '+') + output; }; diff --git a/recorder.js b/recorder.js new file mode 100644 index 000000000..07b95f8af --- /dev/null +++ b/recorder.js @@ -0,0 +1,477 @@ +// Copyright 2012-2018 (c) Peter Širka +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +/** + * @module FrameworkRecorder + * @version 1.0.0 + */ + +const Fs = require('fs'); +const EMPTYBUFFER = U.createBufferSize(1); +const DEFSIZE = 1000; +const ERRREADY = 'Recorder "{0}" is not ready.'; + +function Recorder(name, size) { + + F.path.verify('databases'); + + var t = this; + t.name = name; + t.filename = F.path.databases(name + '.db'); + t.filenameMeta = t.filename + '-meta'; + t.$ready = false; + t.$size = size; + t.header = null; + t.stat = null; + t.changes = 0; + t.bufferappend = []; + t.bufferupdate = []; + t.$events = {}; + + t.$cb_writeupdate = function(err) { + + if (err) { + if (t.$events.error) + t.emit('error', err); + else + F.error(err, 'Recorder: ' + t.name); + } + + t.stat.mtime = NOW; + t.writing = false; + t.flush(); + t.flushmeta(); + }; + + t.$cb_writeappend = function(err, size) { + + if (err) { + if (t.$events.error) + t.emit('error', err); + else + F.error(err, 'Recorder: ' + t.name); + } + + t.writing = false; + t.stat.size += size; + t.stat.atime = NOW; + t.header.count += (size / t.header.size) >> 0; + t.flushheader(); + t.flush(); + }; + + t.$cb_get = function(id, callback) { + t.get(id, callback); + }; + + t.$cb_browse = function(beg, end, callback, done) { + t.browse(beg, end, callback, done); + }; + + t.$cb_insert = function(value, callback, id) { + t.insert(value, callback, id); + }; + + F.path.verify('databases'); + t.open(); +} + +var RP = Recorder.prototype; + +RP.count = function(callback) { + callback(null, this.header.count); + return this; +}; + +RP.emit = function(name, a, b, c, d, e, f, g) { + var evt = this.$events[name]; + if (evt) { + var clean = false; + for (var i = 0, length = evt.length; i < length; i++) { + if (evt[i].$once) + clean = true; + evt[i].call(this, a, b, c, d, e, f, g); + } + if (clean) { + evt = evt.remove(n => n.$once); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + } + return this; +}; + +RP.on = function(name, fn) { + if (this.$ready && (name === 'ready' || name === 'load')) { + fn(); + return this; + } + if (!fn.$once) + this.$free = false; + if (this.$events[name]) + this.$events[name].push(fn); + else + this.$events[name] = [fn]; + return this; +}; + +RP.once = function(name, fn) { + fn.$once = true; + return this.on(name, fn); +}; + +RP.removeListener = function(name, fn) { + var evt = this.$events[name]; + if (evt) { + evt = evt.remove(n => n === fn); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + return this; +}; + +RP.removeAllListeners = function(name) { + if (name === true) + this.$events = EMPTYOBJECT; + else if (name) + this.$events[name] = undefined; + else + this.$events[name] = {}; + return this; +}; + +RP.open = function() { + var self = this; + Fs.stat(self.filename, function(err, stat) { + if (err) { + Fs.open(self.filename, 'w', function(err, fd) { + self.fd = fd; + self.flushheader(function() { + Fs.close(self.fd, function() { + self.open(); + }); + }); + }); + } else { + Fs.readFile(self.filenameMeta, function(err, data) { + self.meta = err ? {} : data.parseJSON(true); + !self.meta && (self.meta = { changes: 0 }); + self.changes = self.meta.changes; + self.stat = stat; + self.$open(); + }); + } + }); + return self; +}; + +RP.resize = function(docSize, callback) { + var self = this; + Fs.open(self.filename + '-tmp', 'w', function(err, fd) { + self.flushheader(function() { + + var index = 200; + var offset = 200; + + var next = function(done) { + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, index, function(err, size) { + if (size) { + var w = U.createBufferSize(docSize); + buf = buf.slice(0, buf.indexOf(EMPTYBUFFER)); + w.fill(buf, 0, buf.length); + index += self.header.size; + Fs.write(fd, w, 0, w.length, offset, function(err, size) { + offset += size; + next(done); + }); + } else + done(); + }); + }; + + next(function() { + // done + Fs.close(fd, function() { + Fs.close(self.fd, function() { + Fs.rename(self.filename + '-tmp', self.filename, function(err) { + callback && callback(err); + }); + }); + }); + }); + + }, fd); + }); + return self; +}; + +RP.ready = function(callback) { + var self = this; + if (self.$ready) + callback(); + else + self.on('ready', callback); + return self; +}; + +RP.flushmeta = function(callback) { + var self = this; + if (self.meta.changes !== self.changes) { + if (self.changes > 1000000000) + self.changes = 1; + self.meta.changes = self.changes; + Fs.writeFile(self.filenameMeta, JSON.stringify(self.meta), function() { + callback && callback(); + }); + } + return self; +}; + +RP.flushheader = function(callback, fd) { + var self = this; + var buf = U.createBufferSize(200); + var header = U.createBuffer('{"name":"Total.js RecorderDB","version":"1.0","size":' + (self.$size || DEFSIZE) + ',"count":' + (self.header ? self.header.count : 0) + '}'); + buf.fill(header, 0, header.length); + Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { + err && console.log(err); + callback && callback(); + }); + return self; +}; + +RP.$open = function() { + var self = this; + Fs.open(self.filename, 'r+', function(err, fd) { + self.fd = fd; + var buf = U.createBufferSize(200); + Fs.read(self.fd, buf, 0, 100, 0, function() { + self.header = buf.slice(0, buf.indexOf(EMPTYBUFFER)).toString('utf8').trim().parseJSON(true); + self.index = self.header.count + 1; + if (self.$size !== self.header.size) { + self.resize(self.$size, function() { + self.open(); + }); + } else { + self.$ready = true; + self.flush(); + self.emit('ready'); + } + }); + }); +}; + +RP.close = function() { + var self = this; + self.fd && Fs.close(self.fd); + self.fd = null; + return self; +}; + +RP.insert = function(value, callback, id) { + + var self = this; + if (self.$ready) { + + var insert = id == null; + + if (insert) { + if (self.meta.removed && self.meta.removed.length) { + id = self.removed.shift(); + insert = false; + } else + id = self.index++; + } + + var type = 1; + switch (typeof(value)) { + case 'number': + type = 2; + break; + case 'boolean': + type = 3; + break; + case 'object': + type = 4; + break; + } + + var val = U.createBuffer(type === 4 ? JSON.stringify(value) : value); + + // - 3 because of "type" (1 byte) + "length of value" (2 bytes) = 3 bytes + var res = U.createBufferSize(self.header.size).fill(val, 4, val.length + 4); + res.writeInt8(type); + res.writeInt16BE(val.length, 1); + + if (insert) { + self.bufferappend.push(res); + self.$events.insert && self.emit('insert', id, value); + } else { + self.bufferupdate.push({ buf: res, pos: id }); + self.$events.update && self.emit('update', id, value); + } + + self.flush(); + callback && callback(null, id); + return id; + } + + if (callback) + setTimeout(self.$cb_insert, 100, value, callback, id); + else + throw new Error(ERRREADY.format(self.name)); +}; + +RP.flush = function() { + + var self = this; + + if (!self.$ready || self.writing) + return self; + + if (self.bufferupdate.length) { + self.writing = true; + var doc = self.bufferupdate.shift(); + Fs.write(self.fd, doc.buf, 0, doc.buf.length, 200 + (doc.pos * self.header.size), self.$cb_writeupdate); + return self; + } else if (!self.bufferappend.length) + return self; + + var buf = self.bufferappend.splice(0, 15); + var data = Buffer.concat(buf); + self.writing = true; + Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); + return self; +}; + +RP.remove = function(id) { + var self = this; + if (self.meta.removed) + self.meta.removed.push(id); + else + self.meta.removed = [id]; + self.changes++; + self.flushmeta(); + self.$events.remove && self.emit('remove', id); + return self; +}; + +RP.update = function(id, value, callback) { + var self = this; + self.insert(value, callback, id - 1); + return self; +}; + +RP.get = RP.read = function(id, callback) { + var self = this; + if (self.$ready) { + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, 200 + ((id - 1) * self.header.size), function(err, size) { + var data = size ? buf.slice(4, buf.readInt16BE(1) + 4).toString('utf8') : null; + switch (buf[0]) { + case 4: // JSON + callback(err, data ? data.parseJSON(true) : null); + break; + case 3: // BOOLEAN + callback(err, data ? data === 'true' : false); + break; + case 2: // NUMBER + callback(err, data ? +data : 0); + break; + case 1: // STRING + default: + callback(err, data ? data : ''); + break; + } + }); + } else + setTimeout(self.$cb_get, 100, id, callback); + return self; +}; + +RP.browse = function(beg, end, callback, done) { + var self = this; + + if (typeof(beg) === 'function') { + done = end; + callback = beg; + end = Number.MAX_SAFE_INTEGER; + beg = 1; + } else if (typeof(end) === 'function') { + done = callback; + callback = end; + end = Number.MAX_SAFE_INTEGER; + } + + if (self.$ready) { + var counter = 1; + var reader = function() { + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, 200 + ((beg - 1) * self.header.size), function(err, size) { + + if (err || !size) { + done && done(); + return; + } + + size = buf.readInt16BE(1) + 4; + + var output; + var data = buf.slice(4, size).toString('utf8'); + + switch (buf[0]) { + case 4: // JSON + output = callback(err, data.parseJSON(true), counter); + break; + case 3: // BOOLEAN + output = callback(err, data === 'true', counter); + break; + case 2: // NUMBER + output = callback(err, +data, counter); + break; + case 1: // STRING + default: + output = callback(err, data, counter); + break; + } + + if (output === false || beg >= end) { + done && done(); + } else { + counter++; + beg++; + reader(); + } + }); + }; + reader(); + } else + setTimeout(self.$cb_browse, 100, beg, end, callback, done); + return self; +}; + +exports.load = function(name, size) { + return new Recorder(name, size); +}; \ No newline at end of file From c0b16a94561a70a9eaadbf56e43a64a971b15252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 10:11:09 +0200 Subject: [PATCH 0469/1669] Added `RECORDER()` as a global variable. --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index 7f8afcc40..1d5a0b34f 100755 --- a/index.js +++ b/index.js @@ -1314,6 +1314,11 @@ global.TABLE = function(name) { return db; }; +global.RECORDER = function(name) { + var key = 'rec_' + name; + return F.databases[key] ? F.databases[key] : F.databases[key] = require('./recorder').load(name, +(F.config['recorder.' + name] || 2000)); +}; + F.stop = F.kill = function(signal) { if (F.isKilled) From b268d72bafba8e67624072a4027b670f56455bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 10:43:19 +0200 Subject: [PATCH 0470/1669] Rename `RECORDER()` to `DATA()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 1d5a0b34f..da763e4cf 100755 --- a/index.js +++ b/index.js @@ -1314,7 +1314,7 @@ global.TABLE = function(name) { return db; }; -global.RECORDER = function(name) { +global.DATA = function(name) { var key = 'rec_' + name; return F.databases[key] ? F.databases[key] : F.databases[key] = require('./recorder').load(name, +(F.config['recorder.' + name] || 2000)); }; From ead83c26e999b99089dbc533cf25e599d868cce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 12:34:04 +0200 Subject: [PATCH 0471/1669] Updated `.json()` serialization about `schema.$clean()`. --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index da763e4cf..be8d01db3 100755 --- a/index.js +++ b/index.js @@ -12027,6 +12027,9 @@ Controller.prototype.$json = function(obj, id, beautify, replacer) { beautify = false; } + if (framework_builders.isSchema(obj)) + obj = obj.$clean(); + var value = beautify ? JSON.stringify(obj, replacer, 4) : JSON.stringify(obj, replacer); return id ? ('') : value; }; @@ -15586,6 +15589,8 @@ function extend_response(PROTO) { PROTO.json = function(obj) { var res = this; F.stats.response.json++; + if (framework_builders.isSchema(obj)) + obj = obj.$clean(); res.options.body = JSON.stringify(obj); res.options.type = CT_JSON; return res.$text(); From 7fe5263bd47286c8fa38c038dc7821e03f2d4dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 12:46:29 +0200 Subject: [PATCH 0472/1669] Fixed code in `F.route()`. --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index be8d01db3..717b3dc04 100755 --- a/index.js +++ b/index.js @@ -1852,7 +1852,8 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (index !== -1) { tmp = url.substring(index); url = url.substring(0, index); - } + } else + tmp = ''; sitemap = F.sitemap(url, true, language); From 81e6a7395c1962d37b0614fa14c5fc73a17051e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 19:31:31 +0200 Subject: [PATCH 0473/1669] Renamed `RECORDER` to `KVALUE`. --- recorder.js => keyvalue.js | 96 +++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 21 deletions(-) rename recorder.js => keyvalue.js (81%) diff --git a/recorder.js b/keyvalue.js similarity index 81% rename from recorder.js rename to keyvalue.js index 07b95f8af..5261d93a8 100644 --- a/recorder.js +++ b/keyvalue.js @@ -20,16 +20,16 @@ // USE OR OTHER DEALINGS IN THE SOFTWARE. /** - * @module FrameworkRecorder + * @module FrameworkKeyValue * @version 1.0.0 */ const Fs = require('fs'); const EMPTYBUFFER = U.createBufferSize(1); -const DEFSIZE = 1000; -const ERRREADY = 'Recorder "{0}" is not ready.'; +const DEFSIZE = 2000; +const ERRREADY = 'Key/Value "{0}" storage is not ready.'; -function Recorder(name, size) { +function KeyValue(name, size) { F.path.verify('databases'); @@ -52,7 +52,7 @@ function Recorder(name, size) { if (t.$events.error) t.emit('error', err); else - F.error(err, 'Recorder: ' + t.name); + F.error(err, 'Key-Value: ' + t.name); } t.stat.mtime = NOW; @@ -67,7 +67,7 @@ function Recorder(name, size) { if (t.$events.error) t.emit('error', err); else - F.error(err, 'Recorder: ' + t.name); + F.error(err, 'Key-Value: ' + t.name); } t.writing = false; @@ -82,6 +82,10 @@ function Recorder(name, size) { t.get(id, callback); }; + t.$cb_read2 = function(id, callback) { + t.read2(id, callback); + }; + t.$cb_browse = function(beg, end, callback, done) { t.browse(beg, end, callback, done); }; @@ -90,11 +94,15 @@ function Recorder(name, size) { t.insert(value, callback, id); }; + t.$cb_write = function(id, prepare, callback) { + t.write(id, prepare, callback); + }; + F.path.verify('databases'); t.open(); } -var RP = Recorder.prototype; +var RP = KeyValue.prototype; RP.count = function(callback) { callback(null, this.header.count); @@ -253,7 +261,7 @@ RP.flushmeta = function(callback) { RP.flushheader = function(callback, fd) { var self = this; var buf = U.createBufferSize(200); - var header = U.createBuffer('{"name":"Total.js RecorderDB","version":"1.0","size":' + (self.$size || DEFSIZE) + ',"count":' + (self.header ? self.header.count : 0) + '}'); + var header = U.createBuffer('{"name":"Total.js KeyValue DB","version":"1.0","size":' + (self.$size || DEFSIZE) + ',"count":' + (self.header ? self.header.count : 0) + '}'); buf.fill(header, 0, header.length); Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { err && console.log(err); @@ -290,7 +298,7 @@ RP.close = function() { return self; }; -RP.insert = function(value, callback, id) { +RP.insert = function(value, callback, edgesid, id) { var self = this; if (self.$ready) { @@ -306,6 +314,7 @@ RP.insert = function(value, callback, id) { } var type = 1; + switch (typeof(value)) { case 'number': type = 2; @@ -320,10 +329,11 @@ RP.insert = function(value, callback, id) { var val = U.createBuffer(type === 4 ? JSON.stringify(value) : value); - // - 3 because of "type" (1 byte) + "length of value" (2 bytes) = 3 bytes - var res = U.createBufferSize(self.header.size).fill(val, 4, val.length + 4); + // 1b TYPE, 4b CONNECTIONSID, 2b DATA-SIZE, DATA + var res = U.createBufferSize(self.header.size).fill(val, 8, val.length + 8); res.writeInt8(type); - res.writeInt16BE(val.length, 1); + res.writeInt32BE(edgesid || 0, 1); + res.writeInt16BE(val.length, 5); if (insert) { self.bufferappend.push(res); @@ -344,6 +354,39 @@ RP.insert = function(value, callback, id) { throw new Error(ERRREADY.format(self.name)); }; +RP.write = function(id, prepare, callback) { + + var self = this; + if (self.$ready) { + + var insert = !id; + + if (insert) { + if (self.meta.removed && self.meta.removed.length) { + id = self.removed.shift(); + insert = false; + } else + id = self.index++; + } + + var res = U.createBufferSize(self.header.size); + + // res.writeInt8(type); + // res.writeInt32BE(edgesid || 0, 1); + + prepare(res); + + self.flush(); + callback && callback(null, id); + return id; + } + + if (callback) + setTimeout(self.$cb_write, 100, id, prepare, callback); + else + throw new Error(ERRREADY.format(self.name)); +}; + RP.flush = function() { var self = this; @@ -359,7 +402,7 @@ RP.flush = function() { } else if (!self.bufferappend.length) return self; - var buf = self.bufferappend.splice(0, 15); + var buf = self.bufferappend.splice(0, 10); var data = Buffer.concat(buf); self.writing = true; Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); @@ -378,9 +421,9 @@ RP.remove = function(id) { return self; }; -RP.update = function(id, value, callback) { +RP.update = function(id, value, callback, edgesid) { var self = this; - self.insert(value, callback, id - 1); + self.insert(value, callback, edgesid, id - 1); return self; }; @@ -389,20 +432,21 @@ RP.get = RP.read = function(id, callback) { if (self.$ready) { var buf = U.createBufferSize(self.header.size); Fs.read(self.fd, buf, 0, buf.length, 200 + ((id - 1) * self.header.size), function(err, size) { - var data = size ? buf.slice(4, buf.readInt16BE(1) + 4).toString('utf8') : null; + var edgesid = buf.readInt32BE(1); + var data = size ? buf.slice(8, buf.readInt16BE(5) + 8).toString('utf8') : null; switch (buf[0]) { case 4: // JSON - callback(err, data ? data.parseJSON(true) : null); + callback(err, data ? data.parseJSON(true) : null, edgesid); break; case 3: // BOOLEAN - callback(err, data ? data === 'true' : false); + callback(err, data ? data === 'true' : false, edgesid); break; case 2: // NUMBER - callback(err, data ? +data : 0); + callback(err, data ? +data : 0, edgesid); break; case 1: // STRING default: - callback(err, data ? data : ''); + callback(err, data ? data : '', edgesid); break; } }); @@ -411,6 +455,16 @@ RP.get = RP.read = function(id, callback) { return self; }; +RP.read2 = function(id, callback) { + var self = this; + if (self.$ready) { + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, 200 + ((id - 1) * self.header.size), err => callback(err, buf)); + } else + setTimeout(self.$cb_read2, 100, id, callback); + return self; +}; + RP.browse = function(beg, end, callback, done) { var self = this; @@ -473,5 +527,5 @@ RP.browse = function(beg, end, callback, done) { }; exports.load = function(name, size) { - return new Recorder(name, size); + return new KeyValue(name, size); }; \ No newline at end of file From 13be17f1b17aac546a6c8b179043ac687a0b0f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 19:31:44 +0200 Subject: [PATCH 0474/1669] Improved `F.use()`. --- changes.txt | 3 ++- index.js | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index e265ff81f..11856699c 100755 --- a/changes.txt +++ b/changes.txt @@ -115,7 +115,8 @@ - updated: `CORS()` tries to join multiple same preferences to one - updated: `U.keywords()` for Chinese/Japan characters - updated: `@{import()}` by adding `manifest` value linked to `/manifest.json` -- updated: internal crypto algorithm to `aes-128-ofb` +- updated: improved crypto algorithm +- updated: `F.use()` supports `function` instead of `middleware` name - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index 717b3dc04..d78b43050 100755 --- a/index.js +++ b/index.js @@ -1314,9 +1314,9 @@ global.TABLE = function(name) { return db; }; -global.DATA = function(name) { - var key = 'rec_' + name; - return F.databases[key] ? F.databases[key] : F.databases[key] = require('./recorder').load(name, +(F.config['recorder.' + name] || 2000)); +global.KVALUE = function(name) { + var key = 'kval_' + name; + return F.databases[key] ? F.databases[key] : F.databases[key] = require('./keyvalue').load(name, +(F.config['keyvalue.' + name] || 2000)); }; F.stop = F.kill = function(signal) { @@ -2674,6 +2674,13 @@ global.MIDDLEWARE = F.middleware = function(name, funcExecute) { * @return {Framework} */ F.use = function(name, url, types, first) { + + if (typeof(name) === 'function') { + var tmp = 'mid' + GUID(5); + MIDDLEWARE(tmp, name); + name = tmp; + } + if (!url && !types) { if (name instanceof Array) { for (var i = 0; i < name.length; i++) From 41cf942aff5c6179e1324c6e31e6e09b19355c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 9 Jul 2018 21:38:54 +0200 Subject: [PATCH 0475/1669] Updated `Image.extent()` by Sarp Aykent. --- image.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/image.js b/image.js index 2977e8a25..0894df10b 100755 --- a/image.js +++ b/image.js @@ -508,7 +508,7 @@ Image.prototype.limit = function(type, value) { return this.push('-limit', type + ' ' + value, 1); }; -Image.prototype.extent = function(w, h) { +Image.prototype.extent = function(w, h, x, y) { var self = this; var size = ''; @@ -520,6 +520,12 @@ Image.prototype.extent = function(w, h) { else if (!w && h) size = 'x' + h; + if (x || y) { + !x && (x = 0); + !y && (y = 0); + size += (x >= 0 ? '+' : '') + x + (y >= 0 ? '+' : '') + y; + } + return self.push('-extent', size, 4, true); }; From e09c3409ddcb1484c0d1d9d7af2c74d77ba56835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Jul 2018 00:02:18 +0200 Subject: [PATCH 0476/1669] Updated `KVALUE`. --- keyvalue.js | 219 +++++++++++++++++++++++++++++++++++++---------- test/test-tmp.js | 92 ++++++++++---------- 2 files changed, 223 insertions(+), 88 deletions(-) diff --git a/keyvalue.js b/keyvalue.js index 5261d93a8..1db314fa0 100644 --- a/keyvalue.js +++ b/keyvalue.js @@ -98,6 +98,10 @@ function KeyValue(name, size) { t.write(id, prepare, callback); }; + t.$cb_link = function(id, toid, type, callback) { + t.link(id, toid, type, callback); + }; + F.path.verify('databases'); t.open(); } @@ -298,7 +302,7 @@ RP.close = function() { return self; }; -RP.insert = function(value, callback, edgesid, id) { +RP.insert = function(value, callback, id) { var self = this; if (self.$ready) { @@ -330,16 +334,20 @@ RP.insert = function(value, callback, edgesid, id) { var val = U.createBuffer(type === 4 ? JSON.stringify(value) : value); // 1b TYPE, 4b CONNECTIONSID, 2b DATA-SIZE, DATA - var res = U.createBufferSize(self.header.size).fill(val, 8, val.length + 8); - res.writeInt8(type); - res.writeInt32BE(edgesid || 0, 1); - res.writeInt16BE(val.length, 5); + var res = insert ? U.createBufferSize(self.header.size).fill(val, 8, val.length + 8) : U.createBufferSize(self.header.size - 5).fill(val, 3, val.length + 3); + + if (insert) { + res.writeInt8(type, 0); + res.writeInt32BE(0, 1); + } + + res.writeInt16BE(val.length, insert ? 5 : 0); if (insert) { self.bufferappend.push(res); self.$events.insert && self.emit('insert', id, value); } else { - self.bufferupdate.push({ buf: res, pos: id }); + self.bufferupdate.push({ buf: res, pos: id, type: type }); self.$events.update && self.emit('update', id, value); } @@ -354,37 +362,147 @@ RP.insert = function(value, callback, edgesid, id) { throw new Error(ERRREADY.format(self.name)); }; -RP.write = function(id, prepare, callback) { +RP.getLinkId = function(id, callback) { + var self = this; + var pos = 200 + ((id - 1) * self.header.size); + var tmp = U.createBufferSize(4); + Fs.read(self.fd, tmp, 0, tmp.length, pos + 1, (err) => callback(err, tmp.readInt32BE(0))); + return self; +}; +RP.setLinkId = function(id, linkid, callback) { var self = this; - if (self.$ready) { + var pos = 200 + ((id - 1) * self.header.size); + var buf = U.createBufferSize(4); + buf.writeInt32BE(linkid); + Fs.write(self.fd, buf, 0, buf.length, pos + 1, function(err, size) { + callback && callback(err); + }); + return self; +}; - var insert = !id; +RP.pushLinkId = function(id, toid, type, callback, parentid) { - if (insert) { - if (self.meta.removed && self.meta.removed.length) { - id = self.removed.shift(); - insert = false; - } else - id = self.index++; - } + var self = this; + var pos = 200 + ((id - 1) * self.header.size); + var buf = U.createBufferSize(self.header.size); + var max = (self.header.size - 9) / 5 >> 0; - var res = U.createBufferSize(self.header.size); + if (id == null) { - // res.writeInt8(type); - // res.writeInt32BE(edgesid || 0, 1); + id = self.index++; - prepare(res); + buf.writeInt8(5); // type, 1b + buf.writeInt32BE(parentid || 0, 1); // link, 4b + buf.writeInt16BE(1, 6); // count, 2b + // TYPE, ID + buf.writeInt8(type, 9); + buf.writeInt32BE(toid, 10); + + self.bufferappend.push(buf); self.flush(); callback && callback(null, id); - return id; + + } else { + + Fs.read(self.fd, buf, 0, buf.length, pos, function(err) { + + if (err) { + callback(err); + return; + } else if (buf[0] !== 5) { + callback(new Error('Invalid value type for linking.')); + return; + } + + var count = buf.readInt16BE(6); + if (count + 1 >= max) { + // we need to create new record because existing buffer is full + self.pushLinkId(0, toid, type, function(err, id) { + // we return new id + callback && callback(err, id); + }, id); + return; + } + + buf.writeInt16BE(count + 1, 6); + + var off = 9 + (count * 5); + buf.writeInt8(type, off); + buf.writeInt32BE(toid, off + 1); + + Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { + callback && callback(err, id); + }); + }); } - if (callback) - setTimeout(self.$cb_write, 100, id, prepare, callback); - else - throw new Error(ERRREADY.format(self.name)); + return self; +}; + +RP.link = function(fromid, toid, type, callback) { + var self = this; + if (self.$ready) { + + var async = []; + var aid = 0; + var bid = 0; + + async.push(function(next) { + self.getLinkId(fromid, function(err, id) { + aid = err ? null : id; + next(); + }); + }); + + async.push(function(next) { + self.getLinkId(toid, function(err, id) { + bid = err ? null : id; + next(); + }); + }); + + async.push(function(next) { + if (aid == null) { + async.length = 0; + next = null; + callback(new Error('Value (from) with "{0}" id does not exist'.format(fromid))); + } else if (bid == null) { + async.length = 0; + next = null; + callback(new Error('Value (to) with "{0}" id does not exist'.format(toid))); + } else + next(); + }); + + async.push(function(next) { + self.pushLinkId(aid == 0 ? null : aid, toid, type, function(err, id) { + if (aid !== id) + self.setLinkId(fromid, id, next); + else + next(); + }); + }); + + async.push(function(next) { + self.pushLinkId(bid == 0 ? null : bid, fromid, type, function(err, id) { + if (bid !== id) + self.setLinkId(toid, id, next); + else + next(); + }); + }); + + async.async(function() { + console.log('DONE'); + callback && callback(null); + }); + + } else + setTimeout(self.$cb_link, 100, fromid, toid, type, callback); + + return self; }; RP.flush = function() { @@ -397,7 +515,8 @@ RP.flush = function() { if (self.bufferupdate.length) { self.writing = true; var doc = self.bufferupdate.shift(); - Fs.write(self.fd, doc.buf, 0, doc.buf.length, 200 + (doc.pos * self.header.size), self.$cb_writeupdate); + var offset = 200 + (doc.pos * self.header.size); + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 5, self.$cb_writeupdate); return self; } else if (!self.bufferappend.length) return self; @@ -421,9 +540,17 @@ RP.remove = function(id) { return self; }; -RP.update = function(id, value, callback, edgesid) { +RP.update = function(id, value, callback) { + var self = this; + self.insert(value, callback, id - 1); + return self; +}; + +RP.traverse = function(id, type, callback) { var self = this; - self.insert(value, callback, edgesid, id - 1); + self.read(id, function(err, doc, linkid) { + console.log(arguments); + }); return self; }; @@ -432,21 +559,33 @@ RP.get = RP.read = function(id, callback) { if (self.$ready) { var buf = U.createBufferSize(self.header.size); Fs.read(self.fd, buf, 0, buf.length, 200 + ((id - 1) * self.header.size), function(err, size) { - var edgesid = buf.readInt32BE(1); - var data = size ? buf.slice(8, buf.readInt16BE(5) + 8).toString('utf8') : null; + var linkid = buf.readInt32BE(1); + var data = buf[0] !== 5 ? (size ? buf.slice(8, buf.readInt16BE(5) + 8).toString('utf8') : null) : null; switch (buf[0]) { + case 5: // LINKS + + var count = buf.readInt16BE(6); // 2b + var links = []; + + for (var i = 0; i < count; i++) { + var pos = 9 + (i * 5); + links.push({ type: buf[pos], id: buf.readInt32BE(pos + 1) }); + } + + callback(err, links, linkid); + break; case 4: // JSON - callback(err, data ? data.parseJSON(true) : null, edgesid); + callback(err, data ? data.parseJSON(true) : null, linkid); break; case 3: // BOOLEAN - callback(err, data ? data === 'true' : false, edgesid); + callback(err, data ? data === 'true' : false, linkid); break; case 2: // NUMBER - callback(err, data ? +data : 0, edgesid); + callback(err, data ? +data : 0, linkid); break; case 1: // STRING default: - callback(err, data ? data : '', edgesid); + callback(err, data ? data : '', linkid); break; } }); @@ -455,16 +594,6 @@ RP.get = RP.read = function(id, callback) { return self; }; -RP.read2 = function(id, callback) { - var self = this; - if (self.$ready) { - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, 200 + ((id - 1) * self.header.size), err => callback(err, buf)); - } else - setTimeout(self.$cb_read2, 100, id, callback); - return self; -}; - RP.browse = function(beg, end, callback, done) { var self = this; @@ -496,6 +625,8 @@ RP.browse = function(beg, end, callback, done) { var data = buf.slice(4, size).toString('utf8'); switch (buf[0]) { + case 5: // LINKS + break; case 4: // JSON output = callback(err, data.parseJSON(true), counter); break; diff --git a/test/test-tmp.js b/test/test-tmp.js index 0b0d4be9d..9f94b7c9f 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,49 +1,53 @@ require('../index'); -const Http = require('http'); -const Https = require('https'); -const Url = require('url'); -function rrr() { - var options = { port: 8080, hostname: '127.0.0.1', method: 'CONNECT', headers: { host: 'www.totaljs.com:443' }}; - var req = Http.request(options); - - req.on('connect', function(res, socket) { - - console.log(res.statusCode); - options = Url.parse('https://www.totaljs.com'); - - var agent = new Https.Agent(); - agent.reuseSocket(socket, req); - options.agent = agent; - - var r = Http.request(options); - - r.on('response', function(res) { - - res.on('data', function(data) { - console.log(data.toString('utf8')); - }); - - res.on('end', function() { - agent.destroy(); - agent = null; - socket.destroy(); - }); - }); - - r.end(); - - }); - - req.end(); -} - -//U.request('http://www.vyvojari.sk', ['get', 'proxy http://127.0.0.1:8080'], console.log); +var insert = false; +var update = false; +var read = false; +var link = true; + +KVALUE('skuska').ready(function() { + // this.insert('Si kokot?', console.log); + //this.count(console.log); + //this.read(20303, console.log); + + + if (insert) { + for (var i = 0; i < 10; i++) + KVALUE('skuska').insert({ id: i + 1, guid: GUID(100) }); + } + + if (link) { + // 11 + //KVALUE('skuska').setLinkId(null, 4, 2, console.log); + //KVALUE('skuska').setLinkId(11, 3, 6, console.log); + //KVALUE('skuska').read(11, console.log); + //KVALUE('skuska').link(2, 6, 1, console.log); + KVALUE('skuska').traverse(2, console.log); + } + + if (read) { + KVALUE('skuska').read(2, console.log); + } + + if (update) { + KVALUE('skuska').update(2, { id: 2, guid: GUID(100), kokotaris: 99 }, console.log); + } + + // KVALUE('skuska').read(2, console.log); + //KVALUE('skuska').link(2, 9, console.log); +}); -RESTBuilder.make(function(builder) { - builder.url('https://www.spektrum-bb.sk'); - builder.proxy('127.0.0.1:8080'); - builder.exec(console.log); +/* +KEYVALUE('skuska').traverse(340304, function(err, val, index) { + console.log(val); + if (index === 6) + return false; }); +*/ +/* -//rrr(); \ No newline at end of file +KEYVALUE('skuska').ready(function() { + for (var i = 0; i < 1000000; i++) + KEYVALUE('skuska').put({ id: i + 1, guid: GUID(100) }); +}); +*/ \ No newline at end of file From 4116bfe39d0ddd74452bd4bfab317889fa1cbb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Jul 2018 15:25:01 +0200 Subject: [PATCH 0477/1669] Updated GraphDB. --- graphdb.js | 1053 +++++++++++++++++++++++++++++++++++++++++++++++++++ index.js | 6 +- keyvalue.js | 662 -------------------------------- 3 files changed, 1056 insertions(+), 665 deletions(-) create mode 100644 graphdb.js delete mode 100644 keyvalue.js diff --git a/graphdb.js b/graphdb.js new file mode 100644 index 000000000..8dce969d3 --- /dev/null +++ b/graphdb.js @@ -0,0 +1,1053 @@ +// Copyright 2012-2018 (c) Peter Širka +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +/** + * @module FrameworkGraphDB + * @version 1.0.0 + */ + +/* +HEADER: VERSION @Int8, COMPRESSION @Int8, REMOVED @Int8, SIZE @UInt16BE, COUNT @UInt32BE, NAME @StringUTF8 +--- 3x TYPES OF RECORD --- +"Node" : TYPE (0-4) @Int8, SIBLINGID @UInt32BE, SIZE_IN_THE_BUFFER @UInt16BE, DATA @StringUTF8 +"Links" : TYPE (5) @Int8, SIBLINGID @UInt32BE, COUNT_IN_THE_BUFFER @UInt16BE, [TYPE @Int8 CONNECTION @Int8 + SIBLINGID @UInt32BE] +"Removed" : TYPE (7) @Int8 +*/ + +// @TODO: +// - check?? if the link doesn't exist +// - removing documents and links (problem) + +const Fs = require('fs'); +const VERSION = 1; +const EMPTYBUFFER = U.createBufferSize(1); +const DEFSIZE = 150; +const BUFFERSIZE = 100; +const HEADERSIZE = 50; +const DELAY = 100; +const ERRREADY = 'GraphDB "{0}" storage is not ready.'; +const DatabaseBuilder = framework_nosql.DatabaseBuilder; + +const NODE_REMOVED = 7; +const NODE_LINKS = 5; + +function GraphDB(name, size) { + + F.path.verify('databases'); + + var t = this; + t.name = name; + t.filename = F.path.databases(name + '.db'); + t.$ready = false; + t.$size = size || DEFSIZE; + + t.header = {}; + t.stat = null; + t.bufferappend = []; + t.bufferupdate = []; + t.bufferremove = []; + t.buffercleaner = []; + t.pending = {}; + t.removed = EMPTYARRAY; + t.$events = {}; + + t.writing = false; + t.reading = false; + t.cleaning = false; + + t.$cb_writeupdate = function(err) { + + if (err) { + if (t.$events.error) + t.emit('error', err); + else + F.error(err, 'GraphDB: ' + t.name); + } + + t.stat.mtime = NOW; + t.writing = false; + t.flush(); + }; + + t.$cb_writeappend = function(err, size) { + + if (err) { + if (t.$events.error) + t.emit('error', err); + else + F.error(err, 'GraphDB: ' + t.name); + } + + t.writing = false; + t.stat.size += size; + t.stat.atime = NOW; + t.header.count += (size / t.header.size) >> 0; + t.flushheader(); + t.flush(); + }; + + t.$cb_get = function(id, callback, type) { + t.get(id, callback, type); + }; + + t.$cb_read2 = function(id, callback) { + t.read2(id, callback); + }; + + t.$cb_browse = function(beg, end, callback, done) { + t.browse(beg, end, callback, done); + }; + + t.$cb_insert = function(value, callback, id) { + t.insert(value, callback, id); + }; + + t.$cb_write = function(id, prepare, callback) { + t.write(id, prepare, callback); + }; + + t.$cb_link = function(id, toid, type, callback) { + t.link(id, toid, type, callback); + }; + + t.$cb_pushlinkid = function(id, toid, type, relation, callback, parentid) { + t.pushLinkId(id, toid, type, relation, callback, parentid); + }; + + t.$cb_getlinkid = function(id, callback) { + t.getLinkId(id, callback); + }; + + t.$cb_setlinkid = function(id, linkid, callback) { + t.setLinkId(id, linkid, callback); + }; + + t.$cb_remove = function(id, callback) { + t.remove(id, callback); + }; + + t.$cb_clean = function(arr, callback) { + t.clean(arr, callback); + }; + + t.$cb_graph = function(id, options, callback, builder) { + t.graph(id, options, callback, builder); + }; + + t.$cb_cleaner = function() { + var ids = t.buffercleaner.splice(0); + t.$cleaner = null; + ids.length && t.clean(ids); + }; + + F.path.verify('databases'); + t.open(); +} + +var GP = GraphDB.prototype; + +GP.scan = function(callback) { + + var self = this; + var remove = []; + var now = Date.now(); + + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" scanning (beg)'.format(self.name)); + + var next = function(index) { + var buf = U.createBufferSize(1); + Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((index - 1) * self.header.size), function(err, size) { + if (size) { + buf[0] === NODE_REMOVED && remove.push(index); + if (index % 5 === 0) + setImmediate(() => next(index + 1)); + else + next(index + 1); + } else { + self.bufferremove = remove; + callback && callback(null, self.bufferremove); + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" scanning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); + } + }); + }; + + next(1); + return self; +}; + +GP.clean = function(arr, callback) { + + var self = this; + + if (!self.$ready || self.cleaning) { + setTimeout(self.$cb_clean, DELAY, arr, callback); + return self; + } + + self.cleaning = true; + + var removed = 0; + var cache = {}; + var now = Date.now(); + + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (beg)'.format(self.name)); + + for (var i = 0; i < arr.length; i++) + cache[arr[i]] = 1; + + var done = function() { + for (var i = 0; i < arr.length; i++) { + if (self.bufferremove.indexOf(arr[i]) === -1) + self.bufferremove.push(arr[i]); + } + self.writing = false; + self.flushheader(); + callback && callback(null, removed); + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); + }; + + var next = function(index) { + var buf = U.createBufferSize(self.header.size); + var position = HEADERSIZE + ((index - 1) * self.header.size); + + Fs.read(self.fd, buf, 0, buf.length, position, function(err, size) { + + if (err || !size) { + done(); + return; + } + + if (buf[0] !== NODE_LINKS) { + next(index + 1); + return; + } + + var id = buf.readUInt32BE(1); + + // Are removed links? + if (cache[id] && buf[0] === NODE_REMOVED) { + next(index + 1); + return; + } + + var count = buf.readUInt16BE(6); // 2b + var buffer = U.createBufferSize(self.header.size); + var off = 9; + var buffercount = 0; + var is = false; + + for (var i = 0; i < count; i++) { + var pos = 9 + (i * 6); + var conn = buf.readUInt32BE(pos + 2); + if (cache[conn]) { + removed++; + is = true; + } else { + buffer.writeInt8(buf[pos], off); + buffer.writeInt8(buf[pos + 1], off + 1); + buffer.writeUInt32BE(buf.readUInt32BE(pos + 2), off + 2); + buffercount++; + off += 6; + } + } + + if (is) { + + buffer.writeInt8(buf[0], 0); // type, 1b + buffer.writeUInt32BE(buf.readUInt32BE(1), 1); // link, 4b + buffer.writeUInt16BE(buffercount, 6); // count, 2b + + // WRITE + Fs.write(self.fd, buffer, 0, buffer.length, position, function(err) { + err && console.log(err); + next(index + 1); + }); + + } else { + if (size) + next(index + 1); + else + done(); + } + }); + }; + + next(1); + return self; +}; + +GP.count = function(callback) { + callback(null, this.header.count); + return this; +}; + +GP.emit = function(name, a, b, c, d, e, f, g) { + var evt = this.$events[name]; + if (evt) { + var clean = false; + for (var i = 0, length = evt.length; i < length; i++) { + if (evt[i].$once) + clean = true; + evt[i].call(this, a, b, c, d, e, f, g); + } + if (clean) { + evt = evt.remove(n => n.$once); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + } + return this; +}; + +GP.on = function(name, fn) { + if (this.$ready && (name === 'ready' || name === 'load')) { + fn(); + return this; + } + if (!fn.$once) + this.$free = false; + if (this.$events[name]) + this.$events[name].push(fn); + else + this.$events[name] = [fn]; + return this; +}; + +GP.once = function(name, fn) { + fn.$once = true; + return this.on(name, fn); +}; + +GP.removeListener = function(name, fn) { + var evt = this.$events[name]; + if (evt) { + evt = evt.remove(n => n === fn); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + return this; +}; + +GP.removeAllListeners = function(name) { + if (name === true) + this.$events = EMPTYOBJECT; + else if (name) + this.$events[name] = undefined; + else + this.$events[name] = {}; + return this; +}; + +GP.open = function() { + var self = this; + Fs.stat(self.filename, function(err, stat) { + if (err) { + Fs.open(self.filename, 'w', function(err, fd) { + self.fd = fd; + self.flushheader(function() { + Fs.close(self.fd, function() { + self.open(); + }); + }); + }); + } else { + self.stat = stat; + self.$open(); + } + }); + return self; +}; + +GP.resize = function(docSize, callback) { + var self = this; + Fs.open(self.filename + '-tmp', 'w', function(err, fd) { + self.flushheader(function() { + + var index = HEADERSIZE; + var offset = HEADERSIZE; + + var next = function(done) { + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, index, function(err, size) { + if (size) { + var w = U.createBufferSize(docSize); + buf = buf.slice(0, buf.indexOf(EMPTYBUFFER)); + w.fill(buf, 0, buf.length); + index += self.header.size; + Fs.write(fd, w, 0, w.length, offset, function(err, size) { + err && self.$events.error && self.emit('error', err); + offset += size; + next(done); + }); + } else + done(); + }); + }; + + next(function() { + Fs.close(fd, function() { + Fs.close(self.fd, function() { + Fs.rename(self.filename + '-tmp', self.filename, function(err) { + err && self.$events.error && self.emit('error', err); + callback && callback(err); + }); + }); + }); + }); + + }, fd); + }); + return self; +}; + +GP.ready = function(callback) { + var self = this; + if (self.$ready) + callback(); + else + self.on('ready', callback); + return self; +}; + +GP.flushheader = function(callback, fd) { + var self = this; + var buf = U.createBufferSize(HEADERSIZE); + + buf.writeInt8(VERSION, 0); // 1b + buf.writeInt8(0, 1); // 1b, COMPRESSION 1: true, 0: false + buf.writeInt8(self.bufferremove.length ? 1 : 0, 2); // 1b, SCAN REMOVED DOCUMENTS 1: true, 0: false + buf.writeUInt16BE(self.$size, 3); // 2b + buf.writeUInt32BE(self.header.count || 0, 6); // 4b + buf.writeUInt32BE(self.header.removed || 0, 10); // 4b + + var str = 'Total.js GraphDB'; + buf.fill(str, 12, str.length + 12); + + Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { + err && console.log(err); + callback && callback(); + }); + + return self; +}; + +GP.$open = function() { + var self = this; + Fs.open(self.filename, 'r+', function(err, fd) { + self.fd = fd; + var buf = U.createBufferSize(HEADERSIZE); + Fs.read(self.fd, buf, 0, buf.length, 0, function() { + self.header.version = buf[0]; + self.header.compress = buf[1] === 1; + self.header.scan = buf[2] === 1; + self.header.size = buf.readUInt16BE(3); + self.header.count = buf.readUInt32BE(6); + self.index = self.header.count + 1; + if (self.$size > self.header.size) { + self.resize(self.$size, function() { + self.open(); + }); + } else { + if (self.header.count && self.header.scan) { + self.scan(); + self.$ready = true; + self.emit('ready'); + } else { + self.$ready = true; + self.flush(); + self.emit('ready'); + } + } + }); + }); +}; + +GP.close = function() { + var self = this; + self.fd && Fs.close(self.fd); + self.fd = null; + return self; +}; + +GP.insert = function(value, callback, id) { + + var self = this; + + if (value instanceof Array) + throw new Error('GraphDB: You can\'t insert an array.'); + + if (value == null) + throw new Error('GraphDB: Value can\'t be nullable.'); + + if (typeof(value) !== 'object') + throw new Error('GraphDB: A value must be object only.'); + + if (self.$ready) { + + var insert = id == null; + var recovered = false; + + if (insert) { + if (self.bufferremove && self.bufferremove.length) { + id = self.bufferremove.shift(); + insert = false; + recovered = true; + } else + id = self.index++; + } + + var type = 0; + var val = U.createBuffer(stringify(value)); + + // 1b TYPE, 4b CONNECTIONSID, 2b DATA-SIZE, DATA + var res = insert ? U.createBufferSize(self.header.size).fill(val, 8, val.length + 8) : U.createBufferSize(self.header.size - 5).fill(val, 3, val.length + 3); + + if (insert) { + res.writeInt8(type, 0); + res.writeUInt32BE(0, 1); + } + + res.writeUInt16BE(val.length, insert ? 5 : 0); + + if (insert) { + self.bufferappend.push(res); + self.$events.insert && self.emit('insert', id, value); + } else { + + self.bufferupdate.push({ buf: res, pos: id - 1, type: type, recovered: recovered }); + + // because of seeking on HDD + if (self.bufferupdate.length > 1) + self.bufferupdate.quicksort('pos'); + + self.$events.update && self.emit('update', id, value); + } + + self.flush(); + callback && callback(null, id); + return id; + } + + if (callback) + setTimeout(self.$cb_insert, DELAY, value, callback, id); + else + throw new Error(ERRREADY.format(self.name)); +}; + +GP.getLinkId = function(id, callback) { + var self = this; + var pos = HEADERSIZE + ((id - 1) * self.header.size); + var tmp = U.createBufferSize(5); + Fs.read(self.fd, tmp, 0, tmp.length, pos, function(err, size) { + callback(err, size && tmp[0] !== NODE_REMOVED ? tmp.readUInt32BE(1) : null); + }); + return self; +}; + +GP.setLinkId = function(id, linkid, callback) { + var self = this; + var pos = HEADERSIZE + ((id - 1) * self.header.size); + var buf = U.createBufferSize(4); + buf.writeUInt32BE(linkid); + Fs.write(self.fd, buf, 0, buf.length, pos + 1, function(err) { + err && self.$events.error && self.emit('error', err); + callback && callback(err); + }); + + return self; +}; + +GP.pushLinkId = function(id, toid, type, relation, callback, parentid) { + + var self = this; + var pos = HEADERSIZE + ((id - 1) * self.header.size); + var buf = U.createBufferSize(self.header.size); + var max = (self.header.size - 9) / 5 >> 0; + + if (id == null) { + + id = self.index++; + + buf.writeInt8(5); // type, 1b + buf.writeUInt32BE(parentid || 0, 1); // link, 4b + buf.writeUInt16BE(1, 6); // count, 2b + + buf.writeInt8(type, 9); // TYPE + buf.writeInt8(typeof(relation) === 'boolean' ? relation ? 1 : 0 : relation, 10); // RELATION TYPE + buf.writeUInt32BE(toid, 11); + + self.bufferappend.push(buf); + self.flush(); + + callback && callback(null, id); + + } else { + + Fs.read(self.fd, buf, 0, buf.length, pos, function(err) { + + if (err) { + callback(err); + return; + } else if (buf[0] !== 5) { + callback(new Error('GraphDB: Invalid value for linking.')); + return; + } + + var count = buf.readUInt16BE(6); + if (count + 1 >= max) { + // we need to create a new record because existing buffer is full + self.pushLinkId(0, toid, type, relation, function(err, id) { + // we return a new id + callback && callback(err, id); + }, id); + } else { + buf.writeUInt16BE(count + 1, 6); + var off = 9 + (count * 6); + buf.writeInt8(type, off); + buf.writeInt8(typeof(relation) === 'boolean' ? relation ? 1 : 0 : relation, off + 1); + buf.writeUInt32BE(toid, off + 2); + Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { + callback && callback(err, id); + }); + } + }); + } + + return self; +}; + +GP.link = function(fromid, toid, type, relation, callback) { + + var self = this; + + if (self.$ready && !self.pending[fromid] && !self.pending[toid]) { + + self.pending[fromid] = 1; + self.pending[toid] = 1; + + var async = []; + var aid = 0; + var bid = 0; + + async.push(function(next) { + self.getLinkId(fromid, function(err, id) { + aid = err ? null : id; + next(); + }); + }); + + async.push(function(next) { + self.getLinkId(toid, function(err, id) { + bid = err ? null : id; + next(); + }); + }); + + async.push(function(next) { + if (aid == null) { + async.length = 0; + next = null; + callback(new Error('GraphDB node (from) with "id:{0}" doesn\'t exist'.format(fromid))); + } else if (bid == null) { + async.length = 0; + next = null; + callback(new Error('GraphDB node (to) with "id:{0}" doesn\'t exist'.format(toid))); + } else + next(); + + if (next == null) { + delete self.pending[fromid]; + delete self.pending[toid]; + } + }); + + // relation: 1/true - they know each other + // relation: 0 - "toid" doesn't know "fromid" + + async.push(function(next) { + self.pushLinkId(aid == 0 ? null : aid, toid, type, 1, function(err, id) { + if (aid !== id) { + aid = id; + self.setLinkId(fromid, id, next); + } else + next(); + }); + }); + + async.push(function(next) { + self.pushLinkId(bid == 0 ? null : bid, fromid, type, relation, function(err, id) { + if (bid !== id) { + bid = id; + self.setLinkId(toid, id, next); + } else + next(); + }); + }); + + async.async(function() { + delete self.pending[fromid]; + delete self.pending[toid]; + callback && callback(null, aid, bid); + }); + + } else + setTimeout(self.$cb_link, DELAY, fromid, toid, type, callback); + + return self; +}; + +GP.setDataType = function(id, type, callback) { + var self = this; + var pos = HEADERSIZE + ((id - 1) * self.header.size); + var buf = U.createBufferSize(1); + buf.writeInt8(type); + Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { + callback && callback(err); + }); + return self; +}; + +GP.flush = function() { + + var self = this; + + if (!self.$ready || self.writing) + return self; + + if (self.bufferupdate.length) { + self.writing = true; + var doc = self.bufferupdate.shift(); + var offset = HEADERSIZE + (doc.pos * self.header.size); + if (doc.recovered) { + var buf = U.createBufferSize(1); + buf.writeInt8(0); + Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { + if (err) { + if (self.$events.error) + self.emit('error', err); + else + F.error(err, 'GraphDB: ' + self.name); + } + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 5, self.$cb_writeupdate); + }); + } else + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 5, self.$cb_writeupdate); + return self; + } else if (!self.bufferappend.length) + return self; + + var buf = self.bufferappend.splice(0, BUFFERSIZE); + var data = Buffer.concat(buf); + self.writing = true; + Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); + return self; +}; + +GP.remove = function(id, callback) { + var self = this; + + if (!self.$ready || self.pending[id]) { + setTimeout(self.$cb_remove, DELAY, id, callback); + return self; + } + + self.pending[id] = 1; + + var buf = U.createBufferSize(5); + + Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { + + if (!size) { + delete self.pending[id]; + callback && callback(null, 0); + return; + } + + if (buf[0] === NODE_REMOVED) { + // Removed + delete self.pending[id]; + callback && callback(null, 0); + return; + } + + var linkid = buf.readUInt32BE(1); + + // @TODO: missing children linkid + self.setDataType(id, NODE_REMOVED, function() { + delete self.pending[id]; + self.$events.remove && self.emit('remove', id); + callback && callback(null, 1); + if (linkid) { + self.setDataType(linkid, NODE_REMOVED, function() { + self.buffercleaner.push(id); + self.buffercleaner.push(linkid); + self.bufferremove.length && self.buffercleaner.push.apply(self.buffercleaner, self.bufferremove); + self.$cleaner && clearTimeout(self.$cleaner); + self.$cleaner = setTimeout(self.$cb_cleaner, 5000); + }); + } else + self.bufferremove.push(id); + }); + }); + + return self; +}; + +GP.update = function(id, value, callback) { + var self = this; + self.insert(value, callback, id - 1); + return self; +}; + +GP.get = GP.read = function(id, callback, type, relation) { + var self = this; + if (self.$ready) { + + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { + + var linkid = buf.readUInt32BE(1); + + switch (buf[0]) { + + case NODE_REMOVED: + // 7: REMOVED DOCUMENT + callback(null, null, 0); + return; + + case 5: // LINKS + + var count = buf.readUInt16BE(6); // 2b + var links = []; + + for (var i = 0; i < count; i++) { + var pos = 9 + (i * 6); + if (type == null && relation == null) + links.push({ TYPE: buf[pos], RELATION: buf[pos + 1], ID: buf.readUInt32BE(pos + 2), INDEX: i }); + else if ((type == null || (type == buf[pos]) && (relation == null || (relation == buf[pos + 1])))) + links.push(buf.readUInt32BE(pos + 2)); + } + + callback(err, links, linkid); + return; + + case 0: + if (size) { + var data = buf.slice(8, buf.readUInt16BE(5) + 8); + callback(err, data ? (eval('({' + data.toString('utf8') + '})')) : null, linkid); + } else + callback(null, null, 0); + return; + } + }); + } else + setTimeout(self.$cb_get, DELAY, id, callback, type, relation); + return self; +}; + +GP.graph = function(id, options, callback) { + + if (typeof(options) === 'function') { + callback = options; + options = EMPTYOBJECT; + } else if (!options) + options = EMPTYOBJECT; + + var self = this; + + if (!self.$ready || self.reading) { + setTimeout(self.$cb_graph, DELAY, id, options, callback); + return self; + } + + self.reading = true; + + self.read(id, function(err, doc, linkid) { + + if (err || !doc) { + self.reading = false; + callback(err, null, 0); + return; + } + + // options.depth + // options.type + // options.relation + + var pending = []; + var tmp = {}; + var count = 1; + var sort = false; + + tmp[id] = 1; + + doc.ID = id; + doc.INDEX = 0; + doc.LEVEL = 0; + doc.NODES = []; + + var reader = function(parent, id, depth) { + + if ((options.depth && depth > options.depth) || (tmp[id])) { + process(); + return; + } + + self.read(id, function(err, links, linkid) { + + if (linkid && !tmp[linkid]) { + pending.push({ id: linkid, parent: parent, depth: depth }); + sort = true; + } + + tmp[linkid] = 1; + + // because of seeking on HDD + links.quicksort('id'); + + links.wait(function(item, next) { + + if ((options.type != null && item.TYPE !== options.type) || (options.relation != null && item.RELATION !== options.relation) || tmp[item.ID]) + return next(); + + tmp[item.ID] = 1; + + self.read(item.ID, function(err, doc, linkid) { + + count++; + + doc.ID = item.ID; + doc.INDEX = item.INDEX; + doc.LEVEL = depth; + doc.NODES = []; + doc.RELATION = item.RELATION; + doc.TYPE = item.TYPE; + parent.NODES.push(doc); + + if (linkid && !tmp[linkid]) { + pending.push({ id: linkid, parent: doc, depth: depth }); + sort = true; + } + + next(); + }); + + }, process); + }); + }; + + var process = function() { + if (pending.length) { + + // because of seeking on HDD + if (sort && pending.length > 1) { + pending.quicksort('id'); + sort = false; + } + + var item = pending.shift(); + reader(item.parent, item.id, item.depth + 1); + } else { + self.reading = false; + callback(null, doc, count); + } + }; + + linkid && pending.push({ id: linkid, parent: doc, depth: 0 }); + process(); + + }, options.type); + + return self; +}; + +GP.browse = function(beg, end, callback, done) { + var self = this; + + if (typeof(beg) === 'function') { + done = end; + callback = beg; + end = Number.MAX_SAFE_INTEGER; + beg = 1; + } else if (typeof(end) === 'function') { + done = callback; + callback = end; + end = Number.MAX_SAFE_INTEGER; + } + + if (self.$ready && !self.reading) { + var counter = 1; + self.reading = true; + var reader = function() { + var buf = U.createBufferSize(self.header.size); + Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((beg - 1) * self.header.size), function(err, size) { + + + if (err || !size) { + self.reading = false; + done && done(); + return; + } + + var output; + + switch (buf[0]) { + case NODE_REMOVED: + case NODE_LINKS: + break; + case 1: + case 0: + output = callback(err, (eval('({' + buf.slice(8, buf.readUInt16BE(5) + 8).toString('utf8') + '})')), buf.readUInt32BE(1), counter); + break; + } + + if (output === false || beg >= end) { + self.reading = false; + done && done(); + } else { + counter++; + beg++; + reader(); + } + }); + }; + reader(); + } else + setTimeout(self.$cb_browse, DELAY, beg, end, callback, done); + return self; +}; + +function stringify(obj) { + var val = JSON.stringify(obj).replace(/"[a-z-0-9]+":/gi, stringifyhelper); + return val.substring(1, val.length - 1); +} + +function stringifyhelper(text) { + return text.substring(1, text.length - 2) + ':'; +} + +exports.load = function(name, size) { + return new GraphDB(name, size); +}; \ No newline at end of file diff --git a/index.js b/index.js index d78b43050..c568923af 100755 --- a/index.js +++ b/index.js @@ -1314,9 +1314,9 @@ global.TABLE = function(name) { return db; }; -global.KVALUE = function(name) { - var key = 'kval_' + name; - return F.databases[key] ? F.databases[key] : F.databases[key] = require('./keyvalue').load(name, +(F.config['keyvalue.' + name] || 2000)); +global.GRAPHDB = function(name) { + var key = 'gdb_' + name; + return F.databases[key] ? F.databases[key] : F.databases[key] = require('./graphdb').load(name, +(F.config['graphdb.' + name] || 0)); }; F.stop = F.kill = function(signal) { diff --git a/keyvalue.js b/keyvalue.js deleted file mode 100644 index 1db314fa0..000000000 --- a/keyvalue.js +++ /dev/null @@ -1,662 +0,0 @@ -// Copyright 2012-2018 (c) Peter Širka -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -/** - * @module FrameworkKeyValue - * @version 1.0.0 - */ - -const Fs = require('fs'); -const EMPTYBUFFER = U.createBufferSize(1); -const DEFSIZE = 2000; -const ERRREADY = 'Key/Value "{0}" storage is not ready.'; - -function KeyValue(name, size) { - - F.path.verify('databases'); - - var t = this; - t.name = name; - t.filename = F.path.databases(name + '.db'); - t.filenameMeta = t.filename + '-meta'; - t.$ready = false; - t.$size = size; - t.header = null; - t.stat = null; - t.changes = 0; - t.bufferappend = []; - t.bufferupdate = []; - t.$events = {}; - - t.$cb_writeupdate = function(err) { - - if (err) { - if (t.$events.error) - t.emit('error', err); - else - F.error(err, 'Key-Value: ' + t.name); - } - - t.stat.mtime = NOW; - t.writing = false; - t.flush(); - t.flushmeta(); - }; - - t.$cb_writeappend = function(err, size) { - - if (err) { - if (t.$events.error) - t.emit('error', err); - else - F.error(err, 'Key-Value: ' + t.name); - } - - t.writing = false; - t.stat.size += size; - t.stat.atime = NOW; - t.header.count += (size / t.header.size) >> 0; - t.flushheader(); - t.flush(); - }; - - t.$cb_get = function(id, callback) { - t.get(id, callback); - }; - - t.$cb_read2 = function(id, callback) { - t.read2(id, callback); - }; - - t.$cb_browse = function(beg, end, callback, done) { - t.browse(beg, end, callback, done); - }; - - t.$cb_insert = function(value, callback, id) { - t.insert(value, callback, id); - }; - - t.$cb_write = function(id, prepare, callback) { - t.write(id, prepare, callback); - }; - - t.$cb_link = function(id, toid, type, callback) { - t.link(id, toid, type, callback); - }; - - F.path.verify('databases'); - t.open(); -} - -var RP = KeyValue.prototype; - -RP.count = function(callback) { - callback(null, this.header.count); - return this; -}; - -RP.emit = function(name, a, b, c, d, e, f, g) { - var evt = this.$events[name]; - if (evt) { - var clean = false; - for (var i = 0, length = evt.length; i < length; i++) { - if (evt[i].$once) - clean = true; - evt[i].call(this, a, b, c, d, e, f, g); - } - if (clean) { - evt = evt.remove(n => n.$once); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - } - return this; -}; - -RP.on = function(name, fn) { - if (this.$ready && (name === 'ready' || name === 'load')) { - fn(); - return this; - } - if (!fn.$once) - this.$free = false; - if (this.$events[name]) - this.$events[name].push(fn); - else - this.$events[name] = [fn]; - return this; -}; - -RP.once = function(name, fn) { - fn.$once = true; - return this.on(name, fn); -}; - -RP.removeListener = function(name, fn) { - var evt = this.$events[name]; - if (evt) { - evt = evt.remove(n => n === fn); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - return this; -}; - -RP.removeAllListeners = function(name) { - if (name === true) - this.$events = EMPTYOBJECT; - else if (name) - this.$events[name] = undefined; - else - this.$events[name] = {}; - return this; -}; - -RP.open = function() { - var self = this; - Fs.stat(self.filename, function(err, stat) { - if (err) { - Fs.open(self.filename, 'w', function(err, fd) { - self.fd = fd; - self.flushheader(function() { - Fs.close(self.fd, function() { - self.open(); - }); - }); - }); - } else { - Fs.readFile(self.filenameMeta, function(err, data) { - self.meta = err ? {} : data.parseJSON(true); - !self.meta && (self.meta = { changes: 0 }); - self.changes = self.meta.changes; - self.stat = stat; - self.$open(); - }); - } - }); - return self; -}; - -RP.resize = function(docSize, callback) { - var self = this; - Fs.open(self.filename + '-tmp', 'w', function(err, fd) { - self.flushheader(function() { - - var index = 200; - var offset = 200; - - var next = function(done) { - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, index, function(err, size) { - if (size) { - var w = U.createBufferSize(docSize); - buf = buf.slice(0, buf.indexOf(EMPTYBUFFER)); - w.fill(buf, 0, buf.length); - index += self.header.size; - Fs.write(fd, w, 0, w.length, offset, function(err, size) { - offset += size; - next(done); - }); - } else - done(); - }); - }; - - next(function() { - // done - Fs.close(fd, function() { - Fs.close(self.fd, function() { - Fs.rename(self.filename + '-tmp', self.filename, function(err) { - callback && callback(err); - }); - }); - }); - }); - - }, fd); - }); - return self; -}; - -RP.ready = function(callback) { - var self = this; - if (self.$ready) - callback(); - else - self.on('ready', callback); - return self; -}; - -RP.flushmeta = function(callback) { - var self = this; - if (self.meta.changes !== self.changes) { - if (self.changes > 1000000000) - self.changes = 1; - self.meta.changes = self.changes; - Fs.writeFile(self.filenameMeta, JSON.stringify(self.meta), function() { - callback && callback(); - }); - } - return self; -}; - -RP.flushheader = function(callback, fd) { - var self = this; - var buf = U.createBufferSize(200); - var header = U.createBuffer('{"name":"Total.js KeyValue DB","version":"1.0","size":' + (self.$size || DEFSIZE) + ',"count":' + (self.header ? self.header.count : 0) + '}'); - buf.fill(header, 0, header.length); - Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { - err && console.log(err); - callback && callback(); - }); - return self; -}; - -RP.$open = function() { - var self = this; - Fs.open(self.filename, 'r+', function(err, fd) { - self.fd = fd; - var buf = U.createBufferSize(200); - Fs.read(self.fd, buf, 0, 100, 0, function() { - self.header = buf.slice(0, buf.indexOf(EMPTYBUFFER)).toString('utf8').trim().parseJSON(true); - self.index = self.header.count + 1; - if (self.$size !== self.header.size) { - self.resize(self.$size, function() { - self.open(); - }); - } else { - self.$ready = true; - self.flush(); - self.emit('ready'); - } - }); - }); -}; - -RP.close = function() { - var self = this; - self.fd && Fs.close(self.fd); - self.fd = null; - return self; -}; - -RP.insert = function(value, callback, id) { - - var self = this; - if (self.$ready) { - - var insert = id == null; - - if (insert) { - if (self.meta.removed && self.meta.removed.length) { - id = self.removed.shift(); - insert = false; - } else - id = self.index++; - } - - var type = 1; - - switch (typeof(value)) { - case 'number': - type = 2; - break; - case 'boolean': - type = 3; - break; - case 'object': - type = 4; - break; - } - - var val = U.createBuffer(type === 4 ? JSON.stringify(value) : value); - - // 1b TYPE, 4b CONNECTIONSID, 2b DATA-SIZE, DATA - var res = insert ? U.createBufferSize(self.header.size).fill(val, 8, val.length + 8) : U.createBufferSize(self.header.size - 5).fill(val, 3, val.length + 3); - - if (insert) { - res.writeInt8(type, 0); - res.writeInt32BE(0, 1); - } - - res.writeInt16BE(val.length, insert ? 5 : 0); - - if (insert) { - self.bufferappend.push(res); - self.$events.insert && self.emit('insert', id, value); - } else { - self.bufferupdate.push({ buf: res, pos: id, type: type }); - self.$events.update && self.emit('update', id, value); - } - - self.flush(); - callback && callback(null, id); - return id; - } - - if (callback) - setTimeout(self.$cb_insert, 100, value, callback, id); - else - throw new Error(ERRREADY.format(self.name)); -}; - -RP.getLinkId = function(id, callback) { - var self = this; - var pos = 200 + ((id - 1) * self.header.size); - var tmp = U.createBufferSize(4); - Fs.read(self.fd, tmp, 0, tmp.length, pos + 1, (err) => callback(err, tmp.readInt32BE(0))); - return self; -}; - -RP.setLinkId = function(id, linkid, callback) { - var self = this; - var pos = 200 + ((id - 1) * self.header.size); - var buf = U.createBufferSize(4); - buf.writeInt32BE(linkid); - Fs.write(self.fd, buf, 0, buf.length, pos + 1, function(err, size) { - callback && callback(err); - }); - return self; -}; - -RP.pushLinkId = function(id, toid, type, callback, parentid) { - - var self = this; - var pos = 200 + ((id - 1) * self.header.size); - var buf = U.createBufferSize(self.header.size); - var max = (self.header.size - 9) / 5 >> 0; - - if (id == null) { - - id = self.index++; - - buf.writeInt8(5); // type, 1b - buf.writeInt32BE(parentid || 0, 1); // link, 4b - buf.writeInt16BE(1, 6); // count, 2b - - // TYPE, ID - buf.writeInt8(type, 9); - buf.writeInt32BE(toid, 10); - - self.bufferappend.push(buf); - self.flush(); - callback && callback(null, id); - - } else { - - Fs.read(self.fd, buf, 0, buf.length, pos, function(err) { - - if (err) { - callback(err); - return; - } else if (buf[0] !== 5) { - callback(new Error('Invalid value type for linking.')); - return; - } - - var count = buf.readInt16BE(6); - if (count + 1 >= max) { - // we need to create new record because existing buffer is full - self.pushLinkId(0, toid, type, function(err, id) { - // we return new id - callback && callback(err, id); - }, id); - return; - } - - buf.writeInt16BE(count + 1, 6); - - var off = 9 + (count * 5); - buf.writeInt8(type, off); - buf.writeInt32BE(toid, off + 1); - - Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { - callback && callback(err, id); - }); - }); - } - - return self; -}; - -RP.link = function(fromid, toid, type, callback) { - var self = this; - if (self.$ready) { - - var async = []; - var aid = 0; - var bid = 0; - - async.push(function(next) { - self.getLinkId(fromid, function(err, id) { - aid = err ? null : id; - next(); - }); - }); - - async.push(function(next) { - self.getLinkId(toid, function(err, id) { - bid = err ? null : id; - next(); - }); - }); - - async.push(function(next) { - if (aid == null) { - async.length = 0; - next = null; - callback(new Error('Value (from) with "{0}" id does not exist'.format(fromid))); - } else if (bid == null) { - async.length = 0; - next = null; - callback(new Error('Value (to) with "{0}" id does not exist'.format(toid))); - } else - next(); - }); - - async.push(function(next) { - self.pushLinkId(aid == 0 ? null : aid, toid, type, function(err, id) { - if (aid !== id) - self.setLinkId(fromid, id, next); - else - next(); - }); - }); - - async.push(function(next) { - self.pushLinkId(bid == 0 ? null : bid, fromid, type, function(err, id) { - if (bid !== id) - self.setLinkId(toid, id, next); - else - next(); - }); - }); - - async.async(function() { - console.log('DONE'); - callback && callback(null); - }); - - } else - setTimeout(self.$cb_link, 100, fromid, toid, type, callback); - - return self; -}; - -RP.flush = function() { - - var self = this; - - if (!self.$ready || self.writing) - return self; - - if (self.bufferupdate.length) { - self.writing = true; - var doc = self.bufferupdate.shift(); - var offset = 200 + (doc.pos * self.header.size); - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 5, self.$cb_writeupdate); - return self; - } else if (!self.bufferappend.length) - return self; - - var buf = self.bufferappend.splice(0, 10); - var data = Buffer.concat(buf); - self.writing = true; - Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); - return self; -}; - -RP.remove = function(id) { - var self = this; - if (self.meta.removed) - self.meta.removed.push(id); - else - self.meta.removed = [id]; - self.changes++; - self.flushmeta(); - self.$events.remove && self.emit('remove', id); - return self; -}; - -RP.update = function(id, value, callback) { - var self = this; - self.insert(value, callback, id - 1); - return self; -}; - -RP.traverse = function(id, type, callback) { - var self = this; - self.read(id, function(err, doc, linkid) { - console.log(arguments); - }); - return self; -}; - -RP.get = RP.read = function(id, callback) { - var self = this; - if (self.$ready) { - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, 200 + ((id - 1) * self.header.size), function(err, size) { - var linkid = buf.readInt32BE(1); - var data = buf[0] !== 5 ? (size ? buf.slice(8, buf.readInt16BE(5) + 8).toString('utf8') : null) : null; - switch (buf[0]) { - case 5: // LINKS - - var count = buf.readInt16BE(6); // 2b - var links = []; - - for (var i = 0; i < count; i++) { - var pos = 9 + (i * 5); - links.push({ type: buf[pos], id: buf.readInt32BE(pos + 1) }); - } - - callback(err, links, linkid); - break; - case 4: // JSON - callback(err, data ? data.parseJSON(true) : null, linkid); - break; - case 3: // BOOLEAN - callback(err, data ? data === 'true' : false, linkid); - break; - case 2: // NUMBER - callback(err, data ? +data : 0, linkid); - break; - case 1: // STRING - default: - callback(err, data ? data : '', linkid); - break; - } - }); - } else - setTimeout(self.$cb_get, 100, id, callback); - return self; -}; - -RP.browse = function(beg, end, callback, done) { - var self = this; - - if (typeof(beg) === 'function') { - done = end; - callback = beg; - end = Number.MAX_SAFE_INTEGER; - beg = 1; - } else if (typeof(end) === 'function') { - done = callback; - callback = end; - end = Number.MAX_SAFE_INTEGER; - } - - if (self.$ready) { - var counter = 1; - var reader = function() { - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, 200 + ((beg - 1) * self.header.size), function(err, size) { - - if (err || !size) { - done && done(); - return; - } - - size = buf.readInt16BE(1) + 4; - - var output; - var data = buf.slice(4, size).toString('utf8'); - - switch (buf[0]) { - case 5: // LINKS - break; - case 4: // JSON - output = callback(err, data.parseJSON(true), counter); - break; - case 3: // BOOLEAN - output = callback(err, data === 'true', counter); - break; - case 2: // NUMBER - output = callback(err, +data, counter); - break; - case 1: // STRING - default: - output = callback(err, data, counter); - break; - } - - if (output === false || beg >= end) { - done && done(); - } else { - counter++; - beg++; - reader(); - } - }); - }; - reader(); - } else - setTimeout(self.$cb_browse, 100, beg, end, callback, done); - return self; -}; - -exports.load = function(name, size) { - return new KeyValue(name, size); -}; \ No newline at end of file From 44eb95754260b7f2a289866d60b52f152f9ffd78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Jul 2018 15:25:09 +0200 Subject: [PATCH 0478/1669] Updated test. --- test/test-tmp.js | 97 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/test/test-tmp.js b/test/test-tmp.js index 9f94b7c9f..4471b4b33 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,40 +1,105 @@ require('../index'); -var insert = false; -var update = false; -var read = false; -var link = true; +const Fs = require('fs'); -KVALUE('skuska').ready(function() { +var insert = 0; +var link = 0; +var update = 0; +var read = 0; +var remove = 1; + +if (insert) { + try { + Fs.unlinkSync(F.path.databases('skuska.db')); + } catch (e) {} +} + +var db = GRAPHDB('skuska'); + +db.ready(function() { // this.insert('Si kokot?', console.log); //this.count(console.log); //this.read(20303, console.log); + //console.log(db); if (insert) { - for (var i = 0; i < 10; i++) - KVALUE('skuska').insert({ id: i + 1, guid: GUID(100) }); + for (var i = 0; i < 1000000; i++) { + if (i % 100000 === 0) + console.log(i); + db.insert({ index: i + 1 }); + } } + db.read(1000001, (err, links) => console.log(1000001, links)); + db.read(1000002, (err, links) => console.log(1000002, links)); + db.read(1000003, (err, links) => console.log(1000003, links)); + db.read(1000004, (err, links) => console.log(1000004, links)); + db.read(1000005, (err, links) => console.log(1000005, links)); + db.read(1000006, (err, links) => console.log(1000006, links)); + + //db.link(100, 1000, 1, 0); + //db.link(100, 10000, 1, 0); + //db.link(100000, 200000, 1, 1); + //db.link(100, 1000000, 1, 0); + + //db.read(1000004, console.log); + if (link) { // 11 - //KVALUE('skuska').setLinkId(null, 4, 2, console.log); - //KVALUE('skuska').setLinkId(11, 3, 6, console.log); - //KVALUE('skuska').read(11, console.log); - //KVALUE('skuska').link(2, 6, 1, console.log); - KVALUE('skuska').traverse(2, console.log); + //GRAPHDB('skuska').setLinkId(null, 4, 2, console.log); + //GRAPHDB('skuska').setLinkId(11, 3, 6, console.log); + //GRAPHDB('skuska').read(11, console.log); + + //GRAPHDB('skuska').read(1001, console.log); + // 100 knows 1000 (but not vice versa) + //GRAPHDB('skuska').link(10, 30, 1, 0, console.log); + //GRAPHDB('skuska').link(20, 30, 1, 0, console.log); + //return; + //return; + + // 100 knows 100000 (but not vice versa) + //GRAPHDB('skuska').link(100, 10000, 1, 0, console.log); + //return; + + //GRAPHDB('skuska').read(100001, console.log); + //return; + + var opt = {}; + + opt.type = 1; + // opt.relation = 1; + + db.graph(100, opt, function(err, doc) { + console.log(JSON.stringify(doc, null, ' ')); + }); + + //db.read(1000005, console.log); + } + + if (remove) { + //console.log(db); + //db.read(51, console.log); + // GRAPHDB('skuska').clean([10, 51], console.log); + setTimeout(function() { + //db.remove(100, console.log); + }, 1000); } if (read) { - KVALUE('skuska').read(2, console.log); + db.read(50, console.log); } if (update) { - KVALUE('skuska').update(2, { id: 2, guid: GUID(100), kokotaris: 99 }, console.log); + db.update(2, { id: 2, guid: GUID(100), kokotaris: 99 }, console.log); } - // KVALUE('skuska').read(2, console.log); - //KVALUE('skuska').link(2, 9, console.log); + // GRAPHDB('skuska').insert({ index: 999999 }, console.log); + + //GRAPHDB('skuska').remove(100, console.log); + + // GRAPHDB('skuska').read(2, console.log); + //GRAPHDB('skuska').link(2, 9, console.log); }); /* From 9dbc10eb43b4d4f89924f653fde0c3fca2b5e279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Jul 2018 20:24:31 +0200 Subject: [PATCH 0479/1669] Added `@{#}` markup as a `default-root`. --- changes.txt | 1 + internal.js | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 11856699c..142fe4cd3 100755 --- a/changes.txt +++ b/changes.txt @@ -93,6 +93,7 @@ - added: `@{sitemap_name2()}` - added: `F.syshash` contains a simple MD5 hash with OS info - added: `SchemaEntity.clear()` for removing all current definition +- added: new view engine markup `@{#}` for simulating of root URL - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/internal.js b/internal.js index 224ef9215..787549320 100755 --- a/internal.js +++ b/internal.js @@ -50,6 +50,7 @@ const REG_7 = /\\/g; const REG_8 = /'/g; const REG_9 = />\n\s+/g; const REG_10 = /(\w|\W)\n\s+ Date: Tue, 10 Jul 2018 20:34:52 +0200 Subject: [PATCH 0480/1669] Added `String.ROOT()`. --- changes.txt | 1 + internal.js | 14 ++------------ utils.js | 10 ++++++++++ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/changes.txt b/changes.txt index 142fe4cd3..2aa4c16a1 100755 --- a/changes.txt +++ b/changes.txt @@ -94,6 +94,7 @@ - added: `F.syshash` contains a simple MD5 hash with OS info - added: `SchemaEntity.clear()` for removing all current definition - added: new view engine markup `@{#}` for simulating of root URL +- added: `String.ROOT()` for replacing `@{#}` markup in strings - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/internal.js b/internal.js index 787549320..c4acb0bb7 100755 --- a/internal.js +++ b/internal.js @@ -50,7 +50,6 @@ const REG_7 = /\\/g; const REG_8 = /'/g; const REG_9 = />\n\s+/g; const REG_10 = /(\w|\W)\n\s+ 22 && l < 30 && this[l] === 'Z' && this[10] === 'T' && this[4] === '-' && this[13] === ':' && this[16] === ':'; }; +String.prototype.ROOT = function() { + return this.replace(REG_ROOT, $urlmaker); +}; + +function $urlmaker(text) { + var c = text[4]; + return F.config['default-root'] ? F.config['default-root'] : (c || ''); +} + if (!String.prototype.trim) { String.prototype.trim = function() { return this.replace(regexpTRIM, ''); From a0a3394a371bd26d484c39ded3fdc87c4391588c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Jul 2018 21:16:57 +0200 Subject: [PATCH 0481/1669] Improved URL remapping. --- index.js | 18 +++++++++++------- utils.js | 28 +++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index c568923af..54b720346 100755 --- a/index.js +++ b/index.js @@ -5214,19 +5214,23 @@ F.onMapping = function(url, def, ispublic, encode) { if (url[0] !== '/') url = '/' + url; + var tmp = url; + if (F.config['default-root']) + tmp = tmp.substring(F.config['default-root'].length - 1); + // component files - if (url[1] === '~') { - var index = url.indexOf('/', 2); - var name = url.substring(2, index); - return F.components.files[name] && F.components.files[name][url.substring(index + 1)] ? (F.path.temp() + url.substring(1)) : null; + if (tmp[1] === '~') { + var index = tmp.indexOf('/', 2); + var name = tmp.substring(2, index); + return F.components.files[name] && F.components.files[name][tmp.substring(index + 1)] ? (F.path.temp() + tmp.substring(1)) : null; } if (F._length_themes) { - var index = url.indexOf('/', 2); + var index = tmp.indexOf('/', 2); if (index !== -1) { - var themeName = url.substring(1, index); + var themeName = tmp.substring(1, index); if (F.themes[themeName]) - return F.themes[themeName] + 'public' + url.substring(index); + return F.themes[themeName] + 'public' + tmp.substring(index); } } diff --git a/utils.js b/utils.js index 00c46dd89..319c4dafb 100755 --- a/utils.js +++ b/utils.js @@ -85,7 +85,10 @@ const PROXYBLACKLIST = { 'localhost': 1, '127.0.0.1': 1, '0.0.0.0': 1 }; const PROXYOPTIONS = { headers: {}, method: 'CONNECT', agent: false }; const PROXYTLS = { headers: {}}; const PROXYOPTIONSHTTP = {}; -const REG_ROOT = /@\{#\}(\/)/g; +const REG_ROOT = /@\{#\}(\/)?/g; +const REG_NOREMAP = /@\{noremap\}(\n)?/g; +const REG_REMAP = /href=".*?"|src=".*?"/gi; +const REG_URLEXT = /(https|http|wss|ws|file):\/\/|\/\/[a-z0-9]|[a-z]:/i; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -2976,10 +2979,26 @@ String.prototype.isJSONDate = function() { return l > 22 && l < 30 && this[l] === 'Z' && this[10] === 'T' && this[4] === '-' && this[13] === ':' && this[16] === ':'; }; -String.prototype.ROOT = function() { - return this.replace(REG_ROOT, $urlmaker); +String.prototype.ROOT = function(noremap) { + + var str = this; + + str = str.replace(REG_NOREMAP, function() { + noremap = true; + return ''; + }).replace(REG_ROOT, $urlmaker); + + if (!noremap && F.config['default-root']) + str = str.replace(REG_REMAP, $urlremap); + + return str; }; +function $urlremap(text) { + var pos = text[0] === 'h' ? 6 : 5; + return REG_URLEXT.test(text) ? text : ((text[0] === 'h' ? 'href' : 'src') + '="' + F.config['default-root'] + (text[pos] === '/' ? text.substring(pos + 1) : text)); +} + function $urlmaker(text) { var c = text[4]; return F.config['default-root'] ? F.config['default-root'] : (c || ''); @@ -6040,5 +6059,4 @@ exports.Callback = function(count, callback) { }; global.WAIT = exports.wait; -!global.F && require('./index'); - +!global.F && require('./index'); \ No newline at end of file From 7912d24aecb791fb332e3b4042dd4dd9fb166245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 10 Jul 2018 23:01:45 +0200 Subject: [PATCH 0482/1669] Improved cleaning. --- graphdb.js | 301 ++++++++++++++++++++++++++--------------------------- 1 file changed, 148 insertions(+), 153 deletions(-) diff --git a/graphdb.js b/graphdb.js index 8dce969d3..ad6c288b8 100644 --- a/graphdb.js +++ b/graphdb.js @@ -40,10 +40,9 @@ const Fs = require('fs'); const VERSION = 1; const EMPTYBUFFER = U.createBufferSize(1); const DEFSIZE = 150; -const BUFFERSIZE = 100; +const BUFFERSIZE = 10; const HEADERSIZE = 50; const DELAY = 100; -const ERRREADY = 'GraphDB "{0}" storage is not ready.'; const DatabaseBuilder = framework_nosql.DatabaseBuilder; const NODE_REMOVED = 7; @@ -64,7 +63,6 @@ function GraphDB(name, size) { t.bufferappend = []; t.bufferupdate = []; t.bufferremove = []; - t.buffercleaner = []; t.pending = {}; t.removed = EMPTYARRAY; t.$events = {}; @@ -152,12 +150,6 @@ function GraphDB(name, size) { t.graph(id, options, callback, builder); }; - t.$cb_cleaner = function() { - var ids = t.buffercleaner.splice(0); - t.$cleaner = null; - ids.length && t.clean(ids); - }; - F.path.verify('databases'); t.open(); } @@ -182,8 +174,7 @@ GP.scan = function(callback) { else next(index + 1); } else { - self.bufferremove = remove; - callback && callback(null, self.bufferremove); + callback && callback(null, remove); F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" scanning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); } }); @@ -193,104 +184,115 @@ GP.scan = function(callback) { return self; }; -GP.clean = function(arr, callback) { +GP.clean = function(callback) { var self = this; if (!self.$ready || self.cleaning) { - setTimeout(self.$cb_clean, DELAY, arr, callback); + setTimeout(self.$cb_clean, DELAY, callback); return self; } self.cleaning = true; - var removed = 0; - var cache = {}; - var now = Date.now(); - - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (beg)'.format(self.name)); - - for (var i = 0; i < arr.length; i++) - cache[arr[i]] = 1; + self.scan(function(err, arr) { - var done = function() { - for (var i = 0; i < arr.length; i++) { - if (self.bufferremove.indexOf(arr[i]) === -1) - self.bufferremove.push(arr[i]); + if (!arr.length) { + callback && callback(null, 0); + return; } - self.writing = false; - self.flushheader(); - callback && callback(null, removed); - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); - }; - var next = function(index) { - var buf = U.createBufferSize(self.header.size); - var position = HEADERSIZE + ((index - 1) * self.header.size); + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (beg)'.format(self.name)); - Fs.read(self.fd, buf, 0, buf.length, position, function(err, size) { + var removed = 0; + var cache = {}; + var now = Date.now(); - if (err || !size) { - done(); - return; - } + for (var i = 0; i < arr.length; i++) + cache[arr[i]] = 1; - if (buf[0] !== NODE_LINKS) { - next(index + 1); - return; + var done = function() { + for (var i = 0; i < arr.length; i++) { + if (self.bufferremove.indexOf(arr[i]) === -1) + self.bufferremove.push(arr[i]); } + self.header.scan = false; + self.writing = false; + self.flushheader(); + callback && callback(null, removed); + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); + self.emit('clean', removed); + }; - var id = buf.readUInt32BE(1); + var next = function(index) { + var buf = U.createBufferSize(self.header.size); + var position = HEADERSIZE + ((index - 1) * self.header.size); - // Are removed links? - if (cache[id] && buf[0] === NODE_REMOVED) { - next(index + 1); - return; - } + Fs.read(self.fd, buf, 0, buf.length, position, function(err, size) { - var count = buf.readUInt16BE(6); // 2b - var buffer = U.createBufferSize(self.header.size); - var off = 9; - var buffercount = 0; - var is = false; - - for (var i = 0; i < count; i++) { - var pos = 9 + (i * 6); - var conn = buf.readUInt32BE(pos + 2); - if (cache[conn]) { - removed++; - is = true; - } else { - buffer.writeInt8(buf[pos], off); - buffer.writeInt8(buf[pos + 1], off + 1); - buffer.writeUInt32BE(buf.readUInt32BE(pos + 2), off + 2); - buffercount++; - off += 6; + if (err || !size) { + done(); + return; } - } - if (is) { + if (buf[0] !== NODE_LINKS) { + next(index + 1); + return; + } - buffer.writeInt8(buf[0], 0); // type, 1b - buffer.writeUInt32BE(buf.readUInt32BE(1), 1); // link, 4b - buffer.writeUInt16BE(buffercount, 6); // count, 2b + var id = buf.readUInt32BE(1); - // WRITE - Fs.write(self.fd, buffer, 0, buffer.length, position, function(err) { - err && console.log(err); + // Are removed links? + if (cache[id] && buf[0] === NODE_REMOVED) { next(index + 1); - }); + return; + } - } else { - if (size) - next(index + 1); - else - done(); - } - }); - }; + var count = buf.readUInt16BE(6); // 2b + var buffer = U.createBufferSize(self.header.size); + var off = 9; + var buffercount = 0; + var is = false; + + for (var i = 0; i < count; i++) { + var pos = 9 + (i * 6); + var conn = buf.readUInt32BE(pos + 2); + if (cache[conn]) { + removed++; + is = true; + } else { + buffer.writeInt8(buf[pos], off); + buffer.writeInt8(buf[pos + 1], off + 1); + buffer.writeUInt32BE(buf.readUInt32BE(pos + 2), off + 2); + buffercount++; + off += 6; + } + } - next(1); + if (is) { + + buffer.writeInt8(buf[0], 0); // type, 1b + buffer.writeUInt32BE(buf.readUInt32BE(1), 1); // link, 4b + buffer.writeUInt16BE(buffercount, 6); // count, 2b + + // WRITE + Fs.write(self.fd, buffer, 0, buffer.length, position, function(err) { + err && console.log(err); + next(index + 1); + }); + + } else { + if (size) + next(index + 1); + else + done(); + } + }); + }; + + next(1); + + }); return self; }; @@ -382,6 +384,8 @@ GP.open = function() { GP.resize = function(docSize, callback) { var self = this; + var now = Date.now(); + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" resizing (beg)'.format(self.name)); Fs.open(self.filename + '-tmp', 'w', function(err, fd) { self.flushheader(function() { @@ -410,6 +414,7 @@ GP.resize = function(docSize, callback) { Fs.close(fd, function() { Fs.close(self.fd, function() { Fs.rename(self.filename + '-tmp', self.filename, function(err) { + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" resizing (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); err && self.$events.error && self.emit('error', err); callback && callback(err); }); @@ -437,7 +442,7 @@ GP.flushheader = function(callback, fd) { buf.writeInt8(VERSION, 0); // 1b buf.writeInt8(0, 1); // 1b, COMPRESSION 1: true, 0: false - buf.writeInt8(self.bufferremove.length ? 1 : 0, 2); // 1b, SCAN REMOVED DOCUMENTS 1: true, 0: false + buf.writeInt8(self.header.scan ? 1 : 0, 2); // 1b, SCAN REMOVED DOCUMENTS 1: true, 0: false buf.writeUInt16BE(self.$size, 3); // 2b buf.writeUInt32BE(self.header.count || 0, 6); // 4b buf.writeUInt32BE(self.header.removed || 0, 10); // 4b @@ -466,19 +471,11 @@ GP.$open = function() { self.header.count = buf.readUInt32BE(6); self.index = self.header.count + 1; if (self.$size > self.header.size) { - self.resize(self.$size, function() { - self.open(); - }); + self.resize(self.$size, () => self.open()); } else { - if (self.header.count && self.header.scan) { - self.scan(); - self.$ready = true; - self.emit('ready'); - } else { - self.$ready = true; - self.flush(); - self.emit('ready'); - } + self.$ready = true; + self.flush(); + self.emit('ready'); } }); }); @@ -495,14 +492,16 @@ GP.insert = function(value, callback, id) { var self = this; - if (value instanceof Array) - throw new Error('GraphDB: You can\'t insert an array.'); - - if (value == null) - throw new Error('GraphDB: Value can\'t be nullable.'); - - if (typeof(value) !== 'object') - throw new Error('GraphDB: A value must be object only.'); + if (value instanceof Array) { + callback && callback(new Error('GraphDB: You can\'t insert an array.')); + return self; + } else if (value == null) { + callback && callback(new Error('GraphDB: Value can\'t be nullable.')); + return self; + } else if (typeof(value) !== 'object') { + callback && callback(new Error('GraphDB: A value must be object only.')); + return self; + } if (self.$ready) { @@ -547,13 +546,10 @@ GP.insert = function(value, callback, id) { self.flush(); callback && callback(null, id); - return id; - } - - if (callback) + } else setTimeout(self.$cb_insert, DELAY, value, callback, id); - else - throw new Error(ERRREADY.format(self.name)); + + return self; }; GP.getLinkId = function(id, callback) { @@ -775,43 +771,40 @@ GP.remove = function(id, callback) { self.pending[id] = 1; - var buf = U.createBufferSize(5); + var removed = []; - Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { - - if (!size) { + var done = function() { + removed.wait(function(id, next) { + self.setDataType(id, NODE_REMOVED, function() { + self.$events.remove && self.emit('remove', id); + next(); + }); + }, function() { delete self.pending[id]; - callback && callback(null, 0); - return; - } + callback && callback(null, removed.length); + }); + }; - if (buf[0] === NODE_REMOVED) { - // Removed - delete self.pending[id]; - callback && callback(null, 0); - return; - } + var find = function(id) { + var buf = U.createBufferSize(5); + Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { + + if (!size || buf[0] === NODE_REMOVED) { + done(); + return; + } - var linkid = buf.readUInt32BE(1); + removed.push(id); - // @TODO: missing children linkid - self.setDataType(id, NODE_REMOVED, function() { - delete self.pending[id]; - self.$events.remove && self.emit('remove', id); - callback && callback(null, 1); - if (linkid) { - self.setDataType(linkid, NODE_REMOVED, function() { - self.buffercleaner.push(id); - self.buffercleaner.push(linkid); - self.bufferremove.length && self.buffercleaner.push.apply(self.buffercleaner, self.bufferremove); - self.$cleaner && clearTimeout(self.$cleaner); - self.$cleaner = setTimeout(self.$cb_cleaner, 5000); - }); - } else - self.bufferremove.push(id); + var link = buf.readUInt32BE(1); + if (link) + find(link); + else + done(); }); - }); + }; + find(id); return self; }; @@ -936,19 +929,21 @@ GP.graph = function(id, options, callback) { self.read(item.ID, function(err, doc, linkid) { - count++; - - doc.ID = item.ID; - doc.INDEX = item.INDEX; - doc.LEVEL = depth; - doc.NODES = []; - doc.RELATION = item.RELATION; - doc.TYPE = item.TYPE; - parent.NODES.push(doc); - - if (linkid && !tmp[linkid]) { - pending.push({ id: linkid, parent: doc, depth: depth }); - sort = true; + if (doc) { + count++; + + doc.ID = item.ID; + doc.INDEX = item.INDEX; + doc.LEVEL = depth; + doc.NODES = []; + doc.RELATION = item.RELATION; + doc.TYPE = item.TYPE; + parent.NODES.push(doc); + + if (linkid && !tmp[linkid]) { + pending.push({ id: linkid, parent: doc, depth: depth }); + sort = true; + } } next(); From ed44ea0ed4dcb3ef79295bed92c83f83e06289a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Jul 2018 22:37:11 +0200 Subject: [PATCH 0483/1669] Added new improvements. --- graphdb.js | 994 ++++++++++++++++++++++++++++++++++++++++++++--------- nosql.js | 146 +++++++- 2 files changed, 966 insertions(+), 174 deletions(-) diff --git a/graphdb.js b/graphdb.js index ad6c288b8..e81265b63 100644 --- a/graphdb.js +++ b/graphdb.js @@ -25,28 +25,38 @@ */ /* -HEADER: VERSION @Int8, COMPRESSION @Int8, REMOVED @Int8, SIZE @UInt16BE, COUNT @UInt32BE, NAME @StringUTF8 +HEADER: VERSION @Int8, COMPRESSION @Int8, SCAN @Int8, CLASSINDEX @Int8, RELATIONINDEX @Int8, SIZE @UInt16BE, COUNT @UInt32BE, NAME @StringUTF8 +CLASSES RAW JSON --- 3x TYPES OF RECORD --- -"Node" : TYPE (0-4) @Int8, SIBLINGID @UInt32BE, SIZE_IN_THE_BUFFER @UInt16BE, DATA @StringUTF8 -"Links" : TYPE (5) @Int8, SIBLINGID @UInt32BE, COUNT_IN_THE_BUFFER @UInt16BE, [TYPE @Int8 CONNECTION @Int8 + SIBLINGID @UInt32BE] -"Removed" : TYPE (7) @Int8 +"Node" : TYPE (0) @Int8, RELATIONID @UInt32BE, PARENTID @UInt32BE, SIZE_IN_THE_BUFFER @UInt16BE, DATA @StringUTF8 +"Links" : TYPE (254) @Int8, RELATIONID @UInt32BE, PARENTID @UInt32BE, COUNT_IN_THE_BUFFER @UInt16BE, [TYPE @Int8 CONNECTION @Int8 + SIBLINGID @UInt32BE] +"Removed" : TYPE (255) @Int8 */ // @TODO: // - check?? if the link doesn't exist // - removing documents and links (problem) +const DEFSIZE = 80; const Fs = require('fs'); const VERSION = 1; const EMPTYBUFFER = U.createBufferSize(1); -const DEFSIZE = 150; const BUFFERSIZE = 10; -const HEADERSIZE = 50; +const INFOSIZE = 50; +const METASIZE = 10240; +const HEADERSIZE = INFOSIZE + METASIZE; const DELAY = 100; const DatabaseBuilder = framework_nosql.DatabaseBuilder; +const REGDATE = /"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+Z"/g; +const REGKEY = /"[a-z-0-9]+":/gi; +const REGTUNESCAPE = /%7C|%0D|%0A/g; +const REGTESCAPETEST = /\||\n|\r/; +const REGTESCAPE = /\||\n|\r/g; +const MAXREADERS = 3; +const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; -const NODE_REMOVED = 7; -const NODE_LINKS = 5; +const NODE_REMOVED = 255; +const NODE_LINKS = 254; function GraphDB(name, size) { @@ -66,9 +76,10 @@ function GraphDB(name, size) { t.pending = {}; t.removed = EMPTYARRAY; t.$events = {}; - + t.$classes = {}; + t.$relations = {}; t.writing = false; - t.reading = false; + t.reading = 0; t.cleaning = false; t.$cb_writeupdate = function(err) { @@ -114,20 +125,20 @@ function GraphDB(name, size) { t.browse(beg, end, callback, done); }; - t.$cb_insert = function(value, callback, id) { - t.insert(value, callback, id); + t.$cb_insert = function(type, value, callback, id) { + t.insert(type, value, callback, id); }; t.$cb_write = function(id, prepare, callback) { t.write(id, prepare, callback); }; - t.$cb_link = function(id, toid, type, callback) { - t.link(id, toid, type, callback); + t.$cb_join = function(type, id, toid, callback) { + t.join(type, id, toid, callback); }; t.$cb_pushlinkid = function(id, toid, type, relation, callback, parentid) { - t.pushLinkId(id, toid, type, relation, callback, parentid); + pushLinkId(t, id, toid, type, relation, callback, parentid); }; t.$cb_getlinkid = function(id, callback) { @@ -150,12 +161,122 @@ function GraphDB(name, size) { t.graph(id, options, callback, builder); }; + t.$cb_class = function(name, declaration, indexer) { + t.class(name, declaration, indexer); + }; + + t.$cb_relation = function(name, both, indexer) { + t.relation(name, both, indexer); + }; + + t.$cb_find = function(type, builder, reverse) { + if (reverse) + t.find2(type, builder); + else + t.find(type, builder); + }; + F.path.verify('databases'); t.open(); } var GP = GraphDB.prototype; +GP.class = function(name, declaration, indexer) { + + var self = this; + if (self.$ready || indexer) { + + var meta = parseSchema(declaration); + var cls = self.$classes[name]; + + meta.name = name; + + if (cls) { + if (cls.raw !== meta.raw) { + // schema is changed + // @todo: re-schemas all classed documents + console.log('SCHEMA IS CHANGED'); + } + } else { + + var index = indexer; + if (indexer == null) { + self.header.classindex++; + index = self.header.classindex; + } + + meta.index = index; + self.$classes[index] = self.$classes[name] = meta; + + if (indexer == null) + self.$flushmeta(); + } + + } else + setTimeout(self.$cb_class, DELAY, name, declaration, indexer); + + return self; +}; + +GP.classes = function(callback) { + var self = this; + var items = []; + for (var i = 0; i < self.header.classindex; i++) { + var item = self.$classes[i + 1]; + item && items.push(item.name); + } + callback(null, items); + return self; +}; + +GP.relation = function(name, both, indexer) { + + var self = this; + if (self.$ready || indexer) { + + var rel = self.$relations[name]; + if (rel) { + + if (both === true && !rel.relation) + self.$flushmeta(); + + } else { + + rel = {}; + + var index = indexer; + if (indexer == null) { + self.header.relationindex++; + index = self.header.relationindex; + } + + rel.index = index; + rel.name = name; + rel.relation = both ? 1 : 0; + self.$relations[index] = self.$relations[name] = rel; + + if (indexer == null) + self.$flushmeta(); + } + + } else + setTimeout(self.$cb_relation, DELAY, name, both, indexer); + + return self; +}; + +GP.relations = function(callback) { + var self = this; + var items = []; + for (var i = 0; i < self.header.relationindex; i++) { + var item = self.$relations[i + 1]; + item && items.push(item.name); + } + callback(null, items); + return self; +}; + GP.scan = function(callback) { var self = this; @@ -248,21 +369,21 @@ GP.clean = function(callback) { return; } - var count = buf.readUInt16BE(6); // 2b + var count = buf.readUInt16BE(9); // 2b var buffer = U.createBufferSize(self.header.size); - var off = 9; + var off = 11; var buffercount = 0; var is = false; for (var i = 0; i < count; i++) { - var pos = 9 + (i * 6); + var pos = 11 + (i * 6); var conn = buf.readUInt32BE(pos + 2); if (cache[conn]) { removed++; is = true; } else { - buffer.writeInt8(buf[pos], off); - buffer.writeInt8(buf[pos + 1], off + 1); + buffer.writeUInt8(buf[pos], off); + buffer.writeUInt8(buf[pos + 1], off + 1); buffer.writeUInt32BE(buf.readUInt32BE(pos + 2), off + 2); buffercount++; off += 6; @@ -271,7 +392,7 @@ GP.clean = function(callback) { if (is) { - buffer.writeInt8(buf[0], 0); // type, 1b + buffer.writeUInt8(buf[0], 0); // type, 1b buffer.writeUInt32BE(buf.readUInt32BE(1), 1); // link, 4b buffer.writeUInt16BE(buffercount, 6); // count, 2b @@ -372,7 +493,7 @@ GP.open = function() { Fs.close(self.fd, function() { self.open(); }); - }); + }, fd); }); } else { self.stat = stat; @@ -437,20 +558,73 @@ GP.ready = function(callback) { }; GP.flushheader = function(callback, fd) { + var self = this; - var buf = U.createBufferSize(HEADERSIZE); + var buf = U.createBufferSize(INFOSIZE); - buf.writeInt8(VERSION, 0); // 1b - buf.writeInt8(0, 1); // 1b, COMPRESSION 1: true, 0: false - buf.writeInt8(self.header.scan ? 1 : 0, 2); // 1b, SCAN REMOVED DOCUMENTS 1: true, 0: false - buf.writeUInt16BE(self.$size, 3); // 2b - buf.writeUInt32BE(self.header.count || 0, 6); // 4b - buf.writeUInt32BE(self.header.removed || 0, 10); // 4b + buf.writeUInt8(VERSION, 0); // 1b + buf.writeUInt8(0, 1); // 1b, COMPRESSION 1: true, 0: false + buf.writeUInt8(self.header.scan ? 1 : 0, 2); // 1b, SCAN REMOVED DOCUMENTS 1: true, 0: false + buf.writeUInt8(self.header.classindex ? self.header.classindex : 0, 3); // 1b + buf.writeUInt8(self.header.relationindex ? self.header.relationindex : 0, 4); // 1b + buf.writeUInt16BE(self.$size, 5); // 2b + buf.writeUInt32BE(self.header.count || 0, 8); // 4b + buf.writeUInt32BE(self.header.removed || 0, 12); // 4b var str = 'Total.js GraphDB'; - buf.fill(str, 12, str.length + 12); + buf.fill(str, 13, str.length + 13); Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { + err && console.log(err); + if (fd) + self.flushmeta(callback, fd); + else + callback && callback(); + }); + + return self; +}; + +GP.$flushmeta = function() { + var self = this; + self.$flushmetadelay && clearImmediate(self.$flushmetadelay); + self.$flushmetadelay = setImmediate(self => self.flushmeta(), self); +}; + +GP.flushmeta = function(callback, fd) { + + var self = this; + var buf = U.createBufferSize(METASIZE); + var meta = {}; + self.$flushmetadelay = null; + + meta.c = []; // classes + meta.r = []; // relations + + if (self.header.classindex) { + for (var i = 0; i < self.header.classindex; i++) { + var item = self.$classes[i + 1]; + item && meta.c.push({ i: i + 1, n: item.name, r: item.raw }); + } + var b = U.createBufferSize(1); + b.writeUInt8(self.header.classindex); + Fs.write(fd || self.fd, b, 0, 1, 3, NOOP); + } + + if (self.header.relationindex) { + for (var i = 0; i < self.header.relationindex; i++) { + var item = self.$relations[i + 1]; + item && meta.r.push({ i: i + 1, n: item.name, r: item.relation }); + } + var b = U.createBufferSize(1); + b.writeUInt8(self.header.relationindex); + Fs.write(fd || self.fd, b, 0, 1, 4, NOOP); + } + + var data = JSON.stringify(meta); + buf.fill(data, 0, data.length); + + Fs.write(fd || self.fd, buf, 0, buf.length, INFOSIZE, function(err) { err && console.log(err); callback && callback(); }); @@ -462,21 +636,52 @@ GP.$open = function() { var self = this; Fs.open(self.filename, 'r+', function(err, fd) { self.fd = fd; - var buf = U.createBufferSize(HEADERSIZE); + var buf = U.createBufferSize(INFOSIZE); Fs.read(self.fd, buf, 0, buf.length, 0, function() { + self.header.version = buf[0]; self.header.compress = buf[1] === 1; self.header.scan = buf[2] === 1; - self.header.size = buf.readUInt16BE(3); - self.header.count = buf.readUInt32BE(6); + self.header.classindex = buf[3]; + self.header.relationindex = buf[4]; + self.header.size = buf.readUInt16BE(5); + self.header.count = buf.readUInt32BE(8); + self.header.buffersize = ((1024 / self.header.size) * BUFFERSIZE) >> 0; + + if (self.header.buffersize < 1) + self.header.buffersize = 1; + self.index = self.header.count + 1; - if (self.$size > self.header.size) { - self.resize(self.$size, () => self.open()); - } else { - self.$ready = true; - self.flush(); - self.emit('ready'); - } + + buf = U.createBufferSize(METASIZE); + Fs.read(self.fd, buf, 0, buf.length, INFOSIZE, function() { + + var data = buf.slice(0, buf.indexOf(EMPTYBUFFER)).toString('utf8').parseJSON(); + if (data) { + // registering classes + if (data.r instanceof Array) { + for (var i = 0; i < data.c.length; i++) { + var item = data.c[i]; + self.class(item.n, item.r, item.i); + } + } + // registering relations + if (data.r instanceof Array) { + for (var i = 0; i < data.r.length; i++) { + var item = data.r[i]; + self.relation(item.n, item.r, item.i); + } + } + } + + if (self.$size > self.header.size) { + self.resize(self.$size, () => self.open()); + } else { + self.$ready = true; + self.flush(); + self.emit('ready'); + } + }); }); }); }; @@ -488,10 +693,16 @@ GP.close = function() { return self; }; -GP.insert = function(value, callback, id) { +GP.insert = function(type, value, callback, id) { var self = this; + if (typeof(type) === 'object') { + callback = value; + value = type; + type = 0; + } + if (value instanceof Array) { callback && callback(new Error('GraphDB: You can\'t insert an array.')); return self; @@ -501,10 +712,31 @@ GP.insert = function(value, callback, id) { } else if (typeof(value) !== 'object') { callback && callback(new Error('GraphDB: A value must be object only.')); return self; + } else if (type > 253) { + callback && callback(new Error('GraphDB: Illegal class value "{0}"'.format(type))); + return self; } if (self.$ready) { + var val; + + if (type) { + var schema = self.$classes[type]; + if (!schema) { + callback && callback(new Error('GraphDB: Schema "{0}" not found.'.format(name))); + return self; + } + val = U.createBuffer(stringifyData(schema, value)); + type = schema.index; + } else + val = U.createBuffer(stringify(value)); + + if (val.length > self.header.size - 11) { + callback(new Error('GraphDB: Data too long.')); + return self; + } + var insert = id == null; var recovered = false; @@ -517,18 +749,15 @@ GP.insert = function(value, callback, id) { id = self.index++; } - var type = 0; - var val = U.createBuffer(stringify(value)); - - // 1b TYPE, 4b CONNECTIONSID, 2b DATA-SIZE, DATA - var res = insert ? U.createBufferSize(self.header.size).fill(val, 8, val.length + 8) : U.createBufferSize(self.header.size - 5).fill(val, 3, val.length + 3); + var res = insert ? U.createBufferSize(self.header.size).fill(val, 11, val.length + 11) : U.createBufferSize(self.header.size - 9).fill(val, 2, val.length + 2); if (insert) { - res.writeInt8(type, 0); - res.writeUInt32BE(0, 1); + res.writeUInt8(type, 0); + res.writeUInt32BE(0, 1); // RELATIONID + res.writeUInt32BE(0, 5); // PARENTID } - res.writeUInt16BE(val.length, insert ? 5 : 0); + res.writeUInt16BE(val.length, insert ? 9 : 0); if (insert) { self.bufferappend.push(res); @@ -547,7 +776,7 @@ GP.insert = function(value, callback, id) { self.flush(); callback && callback(null, id); } else - setTimeout(self.$cb_insert, DELAY, value, callback, id); + setTimeout(self.$cb_insert, DELAY, type, value, callback, id); return self; }; @@ -555,9 +784,9 @@ GP.insert = function(value, callback, id) { GP.getLinkId = function(id, callback) { var self = this; var pos = HEADERSIZE + ((id - 1) * self.header.size); - var tmp = U.createBufferSize(5); - Fs.read(self.fd, tmp, 0, tmp.length, pos, function(err, size) { - callback(err, size && tmp[0] !== NODE_REMOVED ? tmp.readUInt32BE(1) : null); + var tmp = U.createBufferSize(4); + Fs.read(self.fd, tmp, 0, tmp.length, pos + 1, function(err, size) { + callback(err, size && tmp[0] !== NODE_REMOVED ? tmp.readUInt32BE(0) : null); }); return self; }; @@ -575,24 +804,24 @@ GP.setLinkId = function(id, linkid, callback) { return self; }; -GP.pushLinkId = function(id, toid, type, relation, callback, parentid) { +function pushLinkId(self, id, toid, type, initializer, callback, parentid) { - var self = this; var pos = HEADERSIZE + ((id - 1) * self.header.size); var buf = U.createBufferSize(self.header.size); - var max = (self.header.size - 9) / 5 >> 0; + var max = (self.header.size - 11) / 6 >> 0; // 6b --> TYPE 1b + RELATION 1b + ID 4b if (id == null) { id = self.index++; - buf.writeInt8(5); // type, 1b + buf.writeUInt8(NODE_LINKS); // type, 1b buf.writeUInt32BE(parentid || 0, 1); // link, 4b - buf.writeUInt16BE(1, 6); // count, 2b + buf.writeUInt32BE(id, 5); // parent, 4b + buf.writeUInt16BE(1, 9); // count, 2b - buf.writeInt8(type, 9); // TYPE - buf.writeInt8(typeof(relation) === 'boolean' ? relation ? 1 : 0 : relation, 10); // RELATION TYPE - buf.writeUInt32BE(toid, 11); + buf.writeUInt8(type, 11); // TYPE + buf.writeUInt8(initializer ? 1 : 0, 12); // RELATION TYPE + buf.writeUInt32BE(toid, 13); self.bufferappend.push(buf); self.flush(); @@ -606,23 +835,23 @@ GP.pushLinkId = function(id, toid, type, relation, callback, parentid) { if (err) { callback(err); return; - } else if (buf[0] !== 5) { + } else if (buf[0] !== NODE_LINKS) { callback(new Error('GraphDB: Invalid value for linking.')); return; } - var count = buf.readUInt16BE(6); + var count = buf.readUInt16BE(9); if (count + 1 >= max) { // we need to create a new record because existing buffer is full - self.pushLinkId(0, toid, type, relation, function(err, id) { + pushLinkId(self, null, toid, type, initializer, function(err, id) { // we return a new id callback && callback(err, id); }, id); } else { - buf.writeUInt16BE(count + 1, 6); - var off = 9 + (count * 6); - buf.writeInt8(type, off); - buf.writeInt8(typeof(relation) === 'boolean' ? relation ? 1 : 0 : relation, off + 1); + buf.writeUInt16BE(count + 1, 9); // count, 2b + var off = 11 + (count * 6); + buf.writeUInt8(type, off); + buf.writeUInt8(initializer ? 1 : 0, off + 1); buf.writeUInt32BE(toid, off + 2); Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { callback && callback(err, id); @@ -632,14 +861,20 @@ GP.pushLinkId = function(id, toid, type, relation, callback, parentid) { } return self; -}; +} -GP.link = function(fromid, toid, type, relation, callback) { +GP.join = function(type, fromid, toid, callback) { var self = this; if (self.$ready && !self.pending[fromid] && !self.pending[toid]) { + var relation = self.$relations[type]; + if (relation == null) { + callback(new Error('GraphDB: relation "{0}" not found'.format(type))); + return self; + } + self.pending[fromid] = 1; self.pending[toid] = 1; @@ -662,6 +897,7 @@ GP.link = function(fromid, toid, type, relation, callback) { }); async.push(function(next) { + if (aid == null) { async.length = 0; next = null; @@ -683,7 +919,8 @@ GP.link = function(fromid, toid, type, relation, callback) { // relation: 0 - "toid" doesn't know "fromid" async.push(function(next) { - self.pushLinkId(aid == 0 ? null : aid, toid, type, 1, function(err, id) { + + pushLinkId(self, aid == 0 ? null : aid, toid, relation.index, true, function(err, id) { if (aid !== id) { aid = id; self.setLinkId(fromid, id, next); @@ -693,7 +930,7 @@ GP.link = function(fromid, toid, type, relation, callback) { }); async.push(function(next) { - self.pushLinkId(bid == 0 ? null : bid, fromid, type, relation, function(err, id) { + pushLinkId(self, bid == 0 ? null : bid, fromid, relation.index, false, function(err, id) { if (bid !== id) { bid = id; self.setLinkId(toid, id, next); @@ -709,7 +946,7 @@ GP.link = function(fromid, toid, type, relation, callback) { }); } else - setTimeout(self.$cb_link, DELAY, fromid, toid, type, callback); + setTimeout(self.$cb_join, DELAY, type, fromid, toid, callback); return self; }; @@ -718,7 +955,7 @@ GP.setDataType = function(id, type, callback) { var self = this; var pos = HEADERSIZE + ((id - 1) * self.header.size); var buf = U.createBufferSize(1); - buf.writeInt8(type); + buf.writeUInt8(type); Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { callback && callback(err); }); @@ -738,7 +975,7 @@ GP.flush = function() { var offset = HEADERSIZE + (doc.pos * self.header.size); if (doc.recovered) { var buf = U.createBufferSize(1); - buf.writeInt8(0); + buf.writeUInt8(doc.type); Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { if (err) { if (self.$events.error) @@ -746,15 +983,15 @@ GP.flush = function() { else F.error(err, 'GraphDB: ' + self.name); } - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 5, self.$cb_writeupdate); + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 9, self.$cb_writeupdate); }); } else - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 5, self.$cb_writeupdate); + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 9, self.$cb_writeupdate); return self; } else if (!self.bufferappend.length) return self; - var buf = self.bufferappend.splice(0, BUFFERSIZE); + var buf = self.bufferappend.splice(0, self.header.buffersize); var data = Buffer.concat(buf); self.writing = true; Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); @@ -785,7 +1022,7 @@ GP.remove = function(id, callback) { }); }; - var find = function(id) { + var find = function(id, level) { var buf = U.createBufferSize(5); Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { @@ -798,69 +1035,298 @@ GP.remove = function(id, callback) { var link = buf.readUInt32BE(1); if (link) - find(link); - else + find(link, level + 1); + else { + // Doesn't have any relations, it's free + if (level === 0) + self.bufferremove.push(id); done(); + } }); }; - find(id); + find(id, 0); return self; }; GP.update = function(id, value, callback) { var self = this; - self.insert(value, callback, id - 1); + self.insert(0, value, callback, id); + return self; +}; + +GP.modify = function(id, value, callback) { + var self = this; + self.read(id, function(err, doc) { + + var data = framework_builders.isSchema(value) ? value.$clean() : value; + var keys = Object.keys(data); + var is = false; + + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + switch (key[0]) { + case '+': + case '-': + case '*': + case '/': + var tmp = key.substring(1); + if (typeof(doc[tmp]) === 'number') { + if (key[0] === '+') { + doc[tmp] += data[key]; + is = true; + } else if (key[0] === '-') { + doc[tmp] -= data[key]; + is = true; + } else if (key[0] === '*') { + doc[tmp] *= data[key]; + is = true; + } else if (key[0] === '/') { + doc[tmp] = doc[tmp] / data[key]; + is = true; + } + } + break; + default: + if (doc[key] != undefined) { + doc[key] = data[key]; + is = true; + } + break; + } + } + + if (is) + self.insert(0, doc, callback, id); + else + callback(null, id); + }); + return self; }; GP.get = GP.read = function(id, callback, type, relation) { var self = this; - if (self.$ready) { + if (self.$ready && self.reading < MAXREADERS) { + self.reading++; var buf = U.createBufferSize(self.header.size); Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { + self.reading--; var linkid = buf.readUInt32BE(1); switch (buf[0]) { - case NODE_REMOVED: - // 7: REMOVED DOCUMENT + // 255: REMOVED DOCUMENT callback(null, null, 0); return; - case 5: // LINKS + case NODE_LINKS: - var count = buf.readUInt16BE(6); // 2b + // 254: LINKS + var count = buf.readUInt16BE(9); // 2b var links = []; for (var i = 0; i < count; i++) { - var pos = 9 + (i * 6); + var pos = 11 + (i * 6); if (type == null && relation == null) - links.push({ TYPE: buf[pos], RELATION: buf[pos + 1], ID: buf.readUInt32BE(pos + 2), INDEX: i }); + links.push({ RELATION: buf[pos], TYPE: buf[pos + 1], ID: buf.readUInt32BE(pos + 2), INDEX: i }); else if ((type == null || (type == buf[pos]) && (relation == null || (relation == buf[pos + 1])))) links.push(buf.readUInt32BE(pos + 2)); } - callback(err, links, linkid); return; - case 0: + default: if (size) { - var data = buf.slice(8, buf.readUInt16BE(5) + 8); - callback(err, data ? (eval('({' + data.toString('utf8') + '})')) : null, linkid); + var data = buf.slice(11, buf.readUInt16BE(9) + 11); + if (buf[0] == 0) + callback(err, (eval('({' + data.toString('utf8') + '})')), linkid, buf[0]); + else { + var schema = self.$classes[buf[0]]; + if (schema) + callback(err, parseData(schema, data.toString('utf8').split('|')), linkid, schema.name); + else + callback(new Error('GraphDB: Class not found.'), null, 0, 0); + } } else - callback(null, null, 0); + callback(null, null, 0, 0); return; } }); } else setTimeout(self.$cb_get, DELAY, id, callback, type, relation); + + return self; +}; + +GP.find = function(type, builder) { + var self = this; + var builder = new DatabaseBuilder(self); + if (self.$ready && self.reading < MAXREADERS) + setImmediate($find, self, type, builder); + else + setTimeout(self.$cb_find, DELAY, type, builder); + return builder; +}; + +GP.find2 = function(type, builder) { + var self = this; + var builder = new DatabaseBuilder(self); + if (self.$ready && self.reading < MAXREADERS) + setImmediate($find, self, type, builder, true); + else + setTimeout(self.$cb_find, DELAY, type, builder, true); + return builder; +}; + +function $find(self, type, builder, reverse) { + + self.reading++; + + var index = reverse ? (self.header.count - 1) : 0; + var size = self.header.size * self.header.buffersize; + var filter = {}; + + filter.builder = builder; + filter.scalarcount = 0; + filter.filter = builder.makefilter(); + filter.compare = builder.compile(); + filter.index = 0; + filter.count = 0; + filter.counter = 0; + filter.first = builder.$options.first && !builder.$options.sort; + + var classes = null; + + if (type) { + + var cls; + classes = {}; + + if (type instanceof Array) { + for (var i = 0; i < type.length; i++) { + cls = self.$classes[type[i]]; + if (cls) + classes[cls.index] = 1; + } + } else { + cls = self.$classes[type]; + if (cls) + classes[cls.index] = 1; + } + } + + var reader = function() { + + var buf = U.createBufferSize(size); + var offset = HEADERSIZE + (index * self.header.size); + + if (reverse) + index -= self.header.buffersize; + else + index += self.header.buffersize; + + Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { + + if (err || !size) { + self.reading--; + framework_nosql.callback(filter); + return; + } + + while (true) { + + var buffer = buf.slice(0, self.header.size); + if (!buffer.length) + break; + + switch (buffer[0]) { + case NODE_REMOVED: + case NODE_LINKS: + break; + default: + if (!classes || classes[buffer[0]]) { + var docsize = buffer.readUInt16BE(9); + if (docsize) { + + filter.index++; + + var data = buffer.slice(11, docsize + 11).toString('utf8'); + var doc; + + if (buffer[0] == 0) + doc = (eval('({' + data + '})')); + else { + var schema = self.$classes[buffer[0]]; + if (schema) { + doc = parseData(schema, data.split('|')); + doc.CLASS = schema.name; + } + } + + if ((doc && framework_nosql.compare(filter, doc) === false) || (reverse && filter.done)) { + self.reading--; + framework_nosql.callback(filter); + return; + } + } + } + break; + } + + buf = buf.slice(self.header.size); + } + + setImmediate(reader); + }); + }; + + setImmediate(reader); +} + +function GraphDBFilter(db) { + var t = this; + t.db = db; + t.levels = null; +} + +GraphDBFilter.prototype.level = function(num) { + var self = this; + if (self.levels == null) + self.levels = {}; + return self.levels[num] = new DatabaseBuilder(self.db); +}; + +GraphDBFilter.prototype.prepare = function() { + + var self = this; + + if (!self.levels) + return self; + + var arr = Object.keys(self.levels); + + for (var i = 0; i < arr.length; i++) { + var key = arr[i]; + var builder = self.levels[key]; + var filter = {}; + filter.builder = builder; + filter.scalarcount = 0; + filter.filter = builder.makefilter(); + filter.compare = builder.compile(); + filter.index = 0; + filter.count = 0; + filter.counter = 0; + filter.first = builder.$options.first && !builder.$options.sort; + self.levels[key] = filter; + } + return self; }; -GP.graph = function(id, options, callback) { +GP.graph = function(id, options, callback, filter) { if (typeof(options) === 'function') { callback = options; @@ -870,24 +1336,68 @@ GP.graph = function(id, options, callback) { var self = this; - if (!self.$ready || self.reading) { - setTimeout(self.$cb_graph, DELAY, id, options, callback); - return self; + if (!filter) + filter = new GraphDBFilter(self); + + if (!self.$ready || self.reading >= MAXREADERS) { + setTimeout(self.$cb_graph, DELAY, id, options, callback, filter); + return filter; } - self.reading = true; + self.reading++; - self.read(id, function(err, doc, linkid) { + self.read(id, function(err, doc, linkid, cls) { if (err || !doc) { - self.reading = false; + self.reading--; callback(err, null, 0); return; } - // options.depth - // options.type - // options.relation + // options.depth (Int) + // options.relation (String or String Array) + // options.class (String or String Array) + + var relations = null; + var classes = null; + + if (options.relation) { + + var rel; + relations = {}; + + if (options.relation instanceof Array) { + for (var i = 0; i < options.relation.length; i++) { + rel = self.$relations[options.relation[i]]; + if (rel) + relations[rel.index] = rel.relation; + } + } else { + rel = self.$relations[options.relation]; + if (rel) + relations[rel.index] = rel.relation; + } + } + + if (options.class) { + + var cls; + classes = {}; + + if (options.class instanceof Array) { + for (var i = 0; i < options.class.length; i++) { + cls = self.$classes[options.class[i]]; + if (cls) + classes[cls.index] = 1; + } + } else { + cls = self.$classes[options.class]; + if (cls) + classes[cls.index] = cls.class + 1; + } + } + + filter.prepare(); var pending = []; var tmp = {}; @@ -901,6 +1411,9 @@ GP.graph = function(id, options, callback) { doc.LEVEL = 0; doc.NODES = []; + if (cls) + doc.CLASS = cls; + var reader = function(parent, id, depth) { if ((options.depth && depth > options.depth) || (tmp[id])) { @@ -915,33 +1428,51 @@ GP.graph = function(id, options, callback) { sort = true; } - tmp[linkid] = 1; - // because of seeking on HDD links.quicksort('id'); + var fil; + links.wait(function(item, next) { - if ((options.type != null && item.TYPE !== options.type) || (options.relation != null && item.RELATION !== options.relation) || tmp[item.ID]) + var key = item.ID + '-' + item.RELATION; + + if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && relations && !item.TYPE && relations[item.RELATION] === item.TYPE)) return next(); - tmp[item.ID] = 1; + tmp[key] = 1; - self.read(item.ID, function(err, doc, linkid) { + self.read(item.ID, function(err, doc, linkid, cls) { - if (doc) { - count++; + if (doc && (!classes || classes[cls])) { + count++; doc.ID = item.ID; doc.INDEX = item.INDEX; - doc.LEVEL = depth; + doc.LEVEL = depth + 1; doc.NODES = []; - doc.RELATION = item.RELATION; - doc.TYPE = item.TYPE; - parent.NODES.push(doc); + + var rel = self.$relations[item.RELATION]; + + if (rel) { + // doc.RELATION = rel.relation; + doc.RELATION = rel.name; + } + + if (cls) + doc.CLASS = cls; + + fil = filter.levels ? filter.levels[depth + 1] : null; + + if (fil) { + !fil.response && (fil.response = parent.NODES); + if (!framework_nosql.compare(fil, doc)) + linkid = null; + } else + parent.NODES.push(doc); if (linkid && !tmp[linkid]) { - pending.push({ id: linkid, parent: doc, depth: depth }); + pending.push({ id: linkid, parent: doc, depth: depth + 1 }); sort = true; } } @@ -954,6 +1485,7 @@ GP.graph = function(id, options, callback) { }; var process = function() { + if (pending.length) { // because of seeking on HDD @@ -963,9 +1495,19 @@ GP.graph = function(id, options, callback) { } var item = pending.shift(); - reader(item.parent, item.id, item.depth + 1); + reader(item.parent, item.id, item.depth); + } else { - self.reading = false; + + if (filter.levels) { + var keys = Object.keys(filter.levels); + for (var i = 0; i < keys.length; i++) { + var f = filter.levels[keys[i]]; + framework_nosql.callback(f); + } + } + + self.reading--; callback(null, doc, count); } }; @@ -975,72 +1517,180 @@ GP.graph = function(id, options, callback) { }, options.type); - return self; + return filter; }; -GP.browse = function(beg, end, callback, done) { - var self = this; +function stringify(obj) { + var val = JSON.stringify(obj).replace(REGKEY, stringifykey).replace(REGDATE, stringifydate); + return val.substring(1, val.length - 1); +} + +function stringifykey(text) { + return text.substring(1, text.length - 2) + ':'; +} - if (typeof(beg) === 'function') { - done = end; - callback = beg; - end = Number.MAX_SAFE_INTEGER; - beg = 1; - } else if (typeof(end) === 'function') { - done = callback; - callback = end; - end = Number.MAX_SAFE_INTEGER; +function stringifydate(text) { + return 'new Date(' + new Date(text.substring(1, text.length - 2)).getTime() + ')'; +} + +function stringifyData(schema, doc) { + + var output = ''; + var esc = false; + var size = 0; + + for (var i = 0; i < schema.keys.length; i++) { + var key = schema.keys[i]; + var meta = schema.meta[key]; + var val = doc[key]; + + switch (meta.type) { + case 1: // String + val = val ? val : ''; + size += 4; + break; + case 2: // Number + val = (val || 0); + size += 2; + break; + case 3: // Boolean + val = (val == true ? '1' : '0'); + break; + case 4: // Date + // val = val ? val.toISOString() : ''; + val = val ? val.getTime() : ''; + !val && (size += 13); + break; + case 5: // Object + val = val ? JSON.stringify(val) : ''; + size += 4; + break; + } + + if (!esc && (meta.type === 1 || meta.type === 5)) { + val += ''; + if (REGTESCAPETEST.test(val)) { + esc = true; + val = val.replace(REGTESCAPE, regtescape); + } + } + + output += '|' + val; } - if (self.$ready && !self.reading) { - var counter = 1; - self.reading = true; - var reader = function() { - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((beg - 1) * self.header.size), function(err, size) { + return (esc ? '*' : '+') + output; +} +function parseSchema(schema) { - if (err || !size) { - self.reading = false; - done && done(); - return; - } + var obj = {}; + var arr = schema.split('|').trim(); - var output; + obj.meta = {}; + obj.keys = []; + obj.raw = schema; - switch (buf[0]) { - case NODE_REMOVED: - case NODE_LINKS: - break; - case 1: - case 0: - output = callback(err, (eval('({' + buf.slice(8, buf.readUInt16BE(5) + 8).toString('utf8') + '})')), buf.readUInt32BE(1), counter); - break; - } + for (var i = 0; i < arr.length; i++) { + var arg = arr[i].split(':'); + var type = 0; + switch ((arg[1] || '').toLowerCase().trim()) { + case 'number': + type = 2; + break; + case 'boolean': + case 'bool': + type = 3; + break; + case 'date': + type = 4; + break; + case 'object': + type = 5; + break; + case 'string': + default: + type = 1; + break; + } + var name = arg[0].trim(); + obj.meta[name] = { type: type, pos: i }; + obj.keys.push(name); + } - if (output === false || beg >= end) { - self.reading = false; - done && done(); - } else { - counter++; - beg++; - reader(); - } - }); - }; - reader(); - } else - setTimeout(self.$cb_browse, DELAY, beg, end, callback, done); - return self; -}; + return obj; +} -function stringify(obj) { - var val = JSON.stringify(obj).replace(/"[a-z-0-9]+":/gi, stringifyhelper); - return val.substring(1, val.length - 1); +function parseData(schema, lines, cache) { + + var obj = {}; + var esc = lines === '*'; + var val; + + for (var i = 0; i < schema.keys.length; i++) { + var key = schema.keys[i]; + + if (cache && cache !== EMPTYOBJECT && cache[key] != null) { + obj[key] = cache[key]; + continue; + } + + var meta = schema.meta[key]; + if (meta == null) + continue; + + var pos = meta.pos + 1; + + switch (meta.type) { + case 1: // String + obj[key] = lines[pos]; + if (esc && obj[key]) + obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); + break; + case 2: // Number + obj[key] = +lines[pos]; + break; + case 3: // Boolean + val = lines[pos]; + obj[key] = BOOLEAN[val] == 1; + break; + case 4: // Date + val = lines[pos]; + obj[key] = val ? new Date(val[10] === 'T' ? val : +val) : null; + break; + case 5: // Object + val = lines[pos]; + if (esc && val) + val = val.replace(REGTUNESCAPE, regtescapereverse); + obj[key] = val ? val.parseJSON(true) : null; + break; + } + } + + return obj; } -function stringifyhelper(text) { - return text.substring(1, text.length - 2) + ':'; +function regtescapereverse(c) { + switch (c) { + case '%0A': + return '\n'; + case '%0D': + return '\r'; + case '%7C': + return '|'; + } + return c; +} + +function regtescape(c) { + switch (c) { + case '\n': + return '%0A'; + case '\r': + return '%0D'; + case '|': + return '%7C'; + } + return c; } exports.load = function(name, size) { diff --git a/nosql.js b/nosql.js index 64450f581..4889d4fc7 100755 --- a/nosql.js +++ b/nosql.js @@ -822,9 +822,9 @@ DP.modify = function(doc, insert) { var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; var keys = Object.keys(data); + var inc = null; if (keys.length) { - var inc = null; for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { @@ -2603,6 +2603,10 @@ function DatabaseBuilder(db) { DatabaseBuilder.prototype.promise = promise; +DatabaseBuilder.prototype.makefilter = function() { + return { repository: this.$repository, options: this.$options, arg: this.$params, fn: this.$functions }; +}; + DatabaseBuilder.prototype.id = function(id) { this.$options.id = id; return this; @@ -6947,4 +6951,142 @@ function errorhandling(err, builder, response) { function jsonparser(key, value) { return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; -} \ No newline at end of file +} + +// Item requirements: +// item.first = false; +// item.scalarcount = 0; +// item.builder = builder; +// item.compare = builder.compile(); +// item.filter = builder.makefilter(); +// item.index = DOCUMENT_COUNTER; + +exports.compare = function(item, obj) { + + var val; + var builder = item.builder; + + item.filter.index = item.index; + + var output = item.compare(obj, item.filter, item.index); + if (!output) + return; + + item.count++; + + if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) + return; + + item.counter++; + + if (!builder.$inlinesort && !item.done) + item.done = builder.$options.take && builder.$options.take <= item.counter; + + if (item.type) + return; + + switch (builder.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[builder.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[builder.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'max': + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'avg': + val = output[builder.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[builder.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (builder.$inlinesort) + nosqlinlinesorter(item, builder, output); + else if (item.response) + item.response.push(output); + else + item.response = [output]; + break; + } + + return item.first ? false : true; +}; + +exports.callback = function(item, err) { + + var builder = item.builder; + var output; + + if (builder.$options.scalar || !builder.$options.sort) { + + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + + builder.$callback2(errorhandling(err, builder, output), item.type === 1 ? item.count : output, item.count); + return; + } + + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + + builder.$callback2(errorhandling(err, builder, output), item.type === 1 ? item.count : output, item.count); + builder.done(); +}; \ No newline at end of file From 109f0fd8493cc0de205e6033f1382926f6aa9e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Jul 2018 23:25:37 +0200 Subject: [PATCH 0484/1669] Improved code. --- graphdb.js | 92 ++++++++++++++++++++++++++++++++++++++++++++++++------ nosql.js | 3 +- 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/graphdb.js b/graphdb.js index e81265b63..6108af594 100644 --- a/graphdb.js +++ b/graphdb.js @@ -33,10 +33,6 @@ CLASSES RAW JSON "Removed" : TYPE (255) @Int8 */ -// @TODO: -// - check?? if the link doesn't exist -// - removing documents and links (problem) - const DEFSIZE = 80; const Fs = require('fs'); const VERSION = 1; @@ -182,6 +178,72 @@ function GraphDB(name, size) { var GP = GraphDB.prototype; +function extendclass(self, type, old) { + + if (self.$ready && self.reading < MAXREADERS) { + + var size = self.header.size * self.header.buffersize; + var index = 0; + var now = Date.now(); + var cls = self.$classes[type]; + + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" extending class "{1}" (beg)'.format(self.name, cls.name)); + + old = parseSchema(old); + self.reading++; + + var errhandling = F.error(); + + var reader = function() { + + var buf = U.createBufferSize(size); + var offset = HEADERSIZE + (index * self.header.size); + var current = index; + + index += self.header.buffersize; + + Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { + + if (err || !size) { + self.reading--; + self.flushmeta(); + F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" extending class "{1}" (end, {2}s)'.format(self.name, cls.name, (((Date.now() - now) / 1000) >> 0))); + return; + } + + while (true) { + + var buffer = buf.slice(0, self.header.size); + if (!buffer.length) + break; + + current++; + + switch (buffer[0]) { + case NODE_REMOVED: + case NODE_LINKS: + break; + case type: + var docsize = buffer.readUInt16BE(9); + if (docsize) { + var data = parseData(old, buffer.slice(11, docsize + 11).toString('utf8').split('|')); + self.insert(type, data, errhandling, current); + } + break; + } + + buf = buf.slice(self.header.size); + } + + setImmediate(reader); + }); + }; + + setImmediate(reader); + } else + setTimeout(extendclass, DELAY, self, type, old); +} + GP.class = function(name, declaration, indexer) { var self = this; @@ -194,9 +256,9 @@ GP.class = function(name, declaration, indexer) { if (cls) { if (cls.raw !== meta.raw) { - // schema is changed - // @todo: re-schemas all classed documents - console.log('SCHEMA IS CHANGED'); + meta.index = cls.index; + self.$classes[cls.index] = self.$classes[name] = meta; + extendclass(self, meta.index, cls.raw, meta.raw); } } else { @@ -1051,7 +1113,18 @@ GP.remove = function(id, callback) { GP.update = function(id, value, callback) { var self = this; - self.insert(0, value, callback, id); + self.read(id, function(err, doc, linkid, cls) { + if (err || !doc) + callback(err, null); + else + self.insert(cls, typeof(value) === 'function' ? value(doc) : doc, callback, id); + }); + return self; +}; + +GP.update2 = function(cls, id, value, callback) { + var self = this; + self.insert(cls, value, callback, id); return self; }; @@ -1647,7 +1720,8 @@ function parseData(schema, lines, cache) { obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); break; case 2: // Number - obj[key] = +lines[pos]; + val = +lines[pos]; + obj[key] = val < 0 || val > 0 ? val : 0; break; case 3: // Boolean val = lines[pos]; diff --git a/nosql.js b/nosql.js index 4889d4fc7..ec03c396a 100755 --- a/nosql.js +++ b/nosql.js @@ -6766,7 +6766,8 @@ TP.parseData = function(data, cache) { obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); break; case 2: // Number - obj[key] = +data.line[pos]; + val = +data.line[pos]; + obj[key] = val < 0 || val > 0 ? val : 0; break; case 3: // Boolean val = data.line[pos]; From ee490c075c5d83bf68adcdf342e3c816e4e8c861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 11 Jul 2018 23:38:22 +0200 Subject: [PATCH 0485/1669] Improved error handling in GraphDB. --- graphdb.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/graphdb.js b/graphdb.js index 6108af594..89bc09ed1 100644 --- a/graphdb.js +++ b/graphdb.js @@ -33,7 +33,7 @@ CLASSES RAW JSON "Removed" : TYPE (255) @Int8 */ -const DEFSIZE = 80; +const DEFSIZE = 500; const Fs = require('fs'); const VERSION = 1; const EMPTYBUFFER = U.createBufferSize(1); @@ -244,6 +244,11 @@ function extendclass(self, type, old) { setTimeout(extendclass, DELAY, self, type, old); } +GP.errorhandling = function(err, type) { + console.log('GraphDB "{0}" --> "{1}" error: {2}'.format(name, err, type)); + this.$events.error && this.emit('error', err); +}; + GP.class = function(name, declaration, indexer) { var self = this; @@ -460,7 +465,7 @@ GP.clean = function(callback) { // WRITE Fs.write(self.fd, buffer, 0, buffer.length, position, function(err) { - err && console.log(err); + err && self.errorhandling(err, 'clean.write'); next(index + 1); }); @@ -580,11 +585,10 @@ GP.resize = function(docSize, callback) { Fs.read(self.fd, buf, 0, buf.length, index, function(err, size) { if (size) { var w = U.createBufferSize(docSize); - buf = buf.slice(0, buf.indexOf(EMPTYBUFFER)); w.fill(buf, 0, buf.length); index += self.header.size; Fs.write(fd, w, 0, w.length, offset, function(err, size) { - err && self.$events.error && self.emit('error', err); + err && self.errorhandling(err, 'resize.write'); offset += size; next(done); }); @@ -598,7 +602,7 @@ GP.resize = function(docSize, callback) { Fs.close(self.fd, function() { Fs.rename(self.filename + '-tmp', self.filename, function(err) { F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" resizing (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); - err && self.$events.error && self.emit('error', err); + err && self.errorhandling(err, 'resize.rename'); callback && callback(err); }); }); @@ -637,7 +641,7 @@ GP.flushheader = function(callback, fd) { buf.fill(str, 13, str.length + 13); Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { - err && console.log(err); + err && self.errorhandling(err, 'flushheader.write'); if (fd) self.flushmeta(callback, fd); else @@ -687,7 +691,7 @@ GP.flushmeta = function(callback, fd) { buf.fill(data, 0, data.length); Fs.write(fd || self.fd, buf, 0, buf.length, INFOSIZE, function(err) { - err && console.log(err); + err && self.errorhandling(err, 'flushmeta.write'); callback && callback(); }); @@ -859,7 +863,7 @@ GP.setLinkId = function(id, linkid, callback) { var buf = U.createBufferSize(4); buf.writeUInt32BE(linkid); Fs.write(self.fd, buf, 0, buf.length, pos + 1, function(err) { - err && self.$events.error && self.emit('error', err); + err && self.errorhandling(err, 'setLinkId.write'); callback && callback(err); }); From 2e045f0f4179189b25a61ed0965170b74363e04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 12 Jul 2018 15:51:34 +0200 Subject: [PATCH 0486/1669] Uncompleted changes. --- graphdb.js | 311 +++++++++++++++++++++++++++++++++++------------ test/test-tmp.js | 138 ++++++++++++++++++--- 2 files changed, 358 insertions(+), 91 deletions(-) diff --git a/graphdb.js b/graphdb.js index 89bc09ed1..1f4ba4387 100644 --- a/graphdb.js +++ b/graphdb.js @@ -33,7 +33,7 @@ CLASSES RAW JSON "Removed" : TYPE (255) @Int8 */ -const DEFSIZE = 500; +const DEFSIZE = 40; const Fs = require('fs'); const VERSION = 1; const EMPTYBUFFER = U.createBufferSize(1); @@ -60,7 +60,7 @@ function GraphDB(name, size) { var t = this; t.name = name; - t.filename = F.path.databases(name + '.db'); + t.filename = F.path.databases(name + '.gdb'); t.$ready = false; t.$size = size || DEFSIZE; @@ -74,7 +74,8 @@ function GraphDB(name, size) { t.$events = {}; t.$classes = {}; t.$relations = {}; - t.writing = false; + t.appending = false; + t.updating = false; t.reading = 0; t.cleaning = false; @@ -88,7 +89,7 @@ function GraphDB(name, size) { } t.stat.mtime = NOW; - t.writing = false; + t.updating = false; t.flush(); }; @@ -101,7 +102,7 @@ function GraphDB(name, size) { F.error(err, 'GraphDB: ' + t.name); } - t.writing = false; + t.appending = false; t.stat.size += size; t.stat.atime = NOW; t.header.count += (size / t.header.size) >> 0; @@ -134,7 +135,7 @@ function GraphDB(name, size) { }; t.$cb_pushlinkid = function(id, toid, type, relation, callback, parentid) { - pushLinkId(t, id, toid, type, relation, callback, parentid); + pushRelation(t, id, toid, type, relation, callback, parentid); }; t.$cb_getlinkid = function(id, callback) { @@ -172,6 +173,10 @@ function GraphDB(name, size) { t.find(type, builder); }; + t.$flush = function() { + t.flush(); + } + F.path.verify('databases'); t.open(); } @@ -240,16 +245,17 @@ function extendclass(self, type, old) { }; setImmediate(reader); - } else + } else { setTimeout(extendclass, DELAY, self, type, old); + } } GP.errorhandling = function(err, type) { - console.log('GraphDB "{0}" --> "{1}" error: {2}'.format(name, err, type)); + console.log('GraphDB "{0}" --> "{1}" error: {2}'.format(this.name, err, type)); this.$events.error && this.emit('error', err); }; -GP.class = function(name, declaration, indexer) { +GP.class = function(name, declaration, indexer, nodeid) { var self = this; if (self.$ready || indexer) { @@ -262,6 +268,7 @@ GP.class = function(name, declaration, indexer) { if (cls) { if (cls.raw !== meta.raw) { meta.index = cls.index; + meta.nodeid = cls.nodeid; self.$classes[cls.index] = self.$classes[name] = meta; extendclass(self, meta.index, cls.raw, meta.raw); } @@ -271,7 +278,9 @@ GP.class = function(name, declaration, indexer) { if (indexer == null) { self.header.classindex++; index = self.header.classindex; - } + meta.nodeid = nodeid ? nodeid : makeRelation(self); + } else + meta.nodeid = nodeid; meta.index = index; self.$classes[index] = self.$classes[name] = meta; @@ -280,8 +289,9 @@ GP.class = function(name, declaration, indexer) { self.$flushmeta(); } - } else + } else { setTimeout(self.$cb_class, DELAY, name, declaration, indexer); + } return self; }; @@ -297,7 +307,7 @@ GP.classes = function(callback) { return self; }; -GP.relation = function(name, both, indexer) { +GP.relation = function(name, both, indexer, nodeid) { var self = this; if (self.$ready || indexer) { @@ -316,6 +326,7 @@ GP.relation = function(name, both, indexer) { if (indexer == null) { self.header.relationindex++; index = self.header.relationindex; + rel.nodeid = nodeid ? nodeid : makeRelation(self); } rel.index = index; @@ -327,8 +338,9 @@ GP.relation = function(name, both, indexer) { self.$flushmeta(); } - } else + } else { setTimeout(self.$cb_relation, DELAY, name, both, indexer); + } return self; }; @@ -405,7 +417,8 @@ GP.clean = function(callback) { self.bufferremove.push(arr[i]); } self.header.scan = false; - self.writing = false; + self.appending = false; + self.updating = false; self.flushheader(); callback && callback(null, removed); F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); @@ -670,7 +683,7 @@ GP.flushmeta = function(callback, fd) { if (self.header.classindex) { for (var i = 0; i < self.header.classindex; i++) { var item = self.$classes[i + 1]; - item && meta.c.push({ i: i + 1, n: item.name, r: item.raw }); + item && meta.c.push({ i: i + 1, n: item.name, r: item.raw, x: item.nodeid }); } var b = U.createBufferSize(1); b.writeUInt8(self.header.classindex); @@ -680,7 +693,7 @@ GP.flushmeta = function(callback, fd) { if (self.header.relationindex) { for (var i = 0; i < self.header.relationindex; i++) { var item = self.$relations[i + 1]; - item && meta.r.push({ i: i + 1, n: item.name, r: item.relation }); + item && meta.r.push({ i: i + 1, n: item.name, r: item.relation, x: item.nodeid }); } var b = U.createBufferSize(1); b.writeUInt8(self.header.relationindex); @@ -695,6 +708,7 @@ GP.flushmeta = function(callback, fd) { callback && callback(); }); + self.flush(); return self; }; @@ -724,18 +738,19 @@ GP.$open = function() { var data = buf.slice(0, buf.indexOf(EMPTYBUFFER)).toString('utf8').parseJSON(); if (data) { + // registering classes if (data.r instanceof Array) { for (var i = 0; i < data.c.length; i++) { var item = data.c[i]; - self.class(item.n, item.r, item.i); + self.class(item.n, item.r, item.i, item.x); } } // registering relations if (data.r instanceof Array) { for (var i = 0; i < data.r.length; i++) { var item = data.r[i]; - self.relation(item.n, item.r, item.i); + self.relation(item.n, item.r, item.i, item.x); } } } @@ -786,15 +801,19 @@ GP.insert = function(type, value, callback, id) { if (self.$ready) { var val; + var schema; if (type) { - var schema = self.$classes[type]; + schema = self.$classes[type]; + if (!schema) { callback && callback(new Error('GraphDB: Schema "{0}" not found.'.format(name))); return self; } + val = U.createBuffer(stringifyData(schema, value)); type = schema.index; + } else val = U.createBuffer(stringify(value)); @@ -826,11 +845,12 @@ GP.insert = function(type, value, callback, id) { res.writeUInt16BE(val.length, insert ? 9 : 0); if (insert) { - self.bufferappend.push(res); - self.$events.insert && self.emit('insert', id, value); + self.bufferappend.push({ id: id, buf: res, callback: callback, schemaid: schema ? type : 0 }); + self.$events.insert && self.emit('insert', schema ? schema.name : type, id, value); + // schema && schema.nodeid && pushRelationNode(self, schema, id); } else { - self.bufferupdate.push({ buf: res, pos: id - 1, type: type, recovered: recovered }); + self.bufferupdate.push({ buf: res, id: id, type: type, recovered: recovered, offset: 9, callback: callback }); // because of seeking on HDD if (self.bufferupdate.length > 1) @@ -839,10 +859,11 @@ GP.insert = function(type, value, callback, id) { self.$events.update && self.emit('update', id, value); } - self.flush(); - callback && callback(null, id); - } else + setImmediate(self.$flush); + + } else { setTimeout(self.$cb_insert, DELAY, type, value, callback, id); + } return self; }; @@ -870,16 +891,76 @@ GP.setLinkId = function(id, linkid, callback) { return self; }; -function pushLinkId(self, id, toid, type, initializer, callback, parentid) { +GP.findRelation = function(metaid, preparator) { + var self = this; + self.read(metaid, function(err, doc, linkid) { + preparator(doc || [], linkid ? () => self.findRelation(linkid, preparator) : null); + }); + return self; +}; + +GP.findRelationCount = function(cls, callback, count) { + var self = this; + + if (!count) + count = 0; + + self.read(typeof(cls) === 'number' ? cls : self.$classes[cls].nodeid, function(err, doc, linkid) { + count += doc.length; + if (linkid) + self.findRelationCount(linkid, callback, count); + else + callback(err, count); + }); + + return self; +}; + +function makeRelation(self) { + var id = self.index++; + var buf = U.createBufferSize(self.header.size); + buf.writeUInt8(NODE_LINKS); // type, 1b + buf.writeUInt32BE(0, 1); // link, 4b + buf.writeUInt32BE(0, 5); // parent, 4b + buf.writeUInt16BE(0, 9); // count, 2b + + self.bufferappend.push({ id: id, buf: buf }); + self.flush(); + return id; +} + +function pushRelationNode(self, item, id) { + + // Must contain: + // item.nodeid + + self.pending[item.nodeid] = 1; + + pushRelation(self, item.nodeid, id, 0, 0, function(err, nodeid) { + + delete self.pending[item.nodeid]; + + err && self.errorhandling(err, 'pushRelationNode'); + + if (item.nodeid !== nodeid) { + item.nodeid = nodeid; + self.$flushmeta(); + } + + }); +} + +function pushRelation(self, id, toid, type, initializer, callback, parentid) { var pos = HEADERSIZE + ((id - 1) * self.header.size); var buf = U.createBufferSize(self.header.size); - var max = (self.header.size - 11) / 6 >> 0; // 6b --> TYPE 1b + RELATION 1b + ID 4b if (id == null) { id = self.index++; + console.log('CREATE:', 'id:' + id, 'toid:' + toid, 'parentid:' + parentid); + buf.writeUInt8(NODE_LINKS); // type, 1b buf.writeUInt32BE(parentid || 0, 1); // link, 4b buf.writeUInt32BE(id, 5); // parent, 4b @@ -889,47 +970,99 @@ function pushLinkId(self, id, toid, type, initializer, callback, parentid) { buf.writeUInt8(initializer ? 1 : 0, 12); // RELATION TYPE buf.writeUInt32BE(toid, 13); - self.bufferappend.push(buf); + self.bufferappend.push({ id: id, buf: buf }); self.flush(); callback && callback(null, id); } else { - Fs.read(self.fd, buf, 0, buf.length, pos, function(err) { + var cache = self.bufferupdate.findItem('id', id); + if (cache == null) + cache = self.bufferappend.findItem('id', id); - if (err) { - callback(err); - return; - } else if (buf[0] !== NODE_LINKS) { - callback(new Error('GraphDB: Invalid value for linking.')); - return; - } + var max = ((self.header.size - 11) / 6 >> 0); // 6b --> TYPE 1b + RELATION 1b + ID 4b + var update = function(buf, isCache) { var count = buf.readUInt16BE(9); + if (count + 1 >= max) { - // we need to create a new record because existing buffer is full - pushLinkId(self, null, toid, type, initializer, function(err, id) { - // we return a new id + + console.log('INSERT:', 'id:' + id, 'toid:' + toid); + + // We need to create a new record because existing buffer is full + pushRelation(self, null, toid, type, initializer, function(err, id) { + // We return a new id callback && callback(err, id); }, id); + } else { + + // We add a new relation to existing list buf.writeUInt16BE(count + 1, 9); // count, 2b + var off = 11 + (count * 6); + buf.writeUInt8(type, off); buf.writeUInt8(initializer ? 1 : 0, off + 1); buf.writeUInt32BE(toid, off + 2); - Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { - callback && callback(err, id); - }); + + if (!isCache) + self.bufferupdate.push({ id: id, buf: buf, offset: 0 }); + + callback && callback(null, id); } - }); + }; + + if (cache) { + update(cache.buf); + } else { + Fs.read(self.fd, buf, 0, buf.length, pos, function(err) { + if (err) { + callback(err); + return; + } else if (buf[0] !== NODE_LINKS) { + callback(new Error('GraphDB: Invalid value for linking.')); + return; + } + update(buf); + }); + } } return self; } -GP.join = function(type, fromid, toid, callback) { +function updateRelation(self, metaid, toid, type, initializer) { + + var pos = HEADERSIZE + ((metaid - 1) * self.header.size); + var buf = U.createBufferSize(self.header.size); + + Fs.read(self.fd, buf, 0, buf.length, pos, function(err, size) { + + if (err || !size) + return; + + var count = buf.readUInt16BE(9); + + for (var i = 0; i < count; i++) { + var off = 11 + (i * 6); + if (buf.readUInt32BE(off + 2) === toid) { + // we have relation + // update + var data = U.createBufferSize(2); + data.writeUInt8(type == null ? buf[off] : type); + data.writeUInt8(initializer == null ? buf[off + 1] : initializer, 1); + Fs.write(self.fd, data, 0, data.length, pos + off, function(err) { + err && self.errorhandling(err, 'updateRelation.write'); + }); + } + } + + }); +} + +GP.connect = function(type, fromid, toid, callback) { var self = this; @@ -941,6 +1074,11 @@ GP.join = function(type, fromid, toid, callback) { return self; } + if (relation.nodeid) { + pushRelationNode(self, relation, fromid); + relation.relation && pushRelationNode(self, relation, toid); + } + self.pending[fromid] = 1; self.pending[toid] = 1; @@ -985,8 +1123,7 @@ GP.join = function(type, fromid, toid, callback) { // relation: 0 - "toid" doesn't know "fromid" async.push(function(next) { - - pushLinkId(self, aid == 0 ? null : aid, toid, relation.index, true, function(err, id) { + pushRelation(self, aid == 0 ? null : aid, toid, relation.index, true, function(err, id) { if (aid !== id) { aid = id; self.setLinkId(fromid, id, next); @@ -996,7 +1133,7 @@ GP.join = function(type, fromid, toid, callback) { }); async.push(function(next) { - pushLinkId(self, bid == 0 ? null : bid, fromid, relation.index, false, function(err, id) { + pushRelation(self, bid == 0 ? null : bid, fromid, relation.index, false, function(err, id) { if (bid !== id) { bid = id; self.setLinkId(toid, id, next); @@ -1009,10 +1146,12 @@ GP.join = function(type, fromid, toid, callback) { delete self.pending[fromid]; delete self.pending[toid]; callback && callback(null, aid, bid); + self.flush(); }); - } else + } else { setTimeout(self.$cb_join, DELAY, type, fromid, toid, callback); + } return self; }; @@ -1032,13 +1171,13 @@ GP.flush = function() { var self = this; - if (!self.$ready || self.writing) + if (!self.$ready) return self; - if (self.bufferupdate.length) { - self.writing = true; + if (self.bufferupdate.length && !self.updating) { + self.updating = true; var doc = self.bufferupdate.shift(); - var offset = HEADERSIZE + (doc.pos * self.header.size); + var offset = HEADERSIZE + ((doc.id - 1) * self.header.size); if (doc.recovered) { var buf = U.createBufferSize(1); buf.writeUInt8(doc.type); @@ -1049,18 +1188,35 @@ GP.flush = function() { else F.error(err, 'GraphDB: ' + self.name); } - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 9, self.$cb_writeupdate); + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + doc.offset, self.$cb_writeupdate); }); } else - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + 9, self.$cb_writeupdate); - return self; - } else if (!self.bufferappend.length) + Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + doc.offset, self.$cb_writeupdate); + doc.callback && doc.callback(null, doc.id); return self; + } + + if (self.bufferappend.length && !self.appending) { + + var flush = self.bufferappend.splice(0, self.header.buffersize); + var buf = []; + + for (var i = 0; i < flush.length; i++) { + var f = flush[i]; + buf.push(f.buf); + f.callback && f.callback(null, f.id); + + if (f.schemaid) { + console.log('PUSHID', f.id); + pushRelationNode(self, self.$classes[f.schemaid], f.id); + } + } + + var data = Buffer.concat(buf); + self.appending = true; + Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); + } - var buf = self.bufferappend.splice(0, self.header.buffersize); - var data = Buffer.concat(buf); - self.writing = true; - Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); return self; }; @@ -1212,6 +1368,8 @@ GP.get = GP.read = function(id, callback, type, relation) { else if ((type == null || (type == buf[pos]) && (relation == null || (relation == buf[pos + 1])))) links.push(buf.readUInt32BE(pos + 2)); } + + links.reverse(); callback(err, links, linkid); return; @@ -1232,8 +1390,9 @@ GP.get = GP.read = function(id, callback, type, relation) { return; } }); - } else + } else { setTimeout(self.$cb_get, DELAY, id, callback, type, relation); + } return self; }; @@ -1243,8 +1402,9 @@ GP.find = function(type, builder) { var builder = new DatabaseBuilder(self); if (self.$ready && self.reading < MAXREADERS) setImmediate($find, self, type, builder); - else + else { setTimeout(self.$cb_find, DELAY, type, builder); + } return builder; }; @@ -1253,15 +1413,14 @@ GP.find2 = function(type, builder) { var builder = new DatabaseBuilder(self); if (self.$ready && self.reading < MAXREADERS) setImmediate($find, self, type, builder, true); - else + else { setTimeout(self.$cb_find, DELAY, type, builder, true); + } return builder; }; function $find(self, type, builder, reverse) { - self.reading++; - var index = reverse ? (self.header.count - 1) : 0; var size = self.header.size * self.header.buffersize; var filter = {}; @@ -1278,10 +1437,8 @@ function $find(self, type, builder, reverse) { var classes = null; if (type) { - var cls; classes = {}; - if (type instanceof Array) { for (var i = 0; i < type.length; i++) { cls = self.$classes[type[i]]; @@ -1308,7 +1465,6 @@ function $find(self, type, builder, reverse) { Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { if (err || !size) { - self.reading--; framework_nosql.callback(filter); return; } @@ -1344,7 +1500,6 @@ function $find(self, type, builder, reverse) { } if ((doc && framework_nosql.compare(filter, doc) === false) || (reverse && filter.done)) { - self.reading--; framework_nosql.callback(filter); return; } @@ -1458,19 +1613,19 @@ GP.graph = function(id, options, callback, filter) { if (options.class) { - var cls; + var clstmp; classes = {}; if (options.class instanceof Array) { for (var i = 0; i < options.class.length; i++) { - cls = self.$classes[options.class[i]]; - if (cls) - classes[cls.index] = 1; + clstmp = self.$classes[options.class[i]]; + if (clstmp) + classes[clstmp.index] = 1; } } else { - cls = self.$classes[options.class]; - if (cls) - classes[cls.index] = cls.class + 1; + clstmp = self.$classes[options.class]; + if (clstmp) + classes[clstmp.index] = clstmp.class + 1; } } @@ -1498,6 +1653,8 @@ GP.graph = function(id, options, callback, filter) { return; } + tmp[id] = 1; + self.read(id, function(err, links, linkid) { if (linkid && !tmp[linkid]) { @@ -1514,7 +1671,7 @@ GP.graph = function(id, options, callback, filter) { var key = item.ID + '-' + item.RELATION; - if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && relations && !item.TYPE && relations[item.RELATION] === item.TYPE)) + if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && !item.TYPE && !relations) || (relations && relations[item.RELATION] === item.TYPE)) return next(); tmp[key] = 1; diff --git a/test/test-tmp.js b/test/test-tmp.js index 4471b4b33..15712cb79 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -2,41 +2,151 @@ require('../index'); const Fs = require('fs'); -var insert = 0; +var insert = 1; var link = 0; var update = 0; var read = 0; -var remove = 1; +var remove = 0; if (insert) { try { - Fs.unlinkSync(F.path.databases('skuska.db')); + Fs.unlinkSync(F.path.databases('skuska.gdb')); } catch (e) {} } var db = GRAPHDB('skuska'); db.ready(function() { + + db.class('user', 'name:string|age:number'); + db.class('order', 'id:string|price:number'); + db.class('tags', 'name:string'); + db.relation('like', true); + // this.insert('Si kokot?', console.log); //this.count(console.log); //this.read(20303, console.log); + /* + setTimeout(function() { + db.connect('like', 8, 9); + setTimeout(function() { + db.read(2, console.log); + // db.read(6, console.log); + // db.read(13, console.log); + // var opt = {}; + // opt.relation = 'like'; + // opt.class = 'fet'; + // db.graph(8, opt, function(err, doc) { + // console.log(JSON.stringify(doc, null, ' ')); + // }); + }, 500); + }, 500); + */ + + setTimeout(function() { + var count = 0; + + //db.findRelationCount('order', (err, count) => console.log('order', count)); + // db.findRelationCount('user', (err, count) => console.log('user', count)); + // db.findRelationCount('tags', (err, count) => console.log('tags', count)); + + // db.findRelation(db.$classes.order.nodeid, function(id, next) { + + // for (var i = 0; i < id.length; i++) { + // var a = ordersid[id[i].ID]; + + // if (a == null) + // console.log('MISSING', id[i]); + // else + // ordersid[id[i].ID]++; + // } + + // if (next) + // next(); + // else + // console.log(ordersid); + // }); + + //db.read(34, (err, doc) => console.log(34, doc)); + //db.read(35, (err, doc) => console.log(35, doc)); + + // db.read(34, (err, doc) => console.log(34, doc.length)); + // db.read(35, (err, doc) => console.log(35, doc.length)); + // db.read(36, (err, doc) => console.log(36, doc.length)); + // db.read(37, (err, doc) => console.log(37, doc.length)); + + // db.find('user').callback((err, docs) => console.log('users', docs.length)); + // db.find('order').callback((err, docs) => console.log('orders', docs.length)); + // db.find('tags').callback((err, docs) => console.log('tags', docs.length)); + + }, 500); + + // db.find('user').take(100).callback((err, docs, count) => console.log('users', docs.length, count)); + // db.find('order').take(100).callback((err, docs, count) => console.log('orders', docs.length, count)); + // db.find('tags').take(100).callback((err, docs, count) => console.log('tags', docs.length, count)); + + // setTimeout(function() { + // db.find2('user').take(100).callback(console.log); + // }, 100); + + // var sum = 0; + // db.findRelation(db.$classes.user.nodeid, function(id, next) { + // sum += id.length; + // console.log(id); + // if (next) + // next(); + // else + // console.log('--->', sum); + // }); + //console.log(db); if (insert) { - for (var i = 0; i < 1000000; i++) { - if (i % 100000 === 0) - console.log(i); - db.insert({ index: i + 1 }); - } + + var max = 10; + + (max).async(function(i, next) { + if (i && i % 1000 === 0) + console.log('user', i); + db.insert('user', { name: GUID(5), age: i + 20 }, next); + }, function() { + console.log('user done'); + }); +/* + (max).async(function(i, next) { + if (i && i % 100000 === 0) + console.log('order', i); + db.insert('order', { id: UID(), price: (i + 1) }, next); + }, function() { + console.log('order done'); + }); + + (max).async(function(i, next) { + + if (i && i % 100000 === 0) + console.log('tags', i); + + db.insert('tags', { name: GUID(5) }, next); + }, function() { + console.log('tags done'); + });*/ + } - db.read(1000001, (err, links) => console.log(1000001, links)); - db.read(1000002, (err, links) => console.log(1000002, links)); - db.read(1000003, (err, links) => console.log(1000003, links)); - db.read(1000004, (err, links) => console.log(1000004, links)); - db.read(1000005, (err, links) => console.log(1000005, links)); - db.read(1000006, (err, links) => console.log(1000006, links)); + // db.read(1, console.log); + + // db.read(11, console.log); + // null [ { RELATION: 1, TYPE: 1, ID: 5, INDEX: 0 } ] 0 + + // db.connect('like', 1, 5); + + // db.read(1000001, (err, links) => console.log(1000001, links)); + // db.read(1000002, (err, links) => console.log(1000002, links)); + // db.read(1000003, (err, links) => console.log(1000003, links)); + // db.read(1000004, (err, links) => console.log(1000004, links)); + // db.read(1000005, (err, links) => console.log(1000005, links)); + // db.read(1000006, (err, links) => console.log(1000006, links)); //db.link(100, 1000, 1, 0); //db.link(100, 10000, 1, 0); From 0da36b50c66136966bc35667399b93e43d441dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 13 Jul 2018 08:25:52 +0200 Subject: [PATCH 0487/1669] Improved Proxy error. --- utils.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 319c4dafb..bd00468f7 100755 --- a/utils.js +++ b/utils.js @@ -689,14 +689,24 @@ ProxyAgent.prototype.createSocket = function(options, callback) { if (res.statusCode === 200) { callback(socket); } else { - var err = new Error('Proxy could not be established, code: ' + res.statusCode); + var err = new Error('Proxy could not be established (maybe a problem in auth), code: ' + res.statusCode); err.code = 'ECONNRESET'; options.request.emit('error', err); + req.destroy && req.destroy(); + req = null; + self.requests = null; + self.options = null; } }); req.on('error', function(err) { - options.request.emit('error', err); + var e = new Error('Request Proxy "proxy {0} --> target {1}": {2}'.format(PROXYOPTIONS.host + ':' + proxy.port, PROXYOPTIONS.path, err.toString())); + e.code = err.code; + options.request.emit('error', e); + req.destroy && req.destroy(); + req = null; + self.requests = null; + self.options = null; }); req.end(); From 37c25f94cb2a181e323895c38eec587d9eed3bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 13 Jul 2018 10:04:26 +0200 Subject: [PATCH 0488/1669] Added encrypting/decrypting numbers. --- changes.txt | 3 +++ index.js | 2 ++ utils.js | 28 ++++++++++++++++++++++++---- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 2aa4c16a1..509275ffe 100755 --- a/changes.txt +++ b/changes.txt @@ -95,6 +95,9 @@ - added: `SchemaEntity.clear()` for removing all current definition - added: new view engine markup `@{#}` for simulating of root URL - added: `String.ROOT()` for replacing `@{#}` markup in strings +- added: `Number.encrypt(key)` for encrypting number values +- added: `String.decryptnumber(key)` for decrypting of number values +- added: `F.config['secret-numbers']` as a hidden secret for encrypting/decrypting values - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index 54b720346..e4031e670 100755 --- a/index.js +++ b/index.js @@ -647,6 +647,7 @@ function Framework() { version: '1.0.0', author: '', secret: this.syshash, + 'secret-numbers': 'numbers', 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -8878,6 +8879,7 @@ F.$configure_configs = function(arr, rewrite) { switch (name) { case 'secret': + case 'secret-numbers': obj[name] = value; break; case 'default-request-length': diff --git a/utils.js b/utils.js index bd00468f7..f6ac86ad9 100755 --- a/utils.js +++ b/utils.js @@ -3808,7 +3808,7 @@ function checksum(val) { return sum; } -String.prototype.encrypt = function(key, isUnique) { +String.prototype.encrypt = function(key, isUnique, secret) { var str = '0' + this; var data_count = str.length; var key_count = key.length; @@ -3832,10 +3832,10 @@ String.prototype.encrypt = function(key, isUnique) { for (var i = 0; i < str.length; i++) sum += str.charCodeAt(i); - return (sum + checksum(F.config.secret + key)) + '-' + str; + return (sum + checksum((secret || F.config.secret) + key)) + '-' + str; }; -String.prototype.decrypt = function(key) { +String.prototype.decrypt = function(key, secret) { var index = this.indexOf('-'); if (index === -1) @@ -3846,7 +3846,7 @@ String.prototype.decrypt = function(key) { return null; var hash = this.substring(index + 1); - var sum = checksum(F.config.secret + key); + var sum = checksum((secret || F.config.secret) + key); for (var i = 0; i < hash.length; i++) sum += hash.charCodeAt(i); @@ -3879,6 +3879,19 @@ String.prototype.decrypt = function(key) { return counter !== (val.length + key.length) ? null : val; }; +String.prototype.decryptnumber = function(salt) { + var val = this.decrypt(salt, F.config['secret-numbers']); + var num = 0; + if (val) { + num = +val; + if (isNaN(num)) + return 0; + for (var i = 0; i < salt.length; i++) + num -= salt.charCodeAt(i); + } + return num; +}; + String.prototype.base64ToFile = function(filename, callback) { var self = this; var index = self.indexOf(','); @@ -4061,6 +4074,13 @@ String.prototype.removeTags = function() { return this.replace(regexpTags, ''); }; +Number.prototype.encrypt = function(salt) { + var num = this; + for (var i = 0; i < salt.length; i++) + num += salt.charCodeAt(i); + return num.toString().encrypt(salt, false, F.config['secret-numbers']); +}; + Number.prototype.floor = function(decimals) { return Math.floor(this * Math.pow(10, decimals)) / Math.pow(10, decimals); }; From 2547ff259fff6ada7581fcb513cc5d4b86324139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 01:24:03 +0200 Subject: [PATCH 0489/1669] Updated GraphDB. --- graphdb.js | 2875 +++++++++++++++++++++++++++++----------------------- 1 file changed, 1615 insertions(+), 1260 deletions(-) diff --git a/graphdb.js b/graphdb.js index 1f4ba4387..4dcbb5a2f 100644 --- a/graphdb.js +++ b/graphdb.js @@ -24,1500 +24,1836 @@ * @version 1.0.0 */ -/* -HEADER: VERSION @Int8, COMPRESSION @Int8, SCAN @Int8, CLASSINDEX @Int8, RELATIONINDEX @Int8, SIZE @UInt16BE, COUNT @UInt32BE, NAME @StringUTF8 -CLASSES RAW JSON ---- 3x TYPES OF RECORD --- -"Node" : TYPE (0) @Int8, RELATIONID @UInt32BE, PARENTID @UInt32BE, SIZE_IN_THE_BUFFER @UInt16BE, DATA @StringUTF8 -"Links" : TYPE (254) @Int8, RELATIONID @UInt32BE, PARENTID @UInt32BE, COUNT_IN_THE_BUFFER @UInt16BE, [TYPE @Int8 CONNECTION @Int8 + SIBLINGID @UInt32BE] -"Removed" : TYPE (255) @Int8 -*/ - -const DEFSIZE = 40; + const Fs = require('fs'); +const Zlib = require('zlib'); + +const ZLIBOPTIONS = { level: Zlib.constants.Z_FULL_FLUSH, memLevel: Zlib.constants.Z_BEST_COMPRESSION, strategy: Zlib.constants.Z_DEFAULT_STRATEGY }; const VERSION = 1; +const DOCUMENTSIZE = 1000; +const PAGESIZE = 20; +const PAGELIMIT = 50; +const DATAOFFSET = 17; const EMPTYBUFFER = U.createBufferSize(1); -const BUFFERSIZE = 10; -const INFOSIZE = 50; -const METASIZE = 10240; -const HEADERSIZE = INFOSIZE + METASIZE; +const HEADERSIZE = 7000; const DELAY = 100; -const DatabaseBuilder = framework_nosql.DatabaseBuilder; -const REGDATE = /"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+Z"/g; -const REGKEY = /"[a-z-0-9]+":/gi; const REGTUNESCAPE = /%7C|%0D|%0A/g; const REGTESCAPETEST = /\||\n|\r/; const REGTESCAPE = /\||\n|\r/g; -const MAXREADERS = 3; const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; +const DatabaseBuilder = framework_nosql.DatabaseBuilder; -const NODE_REMOVED = 255; -const NODE_LINKS = 254; +const STATE_NORMAL = 1; +const STATE_COMPRESSED = 100; +const STATE_REMOVED = 255; -function GraphDB(name, size) { +// META +const META_PAGE_ADD = 100; +const META_CLASSESRELATIONS = 101; +const META_PAGE_ADD3 = 102; +const META_RELATIONPAGEINDEX = 103; - F.path.verify('databases'); +// OPERATIONS +const NEXT_READY = 1; +const NEXT_INSERT = 2; +const NEXT_RELATION = 3; +const NEXT_UPDATE = 4; +const NEXT_FIND = 5; +const NEXT_REMOVE = 6; - var t = this; - t.name = name; - t.filename = F.path.databases(name + '.gdb'); - t.$ready = false; - t.$size = size || DEFSIZE; - - t.header = {}; - t.stat = null; - t.bufferappend = []; - t.bufferupdate = []; - t.bufferremove = []; - t.pending = {}; - t.removed = EMPTYARRAY; - t.$events = {}; - t.$classes = {}; - t.$relations = {}; - t.appending = false; - t.updating = false; - t.reading = 0; - t.cleaning = false; - - t.$cb_writeupdate = function(err) { +// TYPES +const TYPE_CLASS = 1; +const TYPE_RELATION = 2; +const TYPE_RELATION_DOCUMENT = 3; - if (err) { - if (t.$events.error) - t.emit('error', err); - else - F.error(err, 'GraphDB: ' + t.name); - } +function GraphDB(name) { - t.stat.mtime = NOW; - t.updating = false; - t.flush(); - }; + F.path.verify('databases'); - t.$cb_writeappend = function(err, size) { + var self = this; + self.name = name; + self.filename = F.path.databases(name + '.gdb'); + self.ready = false; + + self.$classes = {}; + self.$relations = {}; + self.$events = {}; + + self.header = {}; + + self.pending = {}; + self.pending.insert = []; + self.pending.find = []; + self.pending.update = []; + self.pending.remove = []; + self.pending.relation = []; + self.pending.meta = []; + + self.states = {}; + self.states.insert = false; + self.states.read = false; + self.states.remove = false; + self.states.update = false; - if (err) { - if (t.$events.error) - t.emit('error', err); - else - F.error(err, 'GraphDB: ' + t.name); - } + F.path.verify('databases'); + // t.open(); - t.appending = false; - t.stat.size += size; - t.stat.atime = NOW; - t.header.count += (size / t.header.size) >> 0; - t.flushheader(); - t.flush(); + self.cb_error = function(err) { + err && console.log(err); }; - t.$cb_get = function(id, callback, type) { - t.get(id, callback, type); + self.cb_next = function(value) { + self.next(value); }; - t.$cb_read2 = function(id, callback) { - t.read2(id, callback); - }; + self.open(); +} - t.$cb_browse = function(beg, end, callback, done) { - t.browse(beg, end, callback, done); - }; +var GP = GraphDB.prototype; - t.$cb_insert = function(type, value, callback, id) { - t.insert(type, value, callback, id); - }; +// ==== DB:HEADER (7000b) +// name (30b) = from: 0 +// version (1b) = from: 30 +// pages (4b) = from: 31 +// pagesize (2b) = from: 35 +// pagelimit (2b) = from: 37 +// documents (4b) = from: 39 +// documentsize (2b) = from: 43 +// classindex (1b) = from: 45 +// relationindex (1b) = from: 46 +// relationnodeindex = from: 47 +// classes + relations = from: 51 + +// ==== DB:PAGE (20b) +// type (1b) = from: 0 +// index (1b) = from: 1 +// documents (2b) = from: 2 +// freeslots (1b) = from: 4 +// parentindex (4b) = from: 5 + +// ==== DB:DOCUMENT (SIZE) +// type (1b) = from: 0 +// index (1b) = from: 1 +// state (1b) = from: 2 +// pageindex (4b) = from: 3 +// relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) +// parentindex (4b) = from: 11 +// size/count (2b) = from: 15 +// data = from: 17 + +// Creates new page +function addPage(self, type, index, parentindex, callback) { + + // @type + // 1: classes + // 2: relations + // 3: relations private + + // @index + // index of value + + // Add a new page + self.header.pages++; + + var indexer = self.header.pages; + var buffer = []; + var page = U.createBufferSize(self.header.pagesize); + + // console.log('CREATING PAGE:', TYPES[type], indexer, type, index); + + page.writeUInt8(type, 0); // type (1:class, 2:relation, 3:private) + page.writeUInt8(index, 1); // index + page.writeUInt16LE(0, 2); // documents + page.writeUInt8(0, 4); // freeslots + page.writeUInt32LE(parentindex, 5); // parentindex + + buffer.push(page); + + for (var i = 0; i < self.header.pagelimit; i++) { + var doc = U.createBufferSize(self.header.documentsize); + doc.writeUInt8(type, 0); + doc.writeUInt8(index, 1); + doc.writeUInt8(STATE_REMOVED, 2); + doc.writeUInt32LE(self.header.pages, 3); + doc.writeUInt32LE(0, 7); // continuerindex + doc.writeUInt32LE(0, 11); // parentindex + doc.writeUInt16LE(0, 15); // size/count + buffer.push(doc); + } - t.$cb_write = function(id, prepare, callback) { - t.write(id, prepare, callback); - }; + buffer = Buffer.concat(buffer); - t.$cb_join = function(type, id, toid, callback) { - t.join(type, id, toid, callback); - }; + var offset = offsetPage(self, indexer); - t.$cb_pushlinkid = function(id, toid, type, relation, callback, parentid) { - pushRelation(t, id, toid, type, relation, callback, parentid); - }; + Fs.write(self.fd, buffer, 0, buffer.length, offset, function(err) { + // @TODO: add check for bytes written and buffer size + err && self.error(err, 'createPage.write'); + !err && updMeta(self, type === TYPE_RELATION_DOCUMENT ? META_PAGE_ADD3 : META_PAGE_ADD); + callback && callback(err, indexer); + }); - t.$cb_getlinkid = function(id, callback) { - t.getLinkId(id, callback); - }; + return indexer; +} - t.$cb_setlinkid = function(id, linkid, callback) { - t.setLinkId(id, linkid, callback); - }; +function addNodeFree(self, meta, callback) { - t.$cb_remove = function(id, callback) { - t.remove(id, callback); - }; + if (!meta.type.findfreeslots) { + addNode(self, meta, callback); + return; + } - t.$cb_clean = function(arr, callback) { - t.clean(arr, callback); - }; + findDocumentFree(self, meta.type.pageindex, function(err, documentindex, pageindex) { - t.$cb_graph = function(id, options, callback, builder) { - t.graph(id, options, callback, builder); - }; + if (!documentindex) { + addNode(self, meta, callback); + return; + } - t.$cb_class = function(name, declaration, indexer) { - t.class(name, declaration, indexer); - }; + var buffer = U.createBufferSize(self.header.documentsize); + buffer.writeUInt8(meta.typeid, 0); // type + buffer.writeUInt8(meta.type.index, 1); // index + buffer.writeUInt32LE(pageindex, 3); // pageindex + buffer.writeUInt8(meta.state || STATE_NORMAL, 2); // state + buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex + buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex + buffer.writeUInt16LE(meta.size, 15); + meta.data && meta.data.copy(buffer, DATAOFFSET); + + Fs.write(self.fd, buffer, 0, buffer.length, offsetDocument(self, documentindex), function() { + meta.type.locked = false; + callback(null, documentindex, pageindex); + }); + }); +} - t.$cb_relation = function(name, both, indexer) { - t.relation(name, both, indexer); - }; +function addNode(self, meta, callback) { - t.$cb_find = function(type, builder, reverse) { - if (reverse) - t.find2(type, builder); - else - t.find(type, builder); - }; + // meta.typeid (1 CLASS, 2 RELATION) + // meta.type (link to type class/relation) + // meta.state + // meta.parentindex + // meta.relationindex + // meta.size + // meta.buffer - t.$flush = function() { - t.flush(); - } + var buf = U.createBufferSize(self.header.pagesize); + var offset = offsetPage(self, meta.type.pageindex); - F.path.verify('databases'); - t.open(); -} + meta.type.locked = true; -var GP = GraphDB.prototype; + Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { -function extendclass(self, type, old) { + if (err) + throw err; - if (self.$ready && self.reading < MAXREADERS) { + if (buf[0] !== meta.typeid) + throw new Error('Not a class page'); - var size = self.header.size * self.header.buffersize; - var index = 0; - var now = Date.now(); - var cls = self.$classes[type]; + if (!meta.type.private && buf[1] !== meta.type.index) + throw new Error('Not same class type'); - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" extending class "{1}" (beg)'.format(self.name, cls.name)); + // type : buf[0] + // index : buf[1] + // documents : buf.readUInt16LE(2) + // freeslots : buf[4] + // parentindex : readUInt32LE(5) - old = parseSchema(old); - self.reading++; + var buffer = U.createBufferSize(self.header.documentsize); + buffer.writeUInt8(buf[0], 0); // type + buffer.writeUInt8(meta.type.index, 1); // index + buffer.writeUInt32LE(meta.type.pageindex, 3); // pageindex + buffer.writeUInt8(meta.state || STATE_NORMAL, 2); // state + buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex + buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex + buffer.writeUInt16LE(meta.size, 15); + meta.data && meta.data.copy(buffer, DATAOFFSET); - var errhandling = F.error(); + var documents = buf.readUInt16LE(2); + var documentsbuf = U.createBufferSize(2); - var reader = function() { + documents++; + documentsbuf.writeUInt16LE(documents); - var buf = U.createBufferSize(size); - var offset = HEADERSIZE + (index * self.header.size); - var current = index; + Fs.write(self.fd, documentsbuf, 0, documentsbuf.length, offset + 2, function(err) { - index += self.header.buffersize; + err && console.log('addNode.write.meta', err); + Fs.write(self.fd, buffer, 0, buffer.length, offset + self.header.pagesize + ((documents - 1) * self.header.documentsize), function(err) { - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { + err && console.log('addNode.write.data', err); - if (err || !size) { - self.reading--; - self.flushmeta(); - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" extending class "{1}" (end, {2}s)'.format(self.name, cls.name, (((Date.now() - now) / 1000) >> 0))); - return; - } + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // continuerindex (4b) = from: 7 + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 - while (true) { + // We must create a new page + if (documents + 1 > self.header.pagelimit) { + addPage(self, meta.typeid, meta.type.index, meta.type.pageindex, function(err, index) { - var buffer = buf.slice(0, self.header.size); - if (!buffer.length) - break; + var documentindex = getDocumentIndex(self, meta.type.pageindex, documents); + meta.type.documentindex = documentindex; + meta.type.pageindex = index; + meta.type.locked = false; - current++; - - switch (buffer[0]) { - case NODE_REMOVED: - case NODE_LINKS: - break; - case type: - var docsize = buffer.readUInt16BE(9); - if (docsize) { - var data = parseData(old, buffer.slice(11, docsize + 11).toString('utf8').split('|')); - self.insert(type, data, errhandling, current); - } - break; - } + // Problem with classes + // meta.type.index = 0; - buf = buf.slice(self.header.size); - } + if (meta.type.private) + self.header.relationpageindex = index; - setImmediate(reader); + updMeta(self, meta.type.private ? META_RELATIONPAGEINDEX : META_CLASSESRELATIONS); + callback(null, documentindex, index); + }); + } else { + var documentindex = getDocumentIndex(self, meta.type.pageindex, documents); + meta.type.locked = false; + meta.type.documentindex = documentindex; + callback(null, documentindex, meta.type.pageindex); + } }); - }; + }); + }); +} - setImmediate(reader); - } else { - setTimeout(extendclass, DELAY, self, type, old); - } +function addDocument(self, cls, value, callback) { + + // meta.typeid (1 CLASS, 2 RELATION) + // meta.type (link to type class/relation) + // meta.state + // meta.parentindex + // meta.relationindex + // meta.size + // meta.data + + var meta = {}; + meta.type = cls; + meta.typeid = TYPE_CLASS; + meta.state = 1; + meta.parentindex = 0; + meta.relationindex = 0; + meta.data = U.createBuffer(stringifyData(cls.schema, value)); + meta.size = meta.data.length; + + var limit = self.header.documentsize - DATAOFFSET; + + if (meta.data.length > limit) { + Zlib.deflate(meta.data, ZLIBOPTIONS, function(err, buf) { + if (err || buf.length > limit) + callback(new Error('GraphDB: Data too long'), 0); + else { + meta.state = STATE_COMPRESSED; + meta.data = buf; + meta.size = buf.length; + addNodeFree(self, meta, callback); + } + }); + } else + addNodeFree(self, meta, callback); } -GP.errorhandling = function(err, type) { - console.log('GraphDB "{0}" --> "{1}" error: {2}'.format(this.name, err, type)); - this.$events.error && this.emit('error', err); -}; +function addRelation(self, relation, indexA, indexB, callback) { -GP.class = function(name, declaration, indexer, nodeid) { + // Workflow: + // Has "A" relation nodes? + // Has "B" relation nodes? + // Create "A" relation with "B" + // Create "B" relation with "A" + // Register relation to global relations - var self = this; - if (self.$ready || indexer) { + var tasks = []; + var relA = null; + var relB = null; - var meta = parseSchema(declaration); - var cls = self.$classes[name]; + var tmprelation = { index: relation.index, pageindex: 0, documentindex: 0, locked: false, private: true }; - meta.name = name; + tasks.push(function(next) { + self.read(indexA, function(err, doc, relid) { + if (doc) { + relA = relid; + next(); + } else { + tasks = null; + next = null; + callback(new Error('GraphDB: Node (A) "{0}" not exists.'.format(indexA))); + } + }); + }); - if (cls) { - if (cls.raw !== meta.raw) { - meta.index = cls.index; - meta.nodeid = cls.nodeid; - self.$classes[cls.index] = self.$classes[name] = meta; - extendclass(self, meta.index, cls.raw, meta.raw); + tasks.push(function(next) { + self.read(indexB, function(err, doc, relid) { + if (doc) { + relB = relid; + next(); + } else { + tasks = null; + next = null; + callback(new Error('GraphDB: Node (B) "{0}" not exists.'.format(indexB))); } - } else { + }); + }); + + tasks.push(function(next) { - var index = indexer; - if (indexer == null) { - self.header.classindex++; - index = self.header.classindex; - meta.nodeid = nodeid ? nodeid : makeRelation(self); + if (relA == 0) { + next(); + return; + } + + checkRelation(self, relation, relA, indexB, function(err, is) { + if (is) { + tasks = null; + next = null; + callback(new Error('GraphDB: Same relation already exists between nodes (A) "{0}" and (B) "{1}".'.format(indexA, indexB))); } else - meta.nodeid = nodeid; + next(); + }); + }); - meta.index = index; - self.$classes[index] = self.$classes[name] = meta; + // Obtaining indexA a relation document + tasks.push(function(next) { + if (relA) + next(); + else { + addRelationDocument(self, relation, indexA, function(err, index) { + relA = index; + next(); + }, true); + } + }); - if (indexer == null) - self.$flushmeta(); + // Obtaining indexB a relation document + tasks.push(function(next) { + if (relB) + next(); + else { + addRelationDocument(self, relation, indexB, function(err, index) { + relB = index; + next(); + }, true); } + }); - } else { - setTimeout(self.$cb_class, DELAY, name, declaration, indexer); - } + // Push "indexB" relation to "indexA" + tasks.push(function(next) { + tmprelation.documentindex = relA; + tmprelation.pageindex = self.header.relationpageindex; + pushRelationDocument(self, relA, tmprelation, indexB, true, function(err, index) { + // Updated relation, document was full + if (relA !== index) { + relA = index; + updDocumentRelation(self, indexA, relA, next); + } else + next(); + }, true); + }); - return self; -}; + tasks.push(function(next) { + tmprelation.documentindex = relB; + tmprelation.pageindex = self.header.relationpageindex; + pushRelationDocument(self, relB, tmprelation, indexA, false, function(err, index) { + // Updated relation, document was full + if (relB !== index) { + relB = index; + updDocumentRelation(self, indexB, relB, next); + } else + next(); + }, true); + }); -GP.classes = function(callback) { - var self = this; - var items = []; - for (var i = 0; i < self.header.classindex; i++) { - var item = self.$classes[i + 1]; - item && items.push(item.name); - } - callback(null, items); - return self; -}; + tasks.push(function(next) { + // console.log('PUSH COMMON', relation.documentindex, indexA); + pushRelationDocument(self, relation.documentindex, relation, indexA, true, next); + }); + + tasks.async(function() { + // console.log('REL ====', relA, relB); + callback(null, true); + }); +} -GP.relation = function(name, both, indexer, nodeid) { +function remRelation(self, relation, indexA, indexB, callback) { - var self = this; - if (self.$ready || indexer) { + var tasks = []; + var relA = null; + var relB = null; - var rel = self.$relations[name]; - if (rel) { + tasks.push(function(next) { + self.read(indexA, function(err, doc, relid) { + if (doc) { + relA = relid; + next(); + } else { + tasks = null; + next = null; + callback(new Error('GraphDB: Node (A) "{0}" not exists.'.format(indexA))); + } + }); + }); - if (both === true && !rel.relation) - self.$flushmeta(); + tasks.push(function(next) { + self.read(indexB, function(err, doc, relid) { + if (doc) { + relB = relid; + next(); + } else { + tasks = null; + next = null; + callback(new Error('GraphDB: Node (B) "{0}" not exists.'.format(indexB))); + } + }); + }); - } else { + tasks.async(function() { + remRelationLink(self, relA, indexB, function(err, countA) { + remRelationLink(self, relB, indexA, function(err, countB) { + remRelationLink(self, relation.documentindex, indexA, function(err, countC) { + callback(null, (countA + countB + countC) === 3); + }); + }); + }); + }); +} + +function remRelationLink(self, index, documentindex, callback, nochild, counter) { + + var buf = U.createBufferSize(self.header.documentsize); + var offset = offsetDocument(self, index); + + !counter && (counter = 0); - rel = {}; + Fs.read(self.fd, buf, 0, buf.length, offset, function() { + + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 + + if ((buf[0] !== TYPE_RELATION && buf[0] !== TYPE_RELATION_DOCUMENT) || (buf[2] === STATE_REMOVED)) { + callback(null, counter); + return; + } + + var relid = buf.readUInt32LE(7); + var count = buf.readUInt16LE(15); + var arr = []; + var is = false; + + for (var i = 0; i < count; i++) { + var off = DATAOFFSET + (i * 6); + var obj = {}; + obj.INDEX = buf[off]; + obj.INIT = buf[off + 1]; + obj.ID = buf.readUInt32LE(off + 2); + if (obj.ID === documentindex && obj.INIT === 1) + is = true; + else + arr.push(obj); + } - var index = indexer; - if (indexer == null) { - self.header.relationindex++; - index = self.header.relationindex; - rel.nodeid = nodeid ? nodeid : makeRelation(self); + if (is) { + count = arr.length; + for (var i = 0; i < count; i++) { + var off = DATAOFFSET + (i * 6); + var obj = arr[i]; + buf.writeUInt8(obj.INDEX, off); + buf.writeUInt8(obj.INIT, off + 1); + buf.writeUInt32LE(obj.ID, off + 2); } + buf.writeUInt16LE(count, 15); + buf.fill(EMPTYBUFFER, DATAOFFSET + ((count + 1) * 6)); + Fs.write(self.fd, buf, 0, buf.length, offset, function() { + counter++; + if (relid && !nochild) + setImmediate(remRelationLink, self, relid, documentindex, callback, null, counter); + else + callback(null, counter); + }); + } else if (relid && !nochild) + setImmediate(remRelationLink, self, relid, documentindex, callback, null, counter); + else + callback(null, counter); + }); +} + +// Traverses all RELATIONS documents and remove specific "documentindex" +function remRelationAll(self, index, documentindex, callback, counter) { - rel.index = index; - rel.name = name; - rel.relation = both ? 1 : 0; - self.$relations[index] = self.$relations[name] = rel; + var buf = U.createBufferSize(self.header.pagelimit * self.header.documentsize); + var offset = offsetDocument(self, index); - if (indexer == null) - self.$flushmeta(); + !counter && (counter = 0); + + Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { + + if (err || !size) { + callback(null, counter); + return; } - } else { - setTimeout(self.$cb_relation, DELAY, name, both, indexer); - } + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 - return self; -}; + var removed = []; -GP.relations = function(callback) { - var self = this; - var items = []; - for (var i = 0; i < self.header.relationindex; i++) { - var item = self.$relations[i + 1]; - item && items.push(item.name); - } - callback(null, items); - return self; -}; + while (true) { + + if (!buf.length) + break; -GP.scan = function(callback) { + index++; - var self = this; - var remove = []; - var now = Date.now(); - - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" scanning (beg)'.format(self.name)); - - var next = function(index) { - var buf = U.createBufferSize(1); - Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((index - 1) * self.header.size), function(err, size) { - if (size) { - buf[0] === NODE_REMOVED && remove.push(index); - if (index % 5 === 0) - setImmediate(() => next(index + 1)); + var data = buf.slice(0, self.header.documentsize); + + if ((data[0] !== TYPE_RELATION && data[0] !== TYPE_RELATION_DOCUMENT) || (data[2] === STATE_REMOVED)) { + buf = buf.slice(self.header.documentsize); + continue; + } + + var count = data.readUInt16LE(15); + var arr = []; + var is = false; + + for (var i = 0; i < count; i++) { + var off = DATAOFFSET + (i * 6); + var obj = {}; + obj.INDEX = data[off]; + obj.INIT = data[off + 1]; + obj.ID = data.readUInt32LE(off + 2); + if (obj.ID === documentindex) + is = true; else - next(index + 1); - } else { - callback && callback(null, remove); - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" scanning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); + arr.push(obj); } - }); - }; - next(1); - return self; -}; + if (is) { -GP.clean = function(callback) { + var newcount = arr.length; - var self = this; + for (var i = 0; i < newcount; i++) { + var off = DATAOFFSET + (i * 6); + var obj = arr[i]; + data.writeUInt8(obj.INDEX, off); + data.writeUInt8(obj.INIT, off + 1); + data.writeUInt32LE(obj.ID, off + 2); + } - if (!self.$ready || self.cleaning) { - setTimeout(self.$cb_clean, DELAY, callback); - return self; - } + data.writeUInt16LE(newcount, 15); + data.fill(EMPTYBUFFER, DATAOFFSET + ((newcount + 1) * 6)); - self.cleaning = true; + removed.push({ index: index - 1, buf: data }); + } - self.scan(function(err, arr) { + buf = buf.slice(self.header.documentsize); + } - if (!arr.length) { - callback && callback(null, 0); + if (!removed.length) { + setImmediate(remRelationAll, self, index, documentindex, callback, counter); return; } - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (beg)'.format(self.name)); + counter += removed.length; + removed.wait(function(item, next) { + Fs.write(self.fd, item.buf, 0, item.buf.length, offsetDocument(self, item.index), next); + }, function() { + setImmediate(remRelationAll, self, index, documentindex, callback, counter); + }); - var removed = 0; - var cache = {}; - var now = Date.now(); + }); +} - for (var i = 0; i < arr.length; i++) - cache[arr[i]] = 1; +function addRelationDocument(self, relation, index, callback, between) { - var done = function() { - for (var i = 0; i < arr.length; i++) { - if (self.bufferremove.indexOf(arr[i]) === -1) - self.bufferremove.push(arr[i]); - } - self.header.scan = false; - self.appending = false; - self.updating = false; - self.flushheader(); - callback && callback(null, removed); - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" cleaning (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); - self.emit('clean', removed); - }; + // meta.typeid (1 CLASS, 2 RELATION, 3 PRIVATE RELATION) + // meta.type (link to type class/relation) + // meta.state + // meta.parentindex + // meta.relationindex + // meta.size + // meta.data - var next = function(index) { - var buf = U.createBufferSize(self.header.size); - var position = HEADERSIZE + ((index - 1) * self.header.size); + var meta = {}; + meta.typeid = between ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; + meta.type = between ? { index: 0, pageindex: self.header.relationpageindex, documentindex: index, locked: false, private: true } : relation; + meta.state = 1; + meta.parentindex = 0; + meta.relationindex = 0; + meta.size = 0; + + // Creates a new node + addNode(self, meta, function(err, relationindex) { + + // Updates exiting document by updating relation index + updDocumentRelation(self, index, relationindex, function(err) { + // Returns a new relation index + callback(err, relationindex); + }); + }); +} - Fs.read(self.fd, buf, 0, buf.length, position, function(err, size) { +function findDocumentFree(self, pageindex, callback, ready) { - if (err || !size) { - done(); - return; - } + var offset = offsetPage(self, pageindex); + var buf = U.createBufferSize(self.header.pagesize); - if (buf[0] !== NODE_LINKS) { - next(index + 1); - return; - } + Fs.read(self.fd, buf, 0, buf.length, offset, function() { - var id = buf.readUInt32BE(1); + // ==== DB:PAGE (20b) + // type (1b) = from: 0 + // index (1b) = from: 1 + // documents (2b) = from: 2 + // freeslots (1b) = from: 4 + // parentindex (4b) = from: 5 - // Are removed links? - if (cache[id] && buf[0] === NODE_REMOVED) { - next(index + 1); - return; - } + var relid = buf.readUInt32LE(5); + if (!relid) { + if (!ready) { + callback(null, 0); + return; + } + } - var count = buf.readUInt16BE(9); // 2b - var buffer = U.createBufferSize(self.header.size); - var off = 11; - var buffercount = 0; - var is = false; + // First page is the last page saved in meta therefore is needed to perform recursive with "ready" + if (!ready) { + findDocumentFree(self, relid, callback, true); + return; + } - for (var i = 0; i < count; i++) { - var pos = 11 + (i * 6); - var conn = buf.readUInt32BE(pos + 2); - if (cache[conn]) { - removed++; - is = true; - } else { - buffer.writeUInt8(buf[pos], off); - buffer.writeUInt8(buf[pos + 1], off + 1); - buffer.writeUInt32BE(buf.readUInt32BE(pos + 2), off + 2); - buffercount++; - off += 6; - } - } + var documents = buf.readUInt16LE(2); + if (documents >= self.header.pagelimit) { + // Finds in parent if exists + if (relid) + findDocumentFree(self, relid, callback, true); + else + callback(null, 0); + return; + } - if (is) { + // Finds a free document slot + var index = getDocumentIndex(self, pageindex) - 1; + var buffer = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - buffer.writeUInt8(buf[0], 0); // type, 1b - buffer.writeUInt32BE(buf.readUInt32BE(1), 1); // link, 4b - buffer.writeUInt16BE(buffercount, 6); // count, 2b + Fs.read(self.fd, buffer, 0, buffer.length, offset + self.header.pagesize, function() { + while (true) { + index++; + var data = buffer.slice(0, self.header.documentsize); + if (!data.length) + break; - // WRITE - Fs.write(self.fd, buffer, 0, buffer.length, position, function(err) { - err && self.errorhandling(err, 'clean.write'); - next(index + 1); + if (data[2] === STATE_REMOVED) { + updPageMeta(self, pageindex, function(err, buf) { + buf.writeUInt16LE(documents + 1, 2); + setImmediate(callback, null, index, pageindex); }); - - } else { - if (size) - next(index + 1); - else - done(); + buffer = buffer.slice(self.header.documentsize); + return; } - }); - }; + } - next(1); + if (relid) + findDocumentFree(self, relid, callback, true); + else + callback(null, 0); + }); }); - return self; -}; +} -GP.count = function(callback) { - callback(null, this.header.count); - return this; -}; +// Finds a free space for new relation in "pushRelationDocument" +function findRelationDocument(self, relid, callback) { -GP.emit = function(name, a, b, c, d, e, f, g) { - var evt = this.$events[name]; - if (evt) { - var clean = false; - for (var i = 0, length = evt.length; i < length; i++) { - if (evt[i].$once) - clean = true; - evt[i].call(this, a, b, c, d, e, f, g); + if (!relid) { + callback(null, 0); + return; + } + + var offset = offsetDocument(self, relid); + var buf = U.createBufferSize(self.header.documentsize); + + Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { + + if (err || !size) { + callback(err, 0); + return; } - if (clean) { - evt = evt.remove(n => n.$once); - if (evt.length) - this.$events[name] = evt; + + var count = buf.readUInt16LE(15); + if (count + 1 > self.header.relationlimit) { + // Checks if the relation index has next relation + + if (relid === buf.readUInt32LE(7)) + return; + + relid = buf.readUInt32LE(7); + if (relid) + setImmediate(findRelationDocument, self, relid, callback); else - this.$events[name] = undefined; + callback(null, 0); + } else { + // Free space for this relation + callback(null, relid); } - } - return this; -}; + }); +} -GP.on = function(name, fn) { - if (this.$ready && (name === 'ready' || name === 'load')) { - fn(); - return this; - } - if (!fn.$once) - this.$free = false; - if (this.$events[name]) - this.$events[name].push(fn); - else - this.$events[name] = [fn]; - return this; -}; +// Pushs "documentindex" to "index" document (document with all relations) +function pushRelationDocument(self, index, relation, documentindex, initializator, callback, between, recovered) { -GP.once = function(name, fn) { - fn.$once = true; - return this.on(name, fn); -}; + var offset = offsetDocument(self, index); + var buf = U.createBufferSize(self.header.documentsize); -GP.removeListener = function(name, fn) { - var evt = this.$events[name]; - if (evt) { - evt = evt.remove(n => n === fn); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - return this; -}; + Fs.read(self.fd, buf, 0, buf.length, offset, function() { -GP.removeAllListeners = function(name) { - if (name === true) - this.$events = EMPTYOBJECT; - else if (name) - this.$events[name] = undefined; - else - this.$events[name] = {}; - return this; -}; + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 -GP.open = function() { - var self = this; - Fs.stat(self.filename, function(err, stat) { - if (err) { - Fs.open(self.filename, 'w', function(err, fd) { - self.fd = fd; - self.flushheader(function() { - Fs.close(self.fd, function() { - self.open(); - }); - }, fd); - }); - } else { - self.stat = stat; - self.$open(); - } - }); - return self; -}; + var count = buf.readUInt16LE(15); + if (count + 1 > self.header.relationlimit) { -GP.resize = function(docSize, callback) { - var self = this; - var now = Date.now(); - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" resizing (beg)'.format(self.name)); - Fs.open(self.filename + '-tmp', 'w', function(err, fd) { - self.flushheader(function() { - - var index = HEADERSIZE; - var offset = HEADERSIZE; - - var next = function(done) { - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, index, function(err, size) { - if (size) { - var w = U.createBufferSize(docSize); - w.fill(buf, 0, buf.length); - index += self.header.size; - Fs.write(fd, w, 0, w.length, offset, function(err, size) { - err && self.errorhandling(err, 'resize.write'); - offset += size; - next(done); - }); - } else - done(); - }); - }; + // @TODO: Find free relation ID document + findRelationDocument(self, buf.readUInt32LE(7), function(err, newindex) { - next(function() { - Fs.close(fd, function() { - Fs.close(self.fd, function() { - Fs.rename(self.filename + '-tmp', self.filename, function(err) { - F.config['nosql-logger'] && PRINTLN('GraphDB "{0}" resizing (end, {1}s)'.format(self.name, (((Date.now() - now) / 1000) >> 0))); - err && self.errorhandling(err, 'resize.rename'); - callback && callback(err); + // Is some relation document exist? + if (newindex && !recovered) { + pushRelationDocument(self, newindex, relation, documentindex, initializator, callback, between, true); + return; + } + + // meta.typeid (1 CLASS, 2 RELATION) + // meta.type (link to type class/relation) + // meta.state + // meta.parentindex + // meta.relationindex + // meta.size + // meta.buffer + + var meta = {}; + meta.typeid = relation.private ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; + meta.type = relation; + meta.state = STATE_NORMAL; + meta.parentindex = 0; + meta.relationindex = index; + meta.size = 0; + + // addPage(self, relation.private ? TYPE_RELATION_DOCUMENT : TYPE_RELATION, relation.index, relation.pageindex, function(err, pageindex) { + addNode(self, meta, function(err, docindex, pageindex) { + + // index = relation.documentindex; + relation.pageindex = pageindex; + // relation.documentindex = getDocumentIndex(self, pageindex); + relation.documentindex = docindex; + + updDocumentRelation(self, relation.documentindex, index, function() { + updDocumentParent(self, index, relation.documentindex, function() { + pushRelationDocument(self, relation.documentindex, relation, documentindex, initializator, callback, between); }); }); }); }); - }, fd); - }); - return self; -}; - -GP.ready = function(callback) { - var self = this; - if (self.$ready) - callback(); - else - self.on('ready', callback); - return self; -}; - -GP.flushheader = function(callback, fd) { + } else { - var self = this; - var buf = U.createBufferSize(INFOSIZE); - - buf.writeUInt8(VERSION, 0); // 1b - buf.writeUInt8(0, 1); // 1b, COMPRESSION 1: true, 0: false - buf.writeUInt8(self.header.scan ? 1 : 0, 2); // 1b, SCAN REMOVED DOCUMENTS 1: true, 0: false - buf.writeUInt8(self.header.classindex ? self.header.classindex : 0, 3); // 1b - buf.writeUInt8(self.header.relationindex ? self.header.relationindex : 0, 4); // 1b - buf.writeUInt16BE(self.$size, 5); // 2b - buf.writeUInt32BE(self.header.count || 0, 8); // 4b - buf.writeUInt32BE(self.header.removed || 0, 12); // 4b - - var str = 'Total.js GraphDB'; - buf.fill(str, 13, str.length + 13); - - Fs.write(fd || self.fd, buf, 0, buf.length, 0, function(err) { - err && self.errorhandling(err, 'flushheader.write'); - if (fd) - self.flushmeta(callback, fd); - else - callback && callback(); - }); + var buffer = U.createBufferSize(6); + buffer.writeUInt8(relation.index, 0); + buffer.writeUInt8(initializator ? 1 : 0, 1); + buffer.writeUInt32LE(documentindex, 2); + buffer.copy(buf, DATAOFFSET + (count * 6)); + buf.writeUInt16LE(count + 1, 15); + + if (buf[2] === STATE_REMOVED) { + // We must update page document counts + var pageindex = Math.ceil(index / self.header.pagelimit); + updPageMeta(self, pageindex, function(err, buf) { + + // type (1b) = from: 0 + // index (1b) = from: 1 + // documents (2b) = from: 2 + // freeslots (1b) = from: 4 + // parentindex (4b) = from: 5 + + buf.writeUInt16LE(buf.readUInt16LE(2) + 1, 2); + setImmediate(function() { + Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { + err && self.error(err, 'pushRelationDocument.read.write'); + callback(null, index); + }); + }); + }); - return self; -}; + buf.writeUInt8(STATE_NORMAL, 2); -GP.$flushmeta = function() { - var self = this; - self.$flushmetadelay && clearImmediate(self.$flushmetadelay); - self.$flushmetadelay = setImmediate(self => self.flushmeta(), self); -}; - -GP.flushmeta = function(callback, fd) { + } else { + // DONE + Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { + err && self.error(err, 'pushRelationDocument.read.write'); + callback(null, index); + }); + } + } - var self = this; - var buf = U.createBufferSize(METASIZE); - var meta = {}; - self.$flushmetadelay = null; + }); +} - meta.c = []; // classes - meta.r = []; // relations +function updDocumentRelation(self, index, relationindex, callback) { - if (self.header.classindex) { - for (var i = 0; i < self.header.classindex; i++) { - var item = self.$classes[i + 1]; - item && meta.c.push({ i: i + 1, n: item.name, r: item.raw, x: item.nodeid }); - } - var b = U.createBufferSize(1); - b.writeUInt8(self.header.classindex); - Fs.write(fd || self.fd, b, 0, 1, 3, NOOP); - } + if (index === relationindex) + throw new Error('FET'); - if (self.header.relationindex) { - for (var i = 0; i < self.header.relationindex; i++) { - var item = self.$relations[i + 1]; - item && meta.r.push({ i: i + 1, n: item.name, r: item.relation, x: item.nodeid }); - } - var b = U.createBufferSize(1); - b.writeUInt8(self.header.relationindex); - Fs.write(fd || self.fd, b, 0, 1, 4, NOOP); - } + var offset = offsetDocument(self, index); + var buf = U.createBufferSize(4); + buf.writeUInt32LE(relationindex); + Fs.write(self.fd, buf, 0, buf.length, offset + 7, callback); +} - var data = JSON.stringify(meta); - buf.fill(data, 0, data.length); +function updDocumentParent(self, index, parentindex, callback) { + var offset = offsetDocument(self, index); + var buf = U.createBufferSize(4); + buf.writeUInt32LE(parentindex); + Fs.write(self.fd, buf, 0, buf.length, offset + 11, callback); +} - Fs.write(fd || self.fd, buf, 0, buf.length, INFOSIZE, function(err) { - err && self.errorhandling(err, 'flushmeta.write'); - callback && callback(); +function updPageMeta(self, index, fn) { + var offset = offsetPage(self, index); + var buf = U.createBufferSize(self.header.pagesize); + Fs.read(self.fd, buf, 0, buf.length, offset, function() { + fn(null, buf); + Fs.write(self.fd, buf, 0, buf.length, offset, self.cb_error); }); +} - self.flush(); - return self; -}; - -GP.$open = function() { - var self = this; - Fs.open(self.filename, 'r+', function(err, fd) { - self.fd = fd; - var buf = U.createBufferSize(INFOSIZE); - Fs.read(self.fd, buf, 0, buf.length, 0, function() { - - self.header.version = buf[0]; - self.header.compress = buf[1] === 1; - self.header.scan = buf[2] === 1; - self.header.classindex = buf[3]; - self.header.relationindex = buf[4]; - self.header.size = buf.readUInt16BE(5); - self.header.count = buf.readUInt32BE(8); - self.header.buffersize = ((1024 / self.header.size) * BUFFERSIZE) >> 0; - - if (self.header.buffersize < 1) - self.header.buffersize = 1; - - self.index = self.header.count + 1; - - buf = U.createBufferSize(METASIZE); - Fs.read(self.fd, buf, 0, buf.length, INFOSIZE, function() { - - var data = buf.slice(0, buf.indexOf(EMPTYBUFFER)).toString('utf8').parseJSON(); - if (data) { - - // registering classes - if (data.r instanceof Array) { - for (var i = 0; i < data.c.length; i++) { - var item = data.c[i]; - self.class(item.n, item.r, item.i, item.x); - } - } - // registering relations - if (data.r instanceof Array) { - for (var i = 0; i < data.r.length; i++) { - var item = data.r[i]; - self.relation(item.n, item.r, item.i, item.x); - } - } - } - - if (self.$size > self.header.size) { - self.resize(self.$size, () => self.open()); - } else { - self.$ready = true; - self.flush(); - self.emit('ready'); - } - }); +function remDocument(self) { + if (!self.ready || self.states.remove || !self.pending.remove.length) + return; + self.states.remove = true; + var doc = self.pending.remove.shift(); + remRelationAll(self, doc.id, doc.id, function() { + remDocumentAll(self, doc.id, function(err, count) { + self.states.remove = false; + doc.callback && doc.callback(err, count); + setImmediate(self.cb_next, NEXT_REMOVE); }); }); -}; - -GP.close = function() { - var self = this; - self.fd && Fs.close(self.fd); - self.fd = null; - return self; -}; - -GP.insert = function(type, value, callback, id) { - - var self = this; - - if (typeof(type) === 'object') { - callback = value; - value = type; - type = 0; - } - - if (value instanceof Array) { - callback && callback(new Error('GraphDB: You can\'t insert an array.')); - return self; - } else if (value == null) { - callback && callback(new Error('GraphDB: Value can\'t be nullable.')); - return self; - } else if (typeof(value) !== 'object') { - callback && callback(new Error('GraphDB: A value must be object only.')); - return self; - } else if (type > 253) { - callback && callback(new Error('GraphDB: Illegal class value "{0}"'.format(type))); - return self; - } +} - if (self.$ready) { +function remDocumentAll(self, index, callback, count) { - var val; - var schema; + var offset = offsetDocument(self, index); + var buf = U.createBufferSize(17); - if (type) { - schema = self.$classes[type]; + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 - if (!schema) { - callback && callback(new Error('GraphDB: Schema "{0}" not found.'.format(name))); - return self; - } + if (!count) + count = 0; - val = U.createBuffer(stringifyData(schema, value)); - type = schema.index; + Fs.read(self.fd, buf, 0, buf.length, offset, function() { - } else - val = U.createBuffer(stringify(value)); + var relid = buf.readUInt32LE(7); - if (val.length > self.header.size - 11) { - callback(new Error('GraphDB: Data too long.')); - return self; + if (buf[2] === STATE_REMOVED) { + if (relid) + remDocumentAll(self, relid, callback, count); + else + callback(null, count); + return; } - var insert = id == null; - var recovered = false; + buf.writeUInt8(STATE_REMOVED, 2); + buf.writeUInt16LE(0, 15); - if (insert) { - if (self.bufferremove && self.bufferremove.length) { - id = self.bufferremove.shift(); - insert = false; - recovered = true; - } else - id = self.index++; - } + if (buf[0] === TYPE_CLASS) + self.$classes[buf[1]].findfreeslots = true; - var res = insert ? U.createBufferSize(self.header.size).fill(val, 11, val.length + 11) : U.createBufferSize(self.header.size - 9).fill(val, 2, val.length + 2); + var pageindex = buf.readUInt32LE(3); - if (insert) { - res.writeUInt8(type, 0); - res.writeUInt32BE(0, 1); // RELATIONID - res.writeUInt32BE(0, 5); // PARENTID - } + Fs.write(self.fd, buf, 0, buf.length, offset, function() { - res.writeUInt16BE(val.length, insert ? 9 : 0); + // Updates "documents" in the current page + updPageMeta(self, pageindex, function(err, buf) { - if (insert) { - self.bufferappend.push({ id: id, buf: res, callback: callback, schemaid: schema ? type : 0 }); - self.$events.insert && self.emit('insert', schema ? schema.name : type, id, value); - // schema && schema.nodeid && pushRelationNode(self, schema, id); - } else { + // type (1b) = from: 0 + // index (1b) = from: 1 + // documents (2b) = from: 2 + // freeslots (1b) = from: 4 + // parentindex (4b) = from: 5 - self.bufferupdate.push({ buf: res, id: id, type: type, recovered: recovered, offset: 9, callback: callback }); + var documents = buf.readUInt16LE(2); + buf.writeUInt16LE(documents > 0 ? documents - 1 : documents, 2); - // because of seeking on HDD - if (self.bufferupdate.length > 1) - self.bufferupdate.quicksort('pos'); + count++; - self.$events.update && self.emit('update', id, value); - } - - setImmediate(self.$flush); - - } else { - setTimeout(self.$cb_insert, DELAY, type, value, callback, id); - } + setImmediate(function() { + if (relid) + remDocumentAll(self, relid, callback, count); + else + callback(null, count); + }); + }); + }); + }); +} - return self; -}; +function offsetPage(self, index) { + return HEADERSIZE + ((index - 1) * (self.header.pagesize + (self.header.pagelimit * self.header.documentsize))); +} -GP.getLinkId = function(id, callback) { - var self = this; - var pos = HEADERSIZE + ((id - 1) * self.header.size); - var tmp = U.createBufferSize(4); - Fs.read(self.fd, tmp, 0, tmp.length, pos + 1, function(err, size) { - callback(err, size && tmp[0] !== NODE_REMOVED ? tmp.readUInt32BE(0) : null); - }); - return self; -}; +function offsetDocument(self, index) { + var page = Math.ceil(index / self.header.pagelimit); + var offset = page * self.header.pagesize; + return HEADERSIZE + offset + ((index - 1) * self.header.documentsize); +} -GP.setLinkId = function(id, linkid, callback) { - var self = this; - var pos = HEADERSIZE + ((id - 1) * self.header.size); - var buf = U.createBufferSize(4); - buf.writeUInt32BE(linkid); - Fs.write(self.fd, buf, 0, buf.length, pos + 1, function(err) { - err && self.errorhandling(err, 'setLinkId.write'); - callback && callback(err); - }); +function getIndexPage(self, offset) { + return ((offset - HEADERSIZE) / (self.header.pagesize + (self.header.pagelimit * self.header.documentsize))); +} - return self; -}; +function getDocumentIndex(self, pageindex, count) { + return ((pageindex - 1) * self.header.pagelimit) + (count || 1); +} -GP.findRelation = function(metaid, preparator) { - var self = this; - self.read(metaid, function(err, doc, linkid) { - preparator(doc || [], linkid ? () => self.findRelation(linkid, preparator) : null); - }); - return self; -}; +function checkRelation(self, relation, indexA, indexB, callback) { -GP.findRelationCount = function(cls, callback, count) { - var self = this; + self.read(indexA, function(err, docs, relid) { - if (!count) - count = 0; + if (docs) { + for (var i = 0; i < docs.length; i++) { + var doc = docs[i]; + if (doc.ID === indexB && (relation.both || doc.INIT)) { + callback(null, true); + return; + } + } + } - self.read(typeof(cls) === 'number' ? cls : self.$classes[cls].nodeid, function(err, doc, linkid) { - count += doc.length; - if (linkid) - self.findRelationCount(linkid, callback, count); + if (relid) + setImmediate(checkRelation, self, relation, relid, indexB, callback); else - callback(err, count); + callback(null, false); }); - - return self; -}; - -function makeRelation(self) { - var id = self.index++; - var buf = U.createBufferSize(self.header.size); - buf.writeUInt8(NODE_LINKS); // type, 1b - buf.writeUInt32BE(0, 1); // link, 4b - buf.writeUInt32BE(0, 5); // parent, 4b - buf.writeUInt16BE(0, 9); // count, 2b - - self.bufferappend.push({ id: id, buf: buf }); - self.flush(); - return id; } -function pushRelationNode(self, item, id) { - - // Must contain: - // item.nodeid +function updMeta(self, type) { + var buf; + switch (type) { + + case META_PAGE_ADD: + buf = U.createBufferSize(4); + buf.writeUInt32LE(self.header.pages); + Fs.write(self.fd, buf, 0, buf.length, 31, self.cb_error); + break; + + case META_PAGE_ADD3: + buf = U.createBufferSize(4); + buf.writeUInt32LE(self.header.pages, 0); + Fs.write(self.fd, buf, 0, buf.length, 31, function() { + buf.writeUInt32LE(self.header.relationpageindex, 0); + Fs.write(self.fd, buf, 0, buf.length, 47, self.cb_error); + }); + break; - self.pending[item.nodeid] = 1; + case META_RELATIONPAGEINDEX: + buf = U.createBufferSize(4); + buf.writeUInt32LE(self.header.relationpageindex, 0); + Fs.write(self.fd, buf, 0, buf.length, 47, self.cb_error); + break; - pushRelation(self, item.nodeid, id, 0, 0, function(err, nodeid) { + case META_CLASSESRELATIONS: - delete self.pending[item.nodeid]; + var obj = {}; + obj.c = []; // classes + obj.r = []; // relations - err && self.errorhandling(err, 'pushRelationNode'); + for (var i = 0; i < self.header.classindex; i++) { + var item = self.$classes[i + 1]; + obj.c.push({ n: item.name, i: item.index, p: item.pageindex, r: item.schema.raw, d: item.documentindex }); + } - if (item.nodeid !== nodeid) { - item.nodeid = nodeid; - self.$flushmeta(); - } + for (var i = 0; i < self.header.relationindex; i++) { + var item = self.$relations[i + 1]; + obj.r.push({ n: item.name, i: item.index, p: item.pageindex, b: item.both ? 1 :0, d: item.documentindex }); + } - }); + buf = U.createBufferSize(HEADERSIZE - 45); + buf.writeUInt8(self.header.classindex, 0); + buf.writeUInt8(self.header.relationindex, 1); + buf.writeUInt32LE(self.header.relationpageindex, 2); + buf.write(JSON.stringify(obj), 6); + Fs.write(self.fd, buf, 0, buf.length, 45, self.cb_error); + break; + } } -function pushRelation(self, id, toid, type, initializer, callback, parentid) { +function insDocument(self) { - var pos = HEADERSIZE + ((id - 1) * self.header.size); - var buf = U.createBufferSize(self.header.size); + if (!self.ready || self.states.insert || !self.pending.insert.length) + return; - if (id == null) { + var doc = self.pending.insert.shift(); + if (doc) { - id = self.index++; - - console.log('CREATE:', 'id:' + id, 'toid:' + toid, 'parentid:' + parentid); + var cls = self.$classes[doc.name]; + if (cls == null) { + doc.callback(new Error('GraphDB: Class "{0}" not found.'.format(doc.name))); + return; + } - buf.writeUInt8(NODE_LINKS); // type, 1b - buf.writeUInt32BE(parentid || 0, 1); // link, 4b - buf.writeUInt32BE(id, 5); // parent, 4b - buf.writeUInt16BE(1, 9); // count, 2b + if (cls.locked || !cls.ready) { + self.pending.insert.push(doc); + setTimeout(self.cb_next, DELAY, NEXT_INSERT); + return; + } - buf.writeUInt8(type, 11); // TYPE - buf.writeUInt8(initializer ? 1 : 0, 12); // RELATION TYPE - buf.writeUInt32BE(toid, 13); + self.states.insert = true; - self.bufferappend.push({ id: id, buf: buf }); - self.flush(); + addDocument(self, cls, doc.value, function(err, id) { + // setTimeout(insDocument, 100, self); + self.states.insert = false; + setImmediate(insDocument, self); + doc.callback(err, id); + }); + } +} - callback && callback(null, id); +function updDocument(self) { - } else { + if (!self.ready || self.states.update || !self.pending.update.length) + return; - var cache = self.bufferupdate.findItem('id', id); - if (cache == null) - cache = self.bufferappend.findItem('id', id); + var upd = self.pending.update.shift(); + if (upd) { + self.states.update = true; - var max = ((self.header.size - 11) / 6 >> 0); // 6b --> TYPE 1b + RELATION 1b + ID 4b + var offset = offsetDocument(self, upd.id); + var buf = U.createBufferSize(self.header.documentsize); - var update = function(buf, isCache) { - var count = buf.readUInt16BE(9); + Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - if (count + 1 >= max) { + if (err) { + self.states.update = false; + upd.callback(err); + setImmediate(self.cb_next, NEXT_UPDATE); + return; + } - console.log('INSERT:', 'id:' + id, 'toid:' + toid); + if (!size) { + upd.callback(null, 0); + self.states.update = false; + setImmediate(self.cb_next, NEXT_UPDATE); + return; + } - // We need to create a new record because existing buffer is full - pushRelation(self, null, toid, type, initializer, function(err, id) { - // We return a new id - callback && callback(err, id); - }, id); + var save = function(err) { + self.states.update = false; + !err && Fs.write(self.fd, buf, 0, buf.length, offset, self.cb_error); + upd.callback(err, err ? 0 : 1); + setImmediate(self.cb_next, NEXT_UPDATE); + }; + var data = buf.slice(DATAOFFSET, buf.readUInt16LE(15) + DATAOFFSET); + var limit = self.header.documentsize - DATAOFFSET; + var schema = self.$classes[buf[1]].schema; + var doc; + + if (buf[2] === STATE_COMPRESSED) { + Zlib.inflate(data, ZLIBOPTIONS, function(err, buffer) { + doc = parseData(schema, buffer.toString('utf8').split('|')); + buffer = U.createBuffer(stringifyData(schema, upd.fn(doc, upd.value))); + if (buffer.length > limit) { + Zlib.deflate(buffer, ZLIBOPTIONS, function(err, buffer) { + if (buffer.length <= limit) { + buf.writeUInt16LE(buffer.length, 15); + buf.writeUInt8(STATE_COMPRESSED, 2); + buffer.copy(buf, DATAOFFSET); + save(); + } else + save(new Error('GraphDB: Data too long')); + }); + } else { + buf.writeUInt16LE(buffer.length, 15); + buf.writeUInt8(STATE_NORMAL, 2); + buffer.copy(buf, DATAOFFSET); + save(); + } + }); } else { + doc = parseData(schema, data.toString('utf8').split('|')); + var o = stringifyData(schema, upd.fn(doc, upd.value)); + var buffer = U.createBuffer(o); + if (buffer.length > limit) { + Zlib.deflate(buffer, ZLIBOPTIONS, function(err, buffer) { + if (buffer.length <= limit) { + buf.writeUInt16LE(buffer.length, 15); + buf.writeUInt8(STATE_COMPRESSED, 2); + buffer.copy(buf, DATAOFFSET); + save(); + } else + save(new Error('GraphDB: Data too long')); + }); + } else { + buf.writeUInt16LE(buffer.length, 15); + buf.writeUInt8(STATE_NORMAL, 2); + buffer.copy(buf, DATAOFFSET); + save(); + } + } + }); + } +} + +function insRelation(self) { - // We add a new relation to existing list - buf.writeUInt16BE(count + 1, 9); // count, 2b + if (!self.ready || self.states.relation) + return; - var off = 11 + (count * 6); + var doc = self.pending.relation.shift(); + if (doc) { - buf.writeUInt8(type, off); - buf.writeUInt8(initializer ? 1 : 0, off + 1); - buf.writeUInt32BE(toid, off + 2); + var rel = self.$relations[doc.name]; + if (rel == null) { + doc.callback(new Error('GraphDB: Relation "{0}" not found.'.format(doc.name))); + return; + } - if (!isCache) - self.bufferupdate.push({ id: id, buf: buf, offset: 0 }); + if (rel.locked || !rel.ready) { + self.pending.relation.push(doc); + setTimeout(insRelation, DELAY, self); + return; + } - callback && callback(null, id); - } - }; + self.states.relation = true; - if (cache) { - update(cache.buf); + if (doc.connect) { + addRelation(self, rel, doc.indexA, doc.indexB, function(err, id) { + self.states.relation = false; + doc.callback(err, id); + setImmediate(insRelation, self); + }); } else { - Fs.read(self.fd, buf, 0, buf.length, pos, function(err) { - if (err) { - callback(err); - return; - } else if (buf[0] !== NODE_LINKS) { - callback(new Error('GraphDB: Invalid value for linking.')); - return; - } - update(buf); + remRelation(self, rel, doc.indexA, doc.indexB, function(err, id) { + self.states.relation = false; + doc.callback(err, id); + setImmediate(insRelation, self); }); } } - - return self; } -function updateRelation(self, metaid, toid, type, initializer) { - - var pos = HEADERSIZE + ((metaid - 1) * self.header.size); - var buf = U.createBufferSize(self.header.size); +GP.create = function(filename, callback) { + var self = this; + Fs.unlink(filename, function() { + var buf = U.createBufferSize(HEADERSIZE); + buf.write('Total.js GraphDB embedded', 0); + buf.writeUInt8(VERSION, 30); // version + buf.writeUInt32LE(0, 31); // pages + buf.writeUInt16LE(PAGESIZE, 35); // pagesize + buf.writeUInt16LE(PAGELIMIT, 37); // pagelimit + buf.writeUInt32LE(0, 39); // documents + buf.writeUInt16LE(DOCUMENTSIZE, 43); // documentsize + buf.writeUInt8(0, 45); // classindex + buf.writeUInt8(0, 46); // relationindex + buf.writeUInt8(0, 47); // relationpageindex + buf.write('{"c":[],"r":[]}', 51); // classes and relations + Fs.open(filename, 'w', function(err, fd) { + Fs.write(fd, buf, 0, buf.length, 0, function(err) { + err && self.error(err, 'create'); + Fs.close(fd, function() { + callback && callback(); + }); + }); + }); + }); + return self; +}; - Fs.read(self.fd, buf, 0, buf.length, pos, function(err, size) { +GP.open = function() { + var self = this; + Fs.stat(self.filename, function(err, stat) { + if (err) { + // file not found + self.create(self.filename, () => self.open()); + } else { + self.header.size = stat.size; + Fs.open(self.filename, 'r+', function(err, fd) { + self.fd = fd; + err && self.error(err, 'open'); + var buf = U.createBufferSize(HEADERSIZE); + Fs.read(self.fd, buf, 0, buf.length, 0, function() { + + self.header.pages = buf.readUInt32LE(31); + self.header.pagesize = buf.readUInt16LE(35); + self.header.pagelimit = buf.readUInt16LE(37); + self.header.documents = buf.readUInt32LE(39); + self.header.documentsize = buf.readUInt16LE(43); + self.header.relationlimit = ((self.header.documentsize - DATAOFFSET) / 6) >> 0; + self.header.classindex = buf[45]; + self.header.relationindex = buf[46]; + self.header.relationpageindex = buf.readUInt32LE(47); + + var data = buf.slice(51, buf.indexOf(EMPTYBUFFER, 51)).toString('utf8'); + var meta = data.parseJSON(true); + + for (var i = 0; i < meta.c.length; i++) { + var item = meta.c[i]; + self.class(item.n, item.r, item); + } - if (err || !size) - return; + for (var i = 0; i < meta.r.length; i++) { + var item = meta.r[i]; + self.relation(item.n, item.b === 1, item); + } - var count = buf.readUInt16BE(9); + !self.header.relationpageindex && addPage(self, TYPE_RELATION_DOCUMENT, 0, 0, function(err, index) { + self.header.relationpageindex = index; + }); - for (var i = 0; i < count; i++) { - var off = 11 + (i * 6); - if (buf.readUInt32BE(off + 2) === toid) { - // we have relation - // update - var data = U.createBufferSize(2); - data.writeUInt8(type == null ? buf[off] : type); - data.writeUInt8(initializer == null ? buf[off + 1] : initializer, 1); - Fs.write(self.fd, data, 0, data.length, pos + off, function(err) { - err && self.errorhandling(err, 'updateRelation.write'); + self.ready = true; + self.next(NEXT_READY); }); - } + }); } - }); -} + return self; +}; -GP.connect = function(type, fromid, toid, callback) { +GP.next = function(type) { var self = this; + var tmp; + switch (type) { + case NEXT_READY: + for (var i = 0; i < self.pending.meta.length; i++) { + tmp = self.pending.meta[i]; + if (tmp.type === TYPE_CLASS) + self.class(tmp.name, tmp.data); + else + self.relation(tmp.name, tmp.data); + } + self.emit('ready'); + break; + case NEXT_INSERT: + insDocument(self); + break; + case NEXT_RELATION: + insRelation(self); + break; + case NEXT_UPDATE: + updDocument(self); + break; + case NEXT_REMOVE: + remDocument(self); + break; + case NEXT_FIND: + if (self.pending.find.length) { + tmp = self.pending.find.shift(); + $find(self, tmp.name, tmp.builder, tmp.reverse); + } + break; + } +}; - if (self.$ready && !self.pending[fromid] && !self.pending[toid]) { +GP.class = function(name, meta, data) { - var relation = self.$relations[type]; - if (relation == null) { - callback(new Error('GraphDB: relation "{0}" not found'.format(type))); - return self; - } + var self = this; - if (relation.nodeid) { - pushRelationNode(self, relation, fromid); - relation.relation && pushRelationNode(self, relation, toid); - } + if (!self.ready && !data) { + self.pending.meta.push({ name: name, data: meta, type: 1 }); + return self; + } - self.pending[fromid] = 1; - self.pending[toid] = 1; + var item = self.$classes[name]; + var save = false; - var async = []; - var aid = 0; - var bid = 0; + if (item == null) { - async.push(function(next) { - self.getLinkId(fromid, function(err, id) { - aid = err ? null : id; - next(); - }); - }); + item = {}; + item.locked = false; - async.push(function(next) { - self.getLinkId(toid, function(err, id) { - bid = err ? null : id; - next(); + if (data) { + item.ready = true; + item.name = name; + item.index = data.i; + item.pageindex = data.p; + item.documentindex = data.d; + item.findfreeslots = true; + } else { + self.header.classindex++; + item.name = name; + item.index = self.header.classindex; + item.ready = false; + item.pageindex = addPage(self, TYPE_CLASS, item.index, 0, function() { + item.ready = true; }); - }); + item.documentindex = getDocumentIndex(self, item.pageindex); + save = true; + } - async.push(function(next) { + item.schema = parseSchema(meta); + self.$classes[item.name] = self.$classes[item.index] = item; - if (aid == null) { - async.length = 0; - next = null; - callback(new Error('GraphDB node (from) with "id:{0}" doesn\'t exist'.format(fromid))); - } else if (bid == null) { - async.length = 0; - next = null; - callback(new Error('GraphDB node (to) with "id:{0}" doesn\'t exist'.format(toid))); - } else - next(); + } else { + // compare + } - if (next == null) { - delete self.pending[fromid]; - delete self.pending[toid]; - } - }); + save && updMeta(self, META_CLASSESRELATIONS); + return self; +}; - // relation: 1/true - they know each other - // relation: 0 - "toid" doesn't know "fromid" +GP.relation = function(name, both, data) { - async.push(function(next) { - pushRelation(self, aid == 0 ? null : aid, toid, relation.index, true, function(err, id) { - if (aid !== id) { - aid = id; - self.setLinkId(fromid, id, next); - } else - next(); - }); - }); + var self = this; - async.push(function(next) { - pushRelation(self, bid == 0 ? null : bid, fromid, relation.index, false, function(err, id) { - if (bid !== id) { - bid = id; - self.setLinkId(toid, id, next); - } else - next(); + if (!self.ready && !data) { + self.pending.meta.push({ name: name, data: both, type: 2 }); + return self; + } + + var self = this; + var item = self.$relations[name]; + var save = false; + + if (item == null) { + + item = {}; + item.ready = true; + item.locked = false; + + if (data) { + item.name = name; + item.index = data.i; + item.pageindex = data.p; + item.documentindex = data.d; + item.both = both; + } else { + self.header.relationindex++; + item.name = name; + item.index = self.header.relationindex; + item.ready = false; + item.both = both; + item.pageindex = addPage(self, TYPE_RELATION, item.index, 0, function() { + item.ready = true; }); - }); + item.documentindex = getDocumentIndex(self, item.pageindex); + save = true; + } - async.async(function() { - delete self.pending[fromid]; - delete self.pending[toid]; - callback && callback(null, aid, bid); - self.flush(); - }); + self.$relations[item.name] = self.$relations[item.index] = item; } else { - setTimeout(self.$cb_join, DELAY, type, fromid, toid, callback); + // compare } + save && updMeta(self, META_CLASSESRELATIONS); return self; }; -GP.setDataType = function(id, type, callback) { +GP.emit = function(name, a, b, c, d, e, f, g) { + var evt = this.$events[name]; + if (evt) { + var clean = false; + for (var i = 0, length = evt.length; i < length; i++) { + if (evt[i].$once) + clean = true; + evt[i].call(this, a, b, c, d, e, f, g); + } + if (clean) { + evt = evt.remove(n => n.$once); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + } + return this; +}; + +GP.on = function(name, fn) { + if (this.$ready && (name === 'ready' || name === 'load')) { + fn(); + return this; + } + if (!fn.$once) + this.$free = false; + if (this.$events[name]) + this.$events[name].push(fn); + else + this.$events[name] = [fn]; + return this; +}; + +GP.once = function(name, fn) { + fn.$once = true; + return this.on(name, fn); +}; + +GP.removeListener = function(name, fn) { + var evt = this.$events[name]; + if (evt) { + evt = evt.remove(n => n === fn); + if (evt.length) + this.$events[name] = evt; + else + this.$events[name] = undefined; + } + return this; +}; + +GP.removeAllListeners = function(name) { + if (name === true) + this.$events = EMPTYOBJECT; + else if (name) + this.$events[name] = undefined; + else + this.$events[name] = {}; + return this; +}; + +function $update(doc, value) { + return value; +} + +function $modify(doc, value) { + var keys = Object.keys(value); + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + + switch (key[0]) { + case '+': + case '-': + case '*': + case '/': + var tmp = key.substring(1); + if (typeof(doc[tmp]) === 'number') { + if (key[0] === '+') + doc[tmp] += value[key]; + else if (key[0] === '-') + doc[tmp] -= value[key]; + else if (key[0] === '*') + doc[tmp] *= value[key]; + else if (key[0] === '/') + doc[tmp] = doc[tmp] / value[key]; + } + break; + default: + if (doc[key] != undefined) + doc[key] = value[key]; + break; + } + } + return doc; +} + +GP.remove = function(id, callback) { var self = this; - var pos = HEADERSIZE + ((id - 1) * self.header.size); - var buf = U.createBufferSize(1); - buf.writeUInt8(type); - Fs.write(self.fd, buf, 0, buf.length, pos, function(err) { - callback && callback(err); - }); + var rem = { id: id, callback: callback || NOOP }; + self.pending.remove.push(rem); + self.next(NEXT_REMOVE); return self; }; -GP.flush = function() { +GP.update = function(id, value, callback) { + var self = this; + var upd = { id: id, value: value, fn: typeof(value) === 'function' ? value : $update, callback: callback || NOOP }; + self.pending.update.push(upd); + self.next(NEXT_UPDATE); + return self; +}; +GP.modify = function(id, value, callback) { var self = this; + var upd = { id: id, value: value, fn: $modify, callback: callback || NOOP }; + self.pending.update.push(upd); + self.next(NEXT_UPDATE); + return self; +}; - if (!self.$ready) - return self; +GP.insert = function(name, value, callback) { + var self = this; + self.pending.insert.push({ name: name, value: value, callback: callback || NOOP }); + self.next(NEXT_INSERT); + return self; +}; - if (self.bufferupdate.length && !self.updating) { - self.updating = true; - var doc = self.bufferupdate.shift(); - var offset = HEADERSIZE + ((doc.id - 1) * self.header.size); - if (doc.recovered) { - var buf = U.createBufferSize(1); - buf.writeUInt8(doc.type); - Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { - if (err) { - if (self.$events.error) - self.emit('error', err); - else - F.error(err, 'GraphDB: ' + self.name); - } - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + doc.offset, self.$cb_writeupdate); - }); - } else - Fs.write(self.fd, doc.buf, 0, doc.buf.length, offset + doc.offset, self.$cb_writeupdate); - doc.callback && doc.callback(null, doc.id); - return self; - } +GP.cursor = function(type, name, callback) { - if (self.bufferappend.length && !self.appending) { + var self = this; + var index; + var tmp; + + switch (type) { + case TYPE_CLASS: + tmp = self.$classes[name]; + index = tmp.pageindex; + break; + case TYPE_RELATION: + tmp = self.$relations[name]; + index = tmp.pageindex; + break; + } - var flush = self.bufferappend.splice(0, self.header.buffersize); - var buf = []; + var offset = offsetPage(self, index); + var buf = U.createBufferSize(PAGESIZE); - for (var i = 0; i < flush.length; i++) { - var f = flush[i]; - buf.push(f.buf); - f.callback && f.callback(null, f.id); + Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - if (f.schemaid) { - console.log('PUSHID', f.id); - pushRelationNode(self, self.$classes[f.schemaid], f.id); - } + if (err) { + callback(err); + return; } - var data = Buffer.concat(buf); - self.appending = true; - Fs.write(self.fd, data, 0, data.length, self.stat.size, self.$cb_writeappend); - } - - return self; -}; + if (buf[0] !== TYPE_CLASS) { + callback(new Error('Invalid page type')); + return; + } -GP.remove = function(id, callback) { - var self = this; + if (buf[1] !== tmp.index) { + callback(new Error('Invalid type index')); + return; + } - if (!self.$ready || self.pending[id]) { - setTimeout(self.$cb_remove, DELAY, id, callback); - return self; - } + var data = {}; + data.count = buf.readUInt16LE(2); + data.parent = buf.readUInt32LE(5); + data.offset = offset; + data.type = buf[0]; + data.index = buf[1]; + data.freeslots = buf[4]; - self.pending[id] = 1; + data.next = function(callback) { - var removed = []; + if (data.parent == 0) { + callback(new Error('This is the last page'), data); + return; + } - var done = function() { - removed.wait(function(id, next) { - self.setDataType(id, NODE_REMOVED, function() { - self.$events.remove && self.emit('remove', id); - next(); + offset = offsetPage(self, data.parent); + Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { + data.count = buf.readUInt16LE(2); + data.parent = buf.readUInt32LE(5); + data.offset = offset; + data.type = buf[0]; + data.index = buf[1]; + data.freeslots = buf[4]; + data.INDEX = getIndexPage(self, offset) + 1; + callback(err, data); }); - }, function() { - delete self.pending[id]; - callback && callback(null, removed.length); - }); - }; + }; - var find = function(id, level) { - var buf = U.createBufferSize(5); - Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { + data.documents = function(callback) { - if (!size || buf[0] === NODE_REMOVED) { - done(); + if (!data.count) { + callback(null, EMPTYARRAY); return; } - removed.push(id); + var index = getIndexPage(self, data.offset) * self.header.pagelimit; + var buffer = U.createBufferSize(self.header.pagelimit * self.header.documentsize); + var offset = data.offset + self.header.pagesize; + var decompress = []; - var link = buf.readUInt32BE(1); - if (link) - find(link, level + 1); - else { - // Doesn't have any relations, it's free - if (level === 0) - self.bufferremove.push(id); - done(); - } - }); - }; + index += self.header.pagelimit + 1; - find(id, 0); - return self; -}; + Fs.read(self.fd, buffer, 0, buffer.length, offset, function(err) { -GP.update = function(id, value, callback) { - var self = this; - self.read(id, function(err, doc, linkid, cls) { - if (err || !doc) - callback(err, null); - else - self.insert(cls, typeof(value) === 'function' ? value(doc) : doc, callback, id); - }); - return self; -}; + if (err) { + callback(err, EMPTYARRAY); + return; + } -GP.update2 = function(cls, id, value, callback) { - var self = this; - self.insert(cls, value, callback, id); - return self; -}; + var arr = []; + while (true) { -GP.modify = function(id, value, callback) { - var self = this; - self.read(id, function(err, doc) { + if (!buffer.length) + break; - var data = framework_builders.isSchema(value) ? value.$clean() : value; - var keys = Object.keys(data); - var is = false; + index--; + var data = buffer.slice(buffer.length - self.header.documentsize); + // index++; + // var data = buffer.slice(0, self.header.documentsize); + if (!data.length) + break; - for (var i = 0; i < keys.length; i++) { - var key = keys[i]; - switch (key[0]) { - case '+': - case '-': - case '*': - case '/': - var tmp = key.substring(1); - if (typeof(doc[tmp]) === 'number') { - if (key[0] === '+') { - doc[tmp] += data[key]; - is = true; - } else if (key[0] === '-') { - doc[tmp] -= data[key]; - is = true; - } else if (key[0] === '*') { - doc[tmp] *= data[key]; - is = true; - } else if (key[0] === '/') { - doc[tmp] = doc[tmp] / data[key]; - is = true; + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // continuerindex (4b) = from: 7 + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 + + if (data[2] !== STATE_REMOVED) { + var raw = data.slice(DATAOFFSET, data.readUInt16LE(15) + DATAOFFSET); + if (type === TYPE_CLASS) { + // Document is compressed + if (data[2] === STATE_COMPRESSED) { + var obj = {}; + obj.CLASS = tmp.name; + obj.ID = index; + obj.BUFFER = raw; + decompress.push({ CLASS: tmp, ID: index, BUFFER: raw, index: arr.push(null) }); + } else { + var obj = parseData(tmp.schema, raw.toString('utf8').split('|')); + obj.CLASS = tmp.name; + obj.ID = index; + arr.push(obj); + } } } - break; - default: - if (doc[key] != undefined) { - doc[key] = data[key]; - is = true; - } - break; - } - } - if (is) - self.insert(0, doc, callback, id); - else - callback(null, id); - }); + buffer = buffer.slice(0, buffer.length - self.header.documentsize); + // buffer = buffer.slice(self.header.documentsize); + } - return self; + if (decompress.length) { + decompress.wait(function(item, next) { + Zlib.inflate(item.BUFFER, ZLIBOPTIONS, function(err, data) { + var obj = parseData(item.CLASS.schema, data.toString('utf8').split('|')); + obj.CLASS = item.CLASS.name; + obj.ID = item.ID; + arr[item.index] = obj; + setImmediate(next); + }); + }, () => callback(null, arr)); + } else + callback(null, arr); + }); + }; + + callback(null, data); + }); }; -GP.get = GP.read = function(id, callback, type, relation) { +GP.read = function(index, callback) { var self = this; - if (self.$ready && self.reading < MAXREADERS) { - - self.reading++; - var buf = U.createBufferSize(self.header.size); - Fs.read(self.fd, buf, 0, buf.length, HEADERSIZE + ((id - 1) * self.header.size), function(err, size) { + var buf = U.createBufferSize(self.header.documentsize); + Fs.read(self.fd, buf, 0, buf.length, offsetDocument(self, index), function(err) { - self.reading--; - var linkid = buf.readUInt32BE(1); - - switch (buf[0]) { - case NODE_REMOVED: - // 255: REMOVED DOCUMENT - callback(null, null, 0); - return; + if (err) { + callback(err); + return; + } - case NODE_LINKS: + if (buf[2] === STATE_REMOVED) { + callback(null, buf[0] === TYPE_CLASS ? null : EMPTYARRAY); + return; + } - // 254: LINKS - var count = buf.readUInt16BE(9); // 2b - var links = []; + var tmp; + + switch(buf[0]) { + case TYPE_CLASS: + tmp = self.$classes[buf[1]]; + if (tmp) { + var data = buf.slice(DATAOFFSET, buf.readUInt16LE(15) + DATAOFFSET); + if (buf[2] === STATE_COMPRESSED) { + Zlib.inflate(data, ZLIBOPTIONS, function(err, data) { + data = parseData(tmp.schema, data.toString('utf8').split('|')); + data.ID = index; + data.CLASS = tmp.name; + callback(null, data, buf.readUInt32LE(7), buf.readUInt32LE(11)); + }); + } else { + data = parseData(tmp.schema, data.toString('utf8').split('|')); + data.ID = index; + data.CLASS = tmp.name; + callback(null, data, buf.readUInt32LE(7), buf.readUInt32LE(11)); + } + } else + callback(new Error('GraphDB: invalid document'), null); + break; + case TYPE_RELATION: + tmp = self.$relations[buf[1]]; + if (tmp) { + var count = buf.readUInt16LE(15); + var arr = []; for (var i = 0; i < count; i++) { - var pos = 11 + (i * 6); - if (type == null && relation == null) - links.push({ RELATION: buf[pos], TYPE: buf[pos + 1], ID: buf.readUInt32BE(pos + 2), INDEX: i }); - else if ((type == null || (type == buf[pos]) && (relation == null || (relation == buf[pos + 1])))) - links.push(buf.readUInt32BE(pos + 2)); + var off = DATAOFFSET + (i * 6); + arr.push({ RELATION: tmp.name, ID: buf.readUInt32LE(off + 2), INIT: buf[1], INDEX: i }); } - links.reverse(); - callback(err, links, linkid); - return; + callback(null, arr, buf.readUInt32LE(7), buf.readUInt32LE(11), 'RELATION'); - default: - if (size) { - var data = buf.slice(11, buf.readUInt16BE(9) + 11); - if (buf[0] == 0) - callback(err, (eval('({' + data.toString('utf8') + '})')), linkid, buf[0]); - else { - var schema = self.$classes[buf[0]]; - if (schema) - callback(err, parseData(schema, data.toString('utf8').split('|')), linkid, schema.name); - else - callback(new Error('GraphDB: Class not found.'), null, 0, 0); - } - } else - callback(null, null, 0, 0); - return; - } - }); - } else { - setTimeout(self.$cb_get, DELAY, id, callback, type, relation); - } + } else + callback(new Error('GraphDB: invalid document'), null); + break; + + case TYPE_RELATION_DOCUMENT: + + var count = buf.readUInt16LE(15); + var arr = []; + + for (var i = 0; i < count; i++) { + var off = DATAOFFSET + (i * 6); + tmp = self.$relations[buf[off]]; + arr.push({ RELATION: tmp.name, ID: buf.readUInt32LE(off + 2), INIT: buf[off + 1], INDEX: i }); + } + + callback(null, arr, buf.readUInt32LE(7), buf.readUInt32LE(11), 'PRIVATE'); + break; + + default: + callback(null, null); + break; + } + }); +}; + +GP.connect = function(name, indexA, indexB, callback) { + var self = this; + self.pending.relation.push({ name: name, indexA: indexA, indexB: indexB, callback: callback, connect: true }); + self.next(NEXT_RELATION); + return self; +}; +GP.disconnect = function(name, indexA, indexB, callback) { + var self = this; + self.pending.relation.push({ name: name, indexA: indexA, indexB: indexB, callback: callback }); + self.next(NEXT_RELATION); return self; }; -GP.find = function(type, builder) { +GP.find = function(cls) { var self = this; var builder = new DatabaseBuilder(self); - if (self.$ready && self.reading < MAXREADERS) - setImmediate($find, self, type, builder); - else { - setTimeout(self.$cb_find, DELAY, type, builder); - } + self.pending.find.push({ name: cls, builder: builder }); + setImmediate(self.cb_next, NEXT_FIND); return builder; }; -GP.find2 = function(type, builder) { +GP.find2 = function(cls) { var self = this; var builder = new DatabaseBuilder(self); - if (self.$ready && self.reading < MAXREADERS) - setImmediate($find, self, type, builder, true); - else { - setTimeout(self.$cb_find, DELAY, type, builder, true); - } + self.pending.find.push({ name: cls, builder: builder, reverse: true }); + setImmediate(self.cb_next, NEXT_FIND); return builder; }; -function $find(self, type, builder, reverse) { - - var index = reverse ? (self.header.count - 1) : 0; - var size = self.header.size * self.header.buffersize; - var filter = {}; - - filter.builder = builder; - filter.scalarcount = 0; - filter.filter = builder.makefilter(); - filter.compare = builder.compile(); - filter.index = 0; - filter.count = 0; - filter.counter = 0; - filter.first = builder.$options.first && !builder.$options.sort; - - var classes = null; - - if (type) { - var cls; - classes = {}; - if (type instanceof Array) { - for (var i = 0; i < type.length; i++) { - cls = self.$classes[type[i]]; - if (cls) - classes[cls.index] = 1; - } - } else { - cls = self.$classes[type]; - if (cls) - classes[cls.index] = 1; - } - } - - var reader = function() { - - var buf = U.createBufferSize(size); - var offset = HEADERSIZE + (index * self.header.size); - - if (reverse) - index -= self.header.buffersize; - else - index += self.header.buffersize; - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err || !size) { - framework_nosql.callback(filter); - return; - } - - while (true) { - - var buffer = buf.slice(0, self.header.size); - if (!buffer.length) - break; - - switch (buffer[0]) { - case NODE_REMOVED: - case NODE_LINKS: - break; - default: - if (!classes || classes[buffer[0]]) { - var docsize = buffer.readUInt16BE(9); - if (docsize) { - - filter.index++; - - var data = buffer.slice(11, docsize + 11).toString('utf8'); - var doc; - - if (buffer[0] == 0) - doc = (eval('({' + data + '})')); - else { - var schema = self.$classes[buffer[0]]; - if (schema) { - doc = parseData(schema, data.split('|')); - doc.CLASS = schema.name; - } - } - - if ((doc && framework_nosql.compare(filter, doc) === false) || (reverse && filter.done)) { - framework_nosql.callback(filter); - return; - } - } - } - break; - } - - buf = buf.slice(self.header.size); - } - - setImmediate(reader); - }); - }; - - setImmediate(reader); -} - function GraphDBFilter(db) { var t = this; t.db = db; @@ -1571,17 +1907,10 @@ GP.graph = function(id, options, callback, filter) { if (!filter) filter = new GraphDBFilter(self); - if (!self.$ready || self.reading >= MAXREADERS) { - setTimeout(self.$cb_graph, DELAY, id, options, callback, filter); - return filter; - } - - self.reading++; - self.read(id, function(err, doc, linkid, cls) { + self.read(id, function(err, doc, linkid) { if (err || !doc) { - self.reading--; callback(err, null, 0); return; } @@ -1602,12 +1931,12 @@ GP.graph = function(id, options, callback, filter) { for (var i = 0; i < options.relation.length; i++) { rel = self.$relations[options.relation[i]]; if (rel) - relations[rel.index] = rel.relation; + relations[rel.name] = rel.both ? 1 : 0; } } else { rel = self.$relations[options.relation]; if (rel) - relations[rel.index] = rel.relation; + relations[rel.name] = rel.both ? 1 : 0; } } @@ -1620,12 +1949,12 @@ GP.graph = function(id, options, callback, filter) { for (var i = 0; i < options.class.length; i++) { clstmp = self.$classes[options.class[i]]; if (clstmp) - classes[clstmp.index] = 1; + classes[clstmp.name] = 1; } } else { clstmp = self.$classes[options.class]; if (clstmp) - classes[clstmp.index] = clstmp.class + 1; + classes[clstmp.name] = clstmp.index + 1; } } @@ -1638,17 +1967,13 @@ GP.graph = function(id, options, callback, filter) { tmp[id] = 1; - doc.ID = id; doc.INDEX = 0; doc.LEVEL = 0; doc.NODES = []; - if (cls) - doc.CLASS = cls; - var reader = function(parent, id, depth) { - if ((options.depth && depth > options.depth) || (tmp[id])) { + if ((options.depth && depth >= options.depth) || (tmp[id])) { process(); return; } @@ -1663,7 +1988,7 @@ GP.graph = function(id, options, callback, filter) { } // because of seeking on HDD - links.quicksort('id'); + links.quicksort('ID'); var fil; @@ -1671,17 +1996,17 @@ GP.graph = function(id, options, callback, filter) { var key = item.ID + '-' + item.RELATION; - if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && !item.TYPE && !relations) || (relations && relations[item.RELATION] === item.TYPE)) + if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && !item.INIT && !relations) || (relations && relations[item.RELATION] === item.TYPE)) return next(); tmp[key] = 1; - self.read(item.ID, function(err, doc, linkid, cls) { + self.read(item.ID, function(err, doc, linkid) { - if (doc && (!classes || classes[cls])) { + if (doc && (!classes || classes[doc.CLASS])) { count++; - doc.ID = item.ID; + doc.INDEX = item.INDEX; doc.LEVEL = depth + 1; doc.NODES = []; @@ -1693,9 +2018,6 @@ GP.graph = function(id, options, callback, filter) { doc.RELATION = rel.name; } - if (cls) - doc.CLASS = cls; - fil = filter.levels ? filter.levels[depth + 1] : null; if (fil) { @@ -1741,7 +2063,6 @@ GP.graph = function(id, options, callback, filter) { } } - self.reading--; callback(null, doc, count); } }; @@ -1754,17 +2075,90 @@ GP.graph = function(id, options, callback, filter) { return filter; }; -function stringify(obj) { - var val = JSON.stringify(obj).replace(REGKEY, stringifykey).replace(REGDATE, stringifydate); - return val.substring(1, val.length - 1); -} +function $find(self, cls, builder, reverse) { + + var filter = {}; + + filter.builder = builder; + filter.scalarcount = 0; + filter.filter = builder.makefilter(); + filter.compare = builder.compile(); + filter.index = 0; + filter.count = 0; + filter.counter = 0; + filter.first = builder.$options.first && !builder.$options.sort; + + var tmp = self.$classes[cls]; + if (!tmp) { + framework_nosql.callback(filter, 'GraphDB: Class "{0}" is not registered.'.format(cls)); + setImmediate(self.cb_next, NEXT_FIND); + return; + } + + var read = function(err, data) { + + if (err || (!data.count && !data.parent)) { + framework_nosql.callback(filter); + return; + } + + data.documents(function(err, docs) { + for (var i = 0; i < docs.length; i++) { + var doc = docs[i]; + filter.index++; + if ((doc && framework_nosql.compare(filter, doc) === false) || (reverse && filter.done)) { + framework_nosql.callback(filter); + data.next = null; + data.documents = null; + data = null; + setImmediate(self.cb_next, NEXT_FIND); + return; + } + } + data.next(read); + }); + }; -function stringifykey(text) { - return text.substring(1, text.length - 2) + ':'; + self.cursor(1, tmp.name, read); } -function stringifydate(text) { - return 'new Date(' + new Date(text.substring(1, text.length - 2)).getTime() + ')'; +function parseSchema(schema) { + + var obj = {}; + var arr = schema.split('|').trim(); + + obj.meta = {}; + obj.keys = []; + obj.raw = schema; + + for (var i = 0; i < arr.length; i++) { + var arg = arr[i].split(':'); + var type = 0; + switch ((arg[1] || '').toLowerCase().trim()) { + case 'number': + type = 2; + break; + case 'boolean': + case 'bool': + type = 3; + break; + case 'date': + type = 4; + break; + case 'object': + type = 5; + break; + case 'string': + default: + type = 1; + break; + } + var name = arg[0].trim(); + obj.meta[name] = { type: type, pos: i }; + obj.keys.push(name); + } + + return obj; } function stringifyData(schema, doc) { @@ -1815,45 +2209,6 @@ function stringifyData(schema, doc) { return (esc ? '*' : '+') + output; } -function parseSchema(schema) { - - var obj = {}; - var arr = schema.split('|').trim(); - - obj.meta = {}; - obj.keys = []; - obj.raw = schema; - - for (var i = 0; i < arr.length; i++) { - var arg = arr[i].split(':'); - var type = 0; - switch ((arg[1] || '').toLowerCase().trim()) { - case 'number': - type = 2; - break; - case 'boolean': - case 'bool': - type = 3; - break; - case 'date': - type = 4; - break; - case 'object': - type = 5; - break; - case 'string': - default: - type = 1; - break; - } - var name = arg[0].trim(); - obj.meta[name] = { type: type, pos: i }; - obj.keys.push(name); - } - - return obj; -} - function parseData(schema, lines, cache) { var obj = {}; From 6f1024ffc23f9708548fb65b4cb35def024468b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 12:18:21 +0200 Subject: [PATCH 0490/1669] Improved `F.stop()`. --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index e4031e670..aa680d84e 100755 --- a/index.js +++ b/index.js @@ -1342,7 +1342,11 @@ F.stop = F.kill = function(signal) { TRY(() => process.send('total:stop')); F.cache.stop(); - F.server && F.server.close && F.server.close(); + + if (F.server) { + F.server.setTimeout(1); + F.server.close(); + } setTimeout(() => process.exit(signal), global.TEST ? 2000 : 100); return F; From de1267823c68adea0b2e671b657d99e5782109af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 15:03:41 +0200 Subject: [PATCH 0491/1669] Updated GraphDB. --- graphdb.js | 333 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 309 insertions(+), 24 deletions(-) diff --git a/graphdb.js b/graphdb.js index 4dcbb5a2f..7ac92897a 100644 --- a/graphdb.js +++ b/graphdb.js @@ -43,8 +43,9 @@ const REGTESCAPE = /\||\n|\r/g; const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; const DatabaseBuilder = framework_nosql.DatabaseBuilder; -const STATE_NORMAL = 1; -const STATE_COMPRESSED = 100; +// STATES +const STATE_UNCOMPRESSED = 1; +const STATE_COMPRESSED = 2; const STATE_REMOVED = 255; // META @@ -60,6 +61,8 @@ const NEXT_RELATION = 3; const NEXT_UPDATE = 4; const NEXT_FIND = 5; const NEXT_REMOVE = 6; +const NEXT_RESIZE = 7; +const NEXT_CONTINUE = 100; // TYPES const TYPE_CLASS = 1; @@ -90,6 +93,7 @@ function GraphDB(name) { self.pending.meta = []; self.states = {}; + self.states.resize = false; self.states.insert = false; self.states.read = false; self.states.remove = false; @@ -186,7 +190,6 @@ function addPage(self, type, index, parentindex, callback) { var offset = offsetPage(self, indexer); Fs.write(self.fd, buffer, 0, buffer.length, offset, function(err) { - // @TODO: add check for bytes written and buffer size err && self.error(err, 'createPage.write'); !err && updMeta(self, type === TYPE_RELATION_DOCUMENT ? META_PAGE_ADD3 : META_PAGE_ADD); callback && callback(err, indexer); @@ -205,17 +208,18 @@ function addNodeFree(self, meta, callback) { findDocumentFree(self, meta.type.pageindex, function(err, documentindex, pageindex) { if (!documentindex) { + meta.type.findfreeslots = false; addNode(self, meta, callback); return; } var buffer = U.createBufferSize(self.header.documentsize); - buffer.writeUInt8(meta.typeid, 0); // type - buffer.writeUInt8(meta.type.index, 1); // index - buffer.writeUInt32LE(pageindex, 3); // pageindex - buffer.writeUInt8(meta.state || STATE_NORMAL, 2); // state - buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex - buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex + buffer.writeUInt8(meta.typeid, 0); // type + buffer.writeUInt8(meta.type.index, 1); // index + buffer.writeUInt32LE(pageindex, 3); // pageindex + buffer.writeUInt8(meta.state || STATE_UNCOMPRESSED, 2); // state + buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex + buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex buffer.writeUInt16LE(meta.size, 15); meta.data && meta.data.copy(buffer, DATAOFFSET); @@ -262,7 +266,7 @@ function addNode(self, meta, callback) { buffer.writeUInt8(buf[0], 0); // type buffer.writeUInt8(meta.type.index, 1); // index buffer.writeUInt32LE(meta.type.pageindex, 3); // pageindex - buffer.writeUInt8(meta.state || STATE_NORMAL, 2); // state + buffer.writeUInt8(meta.state || STATE_UNCOMPRESSED, 2); // state buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex buffer.writeUInt16LE(meta.size, 15); @@ -415,6 +419,10 @@ function addRelation(self, relation, indexA, indexB, callback) { // Obtaining indexA a relation document tasks.push(function(next) { + + if (F.isKilled) + return; + if (relA) next(); else { @@ -427,6 +435,10 @@ function addRelation(self, relation, indexA, indexB, callback) { // Obtaining indexB a relation document tasks.push(function(next) { + + if (F.isKilled) + return; + if (relB) next(); else { @@ -508,6 +520,10 @@ function remRelation(self, relation, indexA, indexB, callback) { }); tasks.async(function() { + + if (F.isKilled) + return; + remRelationLink(self, relA, indexB, function(err, countA) { remRelationLink(self, relB, indexA, function(err, countB) { remRelationLink(self, relation.documentindex, indexA, function(err, countC) { @@ -754,6 +770,10 @@ function findDocumentFree(self, pageindex, callback, ready) { break; if (data[2] === STATE_REMOVED) { + + if (F.isKilled) + return; + updPageMeta(self, pageindex, function(err, buf) { buf.writeUInt16LE(documents + 1, 2); setImmediate(callback, null, index, pageindex); @@ -828,8 +848,6 @@ function pushRelationDocument(self, index, relation, documentindex, initializato var count = buf.readUInt16LE(15); if (count + 1 > self.header.relationlimit) { - - // @TODO: Find free relation ID document findRelationDocument(self, buf.readUInt32LE(7), function(err, newindex) { // Is some relation document exist? @@ -849,7 +867,7 @@ function pushRelationDocument(self, index, relation, documentindex, initializato var meta = {}; meta.typeid = relation.private ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; meta.type = relation; - meta.state = STATE_NORMAL; + meta.state = STATE_UNCOMPRESSED; meta.parentindex = 0; meta.relationindex = index; meta.size = 0; @@ -899,7 +917,7 @@ function pushRelationDocument(self, index, relation, documentindex, initializato }); }); - buf.writeUInt8(STATE_NORMAL, 2); + buf.writeUInt8(STATE_UNCOMPRESSED, 2); } else { // DONE @@ -941,7 +959,7 @@ function updPageMeta(self, index, fn) { } function remDocument(self) { - if (!self.ready || self.states.remove || !self.pending.remove.length) + if (!self.ready || self.states.remove || !self.pending.remove.length || F.isKilled) return; self.states.remove = true; var doc = self.pending.remove.shift(); @@ -1004,7 +1022,6 @@ function remDocumentAll(self, index, callback, count) { var documents = buf.readUInt16LE(2); buf.writeUInt16LE(documents > 0 ? documents - 1 : documents, 2); - count++; setImmediate(function() { @@ -1110,7 +1127,7 @@ function updMeta(self, type) { function insDocument(self) { - if (!self.ready || self.states.insert || !self.pending.insert.length) + if (!self.ready || self.states.insert || !self.pending.insert.length || F.isKilled) return; var doc = self.pending.insert.shift(); @@ -1141,7 +1158,7 @@ function insDocument(self) { function updDocument(self) { - if (!self.ready || self.states.update || !self.pending.update.length) + if (!self.ready || self.states.update || !self.pending.update.length || F.isKilled) return; var upd = self.pending.update.shift(); @@ -1195,7 +1212,7 @@ function updDocument(self) { }); } else { buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_NORMAL, 2); + buf.writeUInt8(STATE_UNCOMPRESSED, 2); buffer.copy(buf, DATAOFFSET); save(); } @@ -1216,7 +1233,7 @@ function updDocument(self) { }); } else { buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_NORMAL, 2); + buf.writeUInt8(STATE_UNCOMPRESSED, 2); buffer.copy(buf, DATAOFFSET); save(); } @@ -1263,7 +1280,7 @@ function insRelation(self) { } } -GP.create = function(filename, callback) { +GP.create = function(filename, documentsize, callback) { var self = this; Fs.unlink(filename, function() { var buf = U.createBufferSize(HEADERSIZE); @@ -1273,7 +1290,7 @@ GP.create = function(filename, callback) { buf.writeUInt16LE(PAGESIZE, 35); // pagesize buf.writeUInt16LE(PAGELIMIT, 37); // pagelimit buf.writeUInt32LE(0, 39); // documents - buf.writeUInt16LE(DOCUMENTSIZE, 43); // documentsize + buf.writeUInt16LE(documentsize, 43); // documentsize buf.writeUInt8(0, 45); // classindex buf.writeUInt8(0, 46); // relationindex buf.writeUInt8(0, 47); // relationpageindex @@ -1295,7 +1312,7 @@ GP.open = function() { Fs.stat(self.filename, function(err, stat) { if (err) { // file not found - self.create(self.filename, () => self.open()); + self.create(self.filename, DOCUMENTSIZE, () => self.open()); } else { self.header.size = stat.size; Fs.open(self.filename, 'r+', function(err, fd) { @@ -1309,6 +1326,14 @@ GP.open = function() { self.header.pagelimit = buf.readUInt16LE(37); self.header.documents = buf.readUInt32LE(39); self.header.documentsize = buf.readUInt16LE(43); + + var size = F.config['graphdb.' + self.name] || DOCUMENTSIZE; + if (size > self.header.documentsize) { + setTimeout(function() { + self.next(NEXT_RESIZE); + }, DELAY); + } + self.header.relationlimit = ((self.header.documentsize - DATAOFFSET) / 6) >> 0; self.header.classindex = buf[45]; self.header.relationindex = buf[46]; @@ -1344,6 +1369,7 @@ GP.next = function(type) { var self = this; var tmp; + switch (type) { case NEXT_READY: for (var i = 0; i < self.pending.meta.length; i++) { @@ -1355,6 +1381,37 @@ GP.next = function(type) { } self.emit('ready'); break; + + case NEXT_RESIZE: + + clearTimeout(self.$resizedelay); + self.$resizedelay = setTimeout(function() { + if (!self.states.resize) { + self.ready = false; + self.states.resize = true; + var size = (F.config['graphdb.' + self.name] || DOCUMENTSIZE); + var meta = { documentsize: size > self.header.documentsize ? size : self.header.documentsize }; + var keys = Object.keys(self.$classes); + + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + var cls = self.$classes[key]; + if (cls.$resize) { + !meta.classes && (meta.classes = {}); + meta.classes[cls.index] = cls.$resize; + cls.$resize = null; + } + } + self.resize(meta, function() { + self.states.resize = false; + self.ready = true; + setImmediate(self.cb_next, NEXT_CONTINUE); + }); + } + }, DELAY); + + break; + case NEXT_INSERT: insDocument(self); break; @@ -1416,7 +1473,12 @@ GP.class = function(name, meta, data) { self.$classes[item.name] = self.$classes[item.index] = item; } else { - // compare + var newschema = parseSchema(meta); + var raw = item.schema.raw; + if (raw !== newschema.raw) { + item.$resize = newschema; + self.next(NEXT_RESIZE); + } } save && updMeta(self, META_CLASSESRELATIONS); @@ -1532,6 +1594,216 @@ GP.removeAllListeners = function(name) { return this; }; +GP.resize = function(meta, callback) { + + // meta.documentsize + // meta.classes + + var self = this; + var filename = self.filename + '-tmp'; + + self.create(filename, meta.documentsize, function(err) { + + if (err) + throw err; + + Fs.open(filename, 'r+', function(err, fd) { + + var offset = HEADERSIZE; + var newoffset = HEADERSIZE; + var size = self.header.pagesize + (self.header.pagelimit * self.header.documentsize); + var newsize = self.header.pagesize + (self.header.pagelimit * meta.documentsize); + var pageindex = 0; + var totaldocuments = 0; + + var finish = function() { + + var buf = U.createBufferSize(HEADERSIZE); + Fs.read(fd, buf, 0, buf.length, 0, function() { + + // ==== DB:HEADER (7000b) + // name (30b) = from: 0 + // version (1b) = from: 30 + // pages (4b) = from: 31 + // pagesize (2b) = from: 35 + // pagelimit (2b) = from: 37 + // documents (4b) = from: 39 + // documentsize (2b) = from: 43 + // classindex (1b) = from: 45 + // relationindex (1b) = from: 46 + // relationnodeindex = from: 47 + // classes + relations = from: 51 + + // buf. + + buf.writeUInt32LE(pageindex > 0 ? (pageindex - 1) : 0, 31); + buf.writeUInt32LE(totaldocuments, 39); + buf.writeUInt16LE(meta.documentsize, 43); + + var obj = {}; + obj.c = []; // classes + obj.r = []; // relations + + for (var i = 0; i < self.header.classindex; i++) { + var item = self.$classes[i + 1]; + var schema = meta.classes[i + 1]; + obj.c.push({ n: item.name, i: item.index, p: item.pageindex, r: schema ? schema.raw : item.schema.raw, d: item.documentindex }); + } + + for (var i = 0; i < self.header.relationindex; i++) { + var item = self.$relations[i + 1]; + obj.r.push({ n: item.name, i: item.index, p: item.pageindex, b: item.both ? 1 :0, d: item.documentindex }); + } + + buf.writeUInt8(self.header.classindex, 45); + buf.writeUInt8(self.header.relationindex, 46); + buf.writeUInt32LE(self.header.relationpageindex, 47); + buf.write(JSON.stringify(obj), 51); + + Fs.write(fd, buf, 0, buf.length, 0, function() { + // console.log(pageindex, meta.documentsize, totaldocuments); + Fs.close(fd, function() { + Fs.close(self.fd, function() { + Fs.copyFile(self.filename, self.filename.replace(/\.gdb$/, NOW.format('_yyyyMMddHHmm') + '.gdp'), function() { + Fs.rename(self.filename + '-tmp', self.filename, function() { + callback(null); + }); + }); + }); + }); + }); + }); + }; + + var readvalue = function(docbuf, callback) { + var data = docbuf.slice(DATAOFFSET, docbuf.readUInt16LE(15) + DATAOFFSET); + if (docbuf[2] === STATE_COMPRESSED) + Zlib.inflate(data, ZLIBOPTIONS, (err, data) => callback(data ? data.toString('utf8') : '')); + else + callback(data.toString('utf8')); + }; + + var writevalue = function(value, callback) { + var maxsize = meta.documentsize - DATAOFFSET; + var data = U.createBuffer(value); + if (data.length > maxsize) { + Zlib.deflate(data, ZLIBOPTIONS, (err, data) => callback((!data || data.length > maxsize) ? EMPTYBUFFER : data)); + } else + callback(data); + }; + + var process = function() { + + pageindex++; + + // ==== DB:PAGE (20b) + // type (1b) = from: 0 + // index (1b) = from: 1 + // documents (2b) = from: 2 + // freeslots (1b) = from: 4 + // parentindex (4b) = from: 5 + + // ==== DB:DOCUMENT (SIZE) + // type (1b) = from: 0 + // index (1b) = from: 1 + // state (1b) = from: 2 + // pageindex (4b) = from: 3 + // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) + // parentindex (4b) = from: 11 + // size/count (2b) = from: 15 + // data = from: 17 + + var buf = U.createBufferSize(size); + + Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { + + if (!size) { + finish(); + return; + } + + var newbuf = U.createBufferSize(newsize); + + // Copies page info + newbuf.fill(buf, 0, self.header.pagesize); + buf = buf.slice(self.header.pagesize); + + var index = self.header.pagesize; + var documents = 0; + + (self.header.pagelimit).async(function(i, next) { + + // Unexpected problem + if (!buf.length) { + next(); + return; + } + + var docbuf = buf.slice(0, self.header.documentsize); + var typeid = docbuf[0]; + var indexid = docbuf[1]; + + if (docbuf[2] !== STATE_REMOVED) { + totaldocuments++; + documents++; + } + + if (docbuf[2] !== STATE_REMOVED && meta.classes && typeid === TYPE_CLASS && meta.classes[indexid]) { + readvalue(docbuf, function(value) { + + // parseData + // stringifyData + value = stringifyData(meta.classes[indexid], parseData(self.$classes[indexid].schema, value.split('|'))); + + writevalue(value, function(value) { + + if (value === EMPTYBUFFER) { + // BIG PROBLEM + docbuf.writeUInt16LE(0, 15); + docbuf.writeUInt8(STATE_REMOVED, 2); + documents--; + } else { + docbuf.writeUInt16LE(value.length, 15); + docbuf.fill(value, DATAOFFSET, DATAOFFSET + value.length); + } + + newbuf.fill(docbuf, index, index + self.header.documentsize); + index += meta.documentsize; + buf = buf.slice(self.header.documentsize); + next(); + }); + + }); + } else { + newbuf.fill(docbuf, index, index + self.header.documentsize); + index += meta.documentsize; + buf = buf.slice(self.header.documentsize); + next(); + } + + }, function() { + + // Update count of documents + if (newbuf.readUInt16LE(2) !== documents) + newbuf.writeUInt16LE(documents, 2); + + Fs.write(fd, newbuf, 0, newbuf.length, newoffset, function() { + offset += size; + newoffset += newsize; + setImmediate(process); + }); + }); + + }); + }; + + process(); + }); + }); + return self; +}; + + function $update(doc, value) { return value; } @@ -1854,6 +2126,19 @@ GP.find2 = function(cls) { return builder; }; +GP.scalar = function(cls, type, field) { + var self = this; + var builder = new DatabaseBuilder(self); + builder.scalar(type, field); + self.pending.find.push({ name: cls, builder: builder }); + setImmediate(self.cb_next, NEXT_FIND); + return builder; +}; + +GP.count = function(cls) { + return this.scalar(cls, 'count', 'ID'); +}; + function GraphDBFilter(db) { var t = this; t.db = db; From 16aa714fcf71794fe537cfd00e0052f417268264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 15:18:17 +0200 Subject: [PATCH 0492/1669] Fixed return value in `graphdb.disconnect()`. --- graphdb.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphdb.js b/graphdb.js index 7ac92897a..097d9ec6b 100644 --- a/graphdb.js +++ b/graphdb.js @@ -527,7 +527,7 @@ function remRelation(self, relation, indexA, indexB, callback) { remRelationLink(self, relA, indexB, function(err, countA) { remRelationLink(self, relB, indexA, function(err, countB) { remRelationLink(self, relation.documentindex, indexA, function(err, countC) { - callback(null, (countA + countB + countC) === 3); + callback(null, (countA + countB + countC) > 1); }); }); }); From d8b95f544a4ab5cb028423dac58667c39e63084c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 19:07:07 +0200 Subject: [PATCH 0493/1669] Improved code. --- graphdb.js | 20 ++++++++++++++------ index.js | 4 +++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/graphdb.js b/graphdb.js index 097d9ec6b..56253925d 100644 --- a/graphdb.js +++ b/graphdb.js @@ -69,6 +69,8 @@ const TYPE_CLASS = 1; const TYPE_RELATION = 2; const TYPE_RELATION_DOCUMENT = 3; +var IMPORTATOPERATIONS = 0; + function GraphDB(name) { F.path.verify('databases'); @@ -423,6 +425,8 @@ function addRelation(self, relation, indexA, indexB, callback) { if (F.isKilled) return; + IMPORTATOPERATIONS++; + if (relA) next(); else { @@ -482,6 +486,7 @@ function addRelation(self, relation, indexA, indexB, callback) { }); tasks.async(function() { + IMPORTATOPERATIONS--; // console.log('REL ====', relA, relB); callback(null, true); }); @@ -524,9 +529,11 @@ function remRelation(self, relation, indexA, indexB, callback) { if (F.isKilled) return; + IMPORTATOPERATIONS++; remRelationLink(self, relA, indexB, function(err, countA) { remRelationLink(self, relB, indexA, function(err, countB) { remRelationLink(self, relation.documentindex, indexA, function(err, countC) { + IMPORTATOPERATIONS--; callback(null, (countA + countB + countC) > 1); }); }); @@ -872,14 +879,9 @@ function pushRelationDocument(self, index, relation, documentindex, initializato meta.relationindex = index; meta.size = 0; - // addPage(self, relation.private ? TYPE_RELATION_DOCUMENT : TYPE_RELATION, relation.index, relation.pageindex, function(err, pageindex) { addNode(self, meta, function(err, docindex, pageindex) { - - // index = relation.documentindex; relation.pageindex = pageindex; - // relation.documentindex = getDocumentIndex(self, pageindex); relation.documentindex = docindex; - updDocumentRelation(self, relation.documentindex, index, function() { updDocumentParent(self, index, relation.documentindex, function() { pushRelationDocument(self, relation.documentindex, relation, documentindex, initializator, callback, between); @@ -898,7 +900,7 @@ function pushRelationDocument(self, index, relation, documentindex, initializato buf.writeUInt16LE(count + 1, 15); if (buf[2] === STATE_REMOVED) { - // We must update page document counts + // We must update counts of documents in the page meta var pageindex = Math.ceil(index / self.header.pagelimit); updPageMeta(self, pageindex, function(err, buf) { @@ -963,8 +965,10 @@ function remDocument(self) { return; self.states.remove = true; var doc = self.pending.remove.shift(); + IMPORTATOPERATIONS++; remRelationAll(self, doc.id, doc.id, function() { remDocumentAll(self, doc.id, function(err, count) { + IMPORTATOPERATIONS--; self.states.remove = false; doc.callback && doc.callback(err, count); setImmediate(self.cb_next, NEXT_REMOVE); @@ -2570,4 +2574,8 @@ function regtescape(c) { exports.load = function(name, size) { return new GraphDB(name, size); +}; + +exports.getImportantOperations = function() { + return IMPORTATOPERATIONS; }; \ No newline at end of file diff --git a/index.js b/index.js index aa680d84e..f3f04cf48 100755 --- a/index.js +++ b/index.js @@ -1336,6 +1336,7 @@ F.stop = F.kill = function(signal) { } framework_nosql.kill(signal); + F.emit('exit', signal); if (!F.isWorker && process.send) @@ -1348,7 +1349,8 @@ F.stop = F.kill = function(signal) { F.server.close(); } - setTimeout(() => process.exit(signal), global.TEST ? 2000 : 100); + var extenddelay = require('./graphdb').getImportantOperations() > 0; + setTimeout(() => process.exit(signal), global.TEST || extenddelay ? 2000 : 300); return F; }; From 67ebca0ebd44f2c84e819a73b209a243dcf60d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 20:28:33 +0200 Subject: [PATCH 0494/1669] Improved code. --- graphdb.js | 1 + nosql.js | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/graphdb.js b/graphdb.js index 56253925d..e71e64dda 100644 --- a/graphdb.js +++ b/graphdb.js @@ -78,6 +78,7 @@ function GraphDB(name) { var self = this; self.name = name; self.filename = F.path.databases(name + '.gdb'); + self.filenameBackup = self.filename.replace(/\.gdb$/, '.gdp-backup'); self.ready = false; self.$classes = {}; diff --git a/nosql.js b/nosql.js index ec03c396a..79e00c9a3 100755 --- a/nosql.js +++ b/nosql.js @@ -155,7 +155,6 @@ exports.worker = function() { FORKCALLBACKS = {}; FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory }); - FORK.send({ TYPE: 'init', directory: F.path.root() }); FORK.on('message', function(msg) { switch (msg.TYPE) { From a5b78937290897117c60f36fb5367919f113e4d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 20:40:53 +0200 Subject: [PATCH 0495/1669] Improved code. --- graphdb.js | 1 - 1 file changed, 1 deletion(-) diff --git a/graphdb.js b/graphdb.js index e71e64dda..fe5e30535 100644 --- a/graphdb.js +++ b/graphdb.js @@ -24,7 +24,6 @@ * @version 1.0.0 */ - const Fs = require('fs'); const Zlib = require('zlib'); From 697e872a325296bbc92225b15cdee007eae4739a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 21:43:44 +0200 Subject: [PATCH 0496/1669] Added `U.encryptID()` and `U.decryptID()`. --- changes.txt | 6 +++--- index.js | 7 +++++-- utils.js | 45 +++++++++++++++++++++++++++------------------ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/changes.txt b/changes.txt index 509275ffe..d04566e2e 100755 --- a/changes.txt +++ b/changes.txt @@ -95,9 +95,9 @@ - added: `SchemaEntity.clear()` for removing all current definition - added: new view engine markup `@{#}` for simulating of root URL - added: `String.ROOT()` for replacing `@{#}` markup in strings -- added: `Number.encrypt(key)` for encrypting number values -- added: `String.decryptnumber(key)` for decrypting of number values -- added: `F.config['secret-numbers']` as a hidden secret for encrypting/decrypting values +- added: `U.decryptID(value, key)` for encrypting number/string values +- added: `U.encryptID(value, key)` for decrypting of number/string values +- added: `F.config['secret-uid']` as a hidden secret for encrypting/decrypting values - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index f3f04cf48..86b23833c 100755 --- a/index.js +++ b/index.js @@ -647,7 +647,7 @@ function Framework() { version: '1.0.0', author: '', secret: this.syshash, - 'secret-numbers': 'numbers', + 'secret-uid': this.syshash.substring(10), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -8885,7 +8885,7 @@ F.$configure_configs = function(arr, rewrite) { switch (name) { case 'secret': - case 'secret-numbers': + case 'secret-uid': obj[name] = value; break; case 'default-request-length': @@ -9033,6 +9033,9 @@ F.$configure_configs = function(arr, rewrite) { U.extend(F.config, obj, rewrite); + if (!F.config['secret-uid']) + F.config['secret-uid'] = (F.config.name).crc32(true).toString(); + var tmp = F.config['mail-smtp-options']; if (typeof(tmp) === 'string' && tmp) { tmp = new Function('return ' + tmp)(); diff --git a/utils.js b/utils.js index f6ac86ad9..fb65075fb 100755 --- a/utils.js +++ b/utils.js @@ -3879,17 +3879,33 @@ String.prototype.decrypt = function(key, secret) { return counter !== (val.length + key.length) ? null : val; }; -String.prototype.decryptnumber = function(salt) { - var val = this.decrypt(salt, F.config['secret-numbers']); - var num = 0; - if (val) { - num = +val; - if (isNaN(num)) - return 0; - for (var i = 0; i < salt.length; i++) - num -= salt.charCodeAt(i); - } - return num; +exports.encryptID = function(val, key) { + + var num = typeof(val) === 'number'; + var sum = 0; + + if (!key) + key = F.config.secret; + + val = val.toString(); + + for (var i = 0; i < val.length; i++) + sum += val.charCodeAt(i); + + for (var i = 0; i < key.length; i++) + sum += key.charCodeAt(i); + + return (num ? 'n' : 'x') + (F.config['secret-uid'] + val + sum + key).crc32(true).toString(16) + 'x' + val; +}; + +exports.decryptID = function(val, key) { + var num = val[0] === 'n'; + var raw = val.substring(val.indexOf('x', 1) + 1); + + if (num) + raw = +raw; + + return exports.encryptID(raw, key) === val ? raw : null; }; String.prototype.base64ToFile = function(filename, callback) { @@ -4074,13 +4090,6 @@ String.prototype.removeTags = function() { return this.replace(regexpTags, ''); }; -Number.prototype.encrypt = function(salt) { - var num = this; - for (var i = 0; i < salt.length; i++) - num += salt.charCodeAt(i); - return num.toString().encrypt(salt, false, F.config['secret-numbers']); -}; - Number.prototype.floor = function(decimals) { return Math.floor(this * Math.pow(10, decimals)) / Math.pow(10, decimals); }; From d4a0f32c28385a46424db3c70b5746d079adcb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 15 Jul 2018 23:49:19 +0200 Subject: [PATCH 0497/1669] Rename methods. --- changes.txt | 8 ++------ utils.js | 6 +++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index d04566e2e..3cf7b22f2 100755 --- a/changes.txt +++ b/changes.txt @@ -5,7 +5,6 @@ - added: (IMPORTANT) Total.js components support nested public files encoded in base64 - added: (IMPORTANT) NoSQL worker - added: (IMPORTANT) NoSQL embedded storage for smaller big data / IoT -- added: (IMPORTANT) Session helper object - added: `debugging` supports live reloading - added: new schema operations: `schema.setInsert()` and `schema.setUpdate()` - added: `RESTBuilder.patch([data])` @@ -55,9 +54,6 @@ - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: config `security.txt` for auto-generating security.txt content (more in docs) - added: config `default-proxy` for default web proxy server -- added: config `default-crypto` for default crypto algorithm -- added: config `default-crypto-key` for default length of key crypto algorithm -- added: `NOSQLSTORAGE(name)` alias for `NOSQL(name).storage` - added: `GUID()` a global alias for `U.GUID()` - added: `VIEW()` a global alias for `F.view()` - added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue @@ -95,8 +91,8 @@ - added: `SchemaEntity.clear()` for removing all current definition - added: new view engine markup `@{#}` for simulating of root URL - added: `String.ROOT()` for replacing `@{#}` markup in strings -- added: `U.decryptID(value, key)` for encrypting number/string values -- added: `U.encryptID(value, key)` for decrypting of number/string values +- added: `U.decryptUID(value, key)` for encrypting number/string values +- added: `U.encryptUID(value, key)` for decrypting of number/string values - added: `F.config['secret-uid']` as a hidden secret for encrypting/decrypting values - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory diff --git a/utils.js b/utils.js index fb65075fb..7fc55221c 100755 --- a/utils.js +++ b/utils.js @@ -3879,7 +3879,7 @@ String.prototype.decrypt = function(key, secret) { return counter !== (val.length + key.length) ? null : val; }; -exports.encryptID = function(val, key) { +exports.encryptUID = function(val, key) { var num = typeof(val) === 'number'; var sum = 0; @@ -3898,14 +3898,14 @@ exports.encryptID = function(val, key) { return (num ? 'n' : 'x') + (F.config['secret-uid'] + val + sum + key).crc32(true).toString(16) + 'x' + val; }; -exports.decryptID = function(val, key) { +exports.decryptUID = function(val, key) { var num = val[0] === 'n'; var raw = val.substring(val.indexOf('x', 1) + 1); if (num) raw = +raw; - return exports.encryptID(raw, key) === val ? raw : null; + return exports.encryptUID(raw, key) === val ? raw : null; }; String.prototype.base64ToFile = function(filename, callback) { From 75b0abc5cf4330a61e12d791ee5aacc138138b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 00:21:17 +0200 Subject: [PATCH 0498/1669] Extend buffer for column declaration. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 79e00c9a3..2ed3c9975 100755 --- a/nosql.js +++ b/nosql.js @@ -530,7 +530,7 @@ function Table(name, filename) { var schema = F.config['table.' + name]; - Fs.createReadStream(t.filename, { end: 1000 }).once('data', function(chunk) { + Fs.createReadStream(t.filename, { end: 1200 }).once('data', function(chunk) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); From 8a836a26f64491036fe51e0aa72903da9e251cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 00:21:24 +0200 Subject: [PATCH 0499/1669] Added new changes. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 3cf7b22f2..611f5a733 100755 --- a/changes.txt +++ b/changes.txt @@ -116,8 +116,8 @@ - updated: `CORS()` tries to join multiple same preferences to one - updated: `U.keywords()` for Chinese/Japan characters - updated: `@{import()}` by adding `manifest` value linked to `/manifest.json` -- updated: improved crypto algorithm - updated: `F.use()` supports `function` instead of `middleware` name +- updated: improved crypto algorithm - fixed: mail attachments - fixed: mail `message.manually()` From b3643fadd683f279bfe567bd158e2278c198941a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 00:53:50 +0200 Subject: [PATCH 0500/1669] Fixed `controller` in `schemas`. --- builders.js | 37 ++++++++++++++++++++++--------------- package.json | 2 +- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/builders.js b/builders.js index 83fdfbc6c..0ec09902d 100755 --- a/builders.js +++ b/builders.js @@ -1167,11 +1167,11 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, if (!isGenerator(self, $type, self[TYPE])) { if (self[TYPE].$newversion) self[TYPE](new SchemaOptions(builder, model, options, function(res) { - self.$process(arguments, model, $type, undefined, builder, res, callback); + self.$process(arguments, model, $type, undefined, builder, res, callback, controller); }, controller)); else self[TYPE](builder, model, options, function(res) { - self.$process(arguments, model, $type, undefined, builder, res, callback); + self.$process(arguments, model, $type, undefined, builder, res, callback, controller); }, controller, skip !== true); return self; } @@ -1248,11 +1248,11 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio if (!isGenerator(self, $type, self.onGet)) { if (self.onGet.$newversion) self.onGet(new SchemaOptions(builder, output, options, function(res) { - self.$process(arguments, output, $type, undefined, builder, res, callback); + self.$process(arguments, output, $type, undefined, builder, res, callback, controller); }, controller)); else self.onGet(builder, output, options, function(res) { - self.$process(arguments, output, $type, undefined, builder, res, callback); + self.$process(arguments, output, $type, undefined, builder, res, callback, controller); }, controller); return self; } @@ -1321,11 +1321,11 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { if (!isGenerator(self, $type, self.onRemove)) { if (self.onRemove.$newversion) self.onRemove(new SchemaOptions(builder, undefined, options, function(res) { - self.$process(arguments, undefined, $type, undefined, builder, res, callback); + self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller)); else self.onRemove(builder, options, function(res) { - self.$process(arguments, undefined, $type, undefined, builder, res, callback); + self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller); return self; } @@ -1392,11 +1392,11 @@ SchemaBuilderEntity.prototype.query = function(options, callback, controller) { if (!isGenerator(self, $type, self.onQuery)) { if (self.onQuery.$newversion) self.onQuery(new SchemaOptions(builder, undefined, options, function(res) { - self.$process(arguments, undefined, $type, undefined, builder, res, callback); + self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller)); else self.onQuery(builder, options, function(res) { - self.$process(arguments, undefined, $type, undefined, builder, res, callback); + self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller); return self; } @@ -2041,7 +2041,7 @@ SchemaBuilderEntity.prototype.transform2 = function(name, options, callback, con return this.transform(name, this.create(), options, callback, true, controller); }; -SchemaBuilderEntity.prototype.$process = function(arg, model, type, name, builder, response, callback) { +SchemaBuilderEntity.prototype.$process = function(arg, model, type, name, builder, response, callback, controller) { var self = this; @@ -2053,6 +2053,10 @@ SchemaBuilderEntity.prototype.$process = function(arg, model, type, name, builde var has = builder.hasError(); has && self.onError && self.onError(builder, model, type, name); + + if (controller && response instanceof SchemaInstance && !response.$$controller) + response.$$controller = controller; + callback(has ? builder : null, response === undefined ? model : response, model); return self; }; @@ -2278,11 +2282,11 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca self.resourcePrefix && builder.setPrefix(self.resourcePrefix); if (item.$newversion) item.call(self, new SchemaOptions(builder, model, options, function(res) { - self.$process(arguments, model, type, name, builder, res, callback); + self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller)); else item.call(self, builder, model, options, function(res) { - self.$process(arguments, model, type, name, builder, res, callback); + self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller, skip !== true); return self; } @@ -2294,6 +2298,9 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca return; } + if (controller && model instanceof SchemaInstance && !model.$$controller) + model.$$controller = controller; + var builder = new ErrorBuilder(); self.resourceName && builder.setResource(self.resourceName); @@ -2302,11 +2309,11 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca if (!isGenerator(self, type + '.' + name, item)) { if (item.$newversion) item.call(self, new SchemaOptions(builder, model, options, function(res) { - self.$process(arguments, model, type, name, builder, res, callback); + self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller)); else item.call(self, builder, model, options, function(res) { - self.$process(arguments, model, type, name, builder, res, callback); + self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller); return; } @@ -2413,11 +2420,11 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac if (!isGenerator(self, 'operation.' + name, operation)) { if (operation.$newversion) { operation.call(self, new SchemaOptions(builder, model, options, function(res) { - self.$process(arguments, model, $type, name, builder, res, callback); + self.$process(arguments, model, $type, name, builder, res, callback, controller); }, controller)); } else operation.call(self, builder, model, options, function(res) { - self.$process(arguments, model, $type, name, builder, res, callback); + self.$process(arguments, model, $type, name, builder, res, callback, controller); }, controller, skip !== true); return self; } diff --git a/package.json b/package.json index bd499271a..fe0868e20 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-86", + "version": "3.0.0-87", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From a88370fe3bb24b89c640a20c7a6b4ad930e8744a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 08:56:29 +0200 Subject: [PATCH 0501/1669] Added missing semicolon. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 7fc55221c..d5178e8dd 100755 --- a/utils.js +++ b/utils.js @@ -1262,7 +1262,7 @@ function download_response(res, uri, options) { // TLS? options.uri = tmp; download_call(options, request_call); - return + return; } if (!options.resolve) { From 13621bf518083474b1707ad1adfec638a746151c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 08:56:48 +0200 Subject: [PATCH 0502/1669] Removed unused argument. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 2ed3c9975..71a383933 100755 --- a/nosql.js +++ b/nosql.js @@ -615,7 +615,7 @@ function Database(name, filename, readonly) { const TP = Table.prototype; const DP = Database.prototype; -TP.view = DP.view = function(name) { +TP.view = DP.view = function() { throw new Error('NoSQL Views are not supported in this version.'); }; From 513a3694fd55cfeb78dbde02246ab5a095778214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 08:57:24 +0200 Subject: [PATCH 0503/1669] Removed duplicated `params` property. --- builders.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/builders.js b/builders.js index 0ec09902d..81779cc33 100755 --- a/builders.js +++ b/builders.js @@ -88,10 +88,6 @@ SchemaOptions.prototype = { get query() { return this.controller ? this.controller.query : null; - }, - - get params() { - return this.controller ? this.controller.params : null; } }; From 7a5f348c7eba0a98b4b085b026c08c93186b0597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 08:57:33 +0200 Subject: [PATCH 0504/1669] Added `mail.filestorage()`. --- changes.txt | 1 + mail.js | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 611f5a733..b695365fa 100755 --- a/changes.txt +++ b/changes.txt @@ -15,6 +15,7 @@ - added: `MailMessage.low()` sets `low` priority of the email messsage - added: `MailMessage.confidential()` sets `Sensitivity` header with `confidential` value - added: `MailMessage.attachmentnosql(db, id, [name])` sends a file from NoSQL embedded database +- added: `MailMessage.filestorage(storage_name, id, [name])` sends a file from FileStorage - added: `SchemaBuilderEntity.$stop()` stops the async list - added: `SchemaOptions.stop()` alias to `$.model.$stop()` - added: `SchemaOptions.next()` alias to `$.model.$next()` diff --git a/mail.js b/mail.js index 01fb3253c..cfb8c5111 100755 --- a/mail.js +++ b/mail.js @@ -267,6 +267,21 @@ Message.prototype.attachment = function(filename, name) { return this; }; +Message.prototype.filestorage = function(storagename, id, name) { + + var extension; + var type; + + if (name) { + extension = framework_utils.getExtension(name); + type = framework_utils.getContentType(extension); + } + + !this.files && (this.files = []); + this.files.push({ storage: storagename, name: name, filename: id, type: type, extension: extension }); + return this; +}; + Message.prototype.attachmentnosql = function(db, id, name) { var extension; @@ -302,7 +317,7 @@ Message.prototype.manually = function() { * @param {String} contentId the Content-ID (e.g. 'AB435BH'), must be unique across the email * @returns {Message} */ -Message.prototype.attachmentInline = function(filename, name, contentId) { +Message.prototype.attachmentInline = Message.prototype.attachmentinline = function(filename, name, contentId) { !name && (name = framework_utils.getName(filename)); !this.files && (this.files = []); var extension = framework_utils.getExtension(name); @@ -383,7 +398,6 @@ Mailer.prototype.destroy = function(obj) { }; const ATTACHMENT_SO = { encoding: 'base64' }; -const ATTACHMENT_SO_NOSQL = { encoding: 'base64', start: 2000 }; Mailer.prototype.$writeattachment = function(obj) { @@ -397,10 +411,26 @@ Mailer.prototype.$writeattachment = function(obj) { var stream; - if (attachment.nosql) { + if (attachment.storage) { + FILESTORAGE(attachment.storage).binary.readbase64(attachment.filename, function(err, stream, meta) { + if (err) { + F.error(err, 'Mail.filestorage()', attachment.filename); + mailer.$writeattachment(obj); + } else { + + if (!attachment.name) { + attachment.name = meta.name; + attachment.type = meta.type; + attachment.extension = U.getExtension(meta.name); + } + + writeattachemnt_stream(attachment, obj, stream); + } + }); + } else if (attachment.nosql) { NOSQL(attachment.nosql).binary.readbase64(attachment.filename, function(err, stream, meta) { if (err) { - F.error(err, 'Mail.attachment()', attachment.filename); + F.error(err, 'Mail.attachmentnosql()', attachment.filename); mailer.$writeattachment(obj); } else { From 5bbaa66b43ebdf402466e891168023d69b5e0b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 09:01:41 +0200 Subject: [PATCH 0505/1669] Updated method name. --- changes.txt | 2 +- mail.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index b695365fa..74af8cf58 100755 --- a/changes.txt +++ b/changes.txt @@ -15,7 +15,7 @@ - added: `MailMessage.low()` sets `low` priority of the email messsage - added: `MailMessage.confidential()` sets `Sensitivity` header with `confidential` value - added: `MailMessage.attachmentnosql(db, id, [name])` sends a file from NoSQL embedded database -- added: `MailMessage.filestorage(storage_name, id, [name])` sends a file from FileStorage +- added: `MailMessage.attachmentfs(storage_name, id, [name])` sends a file from FileStorage - added: `SchemaBuilderEntity.$stop()` stops the async list - added: `SchemaOptions.stop()` alias to `$.model.$stop()` - added: `SchemaOptions.next()` alias to `$.model.$next()` diff --git a/mail.js b/mail.js index cfb8c5111..dd2b2aea2 100755 --- a/mail.js +++ b/mail.js @@ -267,7 +267,7 @@ Message.prototype.attachment = function(filename, name) { return this; }; -Message.prototype.filestorage = function(storagename, id, name) { +Message.prototype.attachmentfs = function(storagename, id, name) { var extension; var type; From a2e7dd315335054882ae36e361fed815636dddce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 16 Jul 2018 16:37:07 +0200 Subject: [PATCH 0506/1669] Fixed extracting bundles and packages in the cluster. --- changes.txt | 2 +- cluster.js | 28 ++++++++++++++---- index.js | 84 ++++++++++++++++++++++++++++++++++------------------- 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/changes.txt b/changes.txt index 74af8cf58..88cfa1e5b 100755 --- a/changes.txt +++ b/changes.txt @@ -73,7 +73,6 @@ - added: `versions` file supports `auto` value for generating auto-checksum of files - added: `F.load()` supports `test` - added: NoSQL binary supports `custom` small data attributes -- added: `@{root}` for getting sub-root path - added: CSS and JS supports a simple View Engine markup (config + resources + F.global) - added: `controller.split` alias to `controller.req.split` - added: nicer error response messages @@ -91,6 +90,7 @@ - added: `F.syshash` contains a simple MD5 hash with OS info - added: `SchemaEntity.clear()` for removing all current definition - added: new view engine markup `@{#}` for simulating of root URL +- added: new view engine command `@{root}` for getting sub-root path - added: `String.ROOT()` for replacing `@{#}` markup in strings - added: `U.decryptUID(value, key)` for encrypting number/string values - added: `U.encryptUID(value, key)` for decrypting of number/string values diff --git a/cluster.js b/cluster.js index 6696bea0f..2c1179e29 100644 --- a/cluster.js +++ b/cluster.js @@ -21,7 +21,7 @@ /** * @module FrameworkCluster - * @version 2.9.2 + * @version 3.0.0 */ const Cluster = require('cluster'); @@ -36,6 +36,7 @@ var CALLBACKS = {}; var OPTIONS = {}; var THREADS = 0; var MASTER = null; +var CONTINUE = false; exports.on = function(name, callback) { !MASTER && (MASTER = {}); @@ -192,15 +193,30 @@ function master(count, mode, options, callback) { THREADS = count; - for (var i = 0; i < count; i++) - exec(i); + var can = function(cb) { + if (CONTINUE) + cb(); + else + setTimeout(can, 1000, cb); + }; + + count.async(function(i, next) { + exec(Math.abs(i - THREADS)); + can(next); + }, function() { + callback && callback(FORKS); + }); process.title = 'total: cluster'; - callback && callback(FORKS); } function message(m) { + if (m === 'total:ready') { + CONTINUE = true; + return; + } + if (m.TYPE === 'master') { if (MASTER && MASTER[m.name]) { for (var i = 0, length = MASTER[m.name].length; i < length; i++) @@ -225,6 +241,7 @@ function exec(index) { var fork = Cluster.fork(); fork.$id = index.toString(); fork.on('message', message); + if (FORKS[index]) FORKS[index] = fork; else @@ -232,7 +249,7 @@ function exec(index) { (function(fork) { setTimeout(function() { OPTIONS.options.id = fork.$id; - fork.send({ TYPE: 'init', id: fork.$id, mode: OPTIONS.mode, options: OPTIONS.options, threads: OPTIONS.count, index: index }); + fork.send({ TYPE: 'init', bundling: !CONTINUE, id: fork.$id, mode: OPTIONS.mode, options: OPTIONS.options, threads: OPTIONS.count, index: index }); }, fork.$id * 500); })(fork); } @@ -249,6 +266,7 @@ function on_init(msg) { CLUSTER_REQ.id = msg.id; CLUSTER_RES.id = msg.id; THREADS = msg.threads; + msg.options.bundling = msg.bundling; F.http(msg.mode, msg.options); F.isCluster = true; F.removeListener(msg.TYPE, on_init); diff --git a/index.js b/index.js index 86b23833c..954d775dc 100755 --- a/index.js +++ b/index.js @@ -752,6 +752,7 @@ function Framework() { }; global.G = this.global = {}; + this.$bundling = true; this.resources = {}; this.connections = {}; this.functions = {}; @@ -3443,10 +3444,12 @@ F.$bundle = function(callback) { try { Fs.statSync(F.path.root(F.config['directory-bundles'])); - makebundle(); - return; + if (F.$bundling) { + makebundle(); + return; + } else + F.directory = HEADERS.workers.cwd = directory = F.path.root(F.config['directory-src']); } catch(e) {} - callback(); }; @@ -3555,10 +3558,13 @@ F.$load = function(types, targetdirectory, callback, packageName) { files.wait(function(filename, next) { - var stream = Fs.createReadStream(filename); - var writer = Fs.createWriteStream(Path.join(dir, filename.replace(item.filename, '').replace(/\.package$/i, ''))); - stream.pipe(writer); - writer.on('finish', next); + if (F.$bundling) { + var stream = Fs.createReadStream(filename); + var writer = Fs.createWriteStream(Path.join(dir, filename.replace(item.filename, '').replace(/\.package$/i, ''))); + stream.pipe(writer); + writer.on('finish', next); + } else + next(); }, function() { @@ -3962,8 +3968,10 @@ F.install = function(type, name, declaration, options, callback, internal, useRe content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); - content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); - content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + if (F.$bundling) { + content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + } if (content.js) F.components.js = true; @@ -3978,7 +3986,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (content.body) { F.components.views[name] = '.' + F.path.temp('component_' + name); - Fs.writeFile(F.components.views[name].substring(1) + '.html', U.minifyHTML(content.body), NOOP); + F.$bundling && Fs.writeFile(F.components.views[name].substring(1) + '.html', U.minifyHTML(content.body), NOOP); } else delete F.components.views[name]; @@ -4066,8 +4074,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe var dir = F.config['directory-temp'][0] === '~' ? Path.join(F.config['directory-temp'].substring(1), id + '.package') : Path.join(F.path.root(), F.config['directory-temp'], id + '.package'); F.routes.packages[id] = dir; - F.restore(declaration, dir, function() { + var restorecb = function() { var filename = Path.join(dir, 'index.js'); if (!existsSync(filename)) { next && next(); @@ -4088,7 +4096,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe callback && callback(err, name); }, internal, useRequired, true, undefined); next && next(); - }); + }; + + if (F.$bundling) + F.restore(declaration, dir, restorecb); + else + restorecb(); return F; } @@ -4271,7 +4284,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp = F.path.temp('isomorphic_' + name + '.min.js'); F.map(framework_internal.preparePath(obj.url), tmp); F.isomorphic[name] = obj; - Fs.writeFileSync(tmp, prepare_isomorphic(name, framework_internal.compile_javascript(content, '#' + name))); + + F.$bundling && Fs.writeFileSync(tmp, prepare_isomorphic(name, framework_internal.compile_javascript(content, '#' + name))); F.consoledebug('install', type + '#' + name); next && next(); @@ -6742,7 +6756,7 @@ global.LOAD = F.load = function(debug, types, pwd) { } }); }); - }); + }, !types || types.indexOf('bundles') !== -1); return F; }; @@ -6779,10 +6793,12 @@ F.initialize = function(http, debug, options, restart) { F.config.debug = debug; F.isDebug = debug; + if (options.bundling != null) + F.$bundling = options.bundling == true; + global.DEBUG = debug; global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; - F.$bundle(function() { F.$configure_configs(); @@ -6924,9 +6940,15 @@ F.http = function(mode, options, middleware) { options == null && (options = {}); !options.port && (options.port = +process.argv[2]); + if (options.port && isNaN(options.port)) + options.port = 0; + if (typeof(middleware) === 'function') options.middleware = middleware; + if (options.bundling != null) + F.$bundling = options.bundling; + var http = require('http'); extend_request(http.IncomingMessage.prototype); extend_response(http.ServerResponse.prototype); @@ -8012,30 +8034,32 @@ F.test = function() { * @return {Framework} */ F.clear = function(callback, isInit) { + var dir = F.path.temp(); var plus = F.id ? 'i-' + F.id + '_' : ''; if (isInit) { if (!F.config['allow-clear-temp']) { - // clears only JS and CSS files - U.ls(dir, function(files) { - F.unlink(files, function() { - callback && callback(); + if (F.$bundling) { + // clears only JS and CSS files + U.ls(dir, function(files) { + F.unlink(files, function() { + callback && callback(); + }); + }, function(filename, folder) { + if (folder || (plus && !filename.substring(dir.length).startsWith(plus))) + return false; + if (filename.indexOf('.package') !== -1) + return true; + var ext = U.getExtension(filename); + return ext === 'js' || ext === 'css' || ext === 'tmp' || ext === 'upload' || ext === 'html' || ext === 'htm'; }); - }, function(filename, folder) { - if (folder || (plus && !filename.substring(dir.length).startsWith(plus))) - return false; - if (filename.indexOf('.package') !== -1) - return true; - var ext = U.getExtension(filename); - return ext === 'js' || ext === 'css' || ext === 'tmp' || ext === 'upload' || ext === 'html' || ext === 'htm'; - }); - + } return F; } } - if (!existsSync(dir)) { + if (!existsSync(dir) || !F.$bundling) { callback && callback(); return F; } @@ -16844,7 +16868,7 @@ function parseComponent(body, filename) { var name = base64.substring(base64.lastIndexOf('name="', tmp), tmp); name = name.substring(6, name.length - 1); base64 = base64.substring(tmp + 1); - Fs.writeFile(U.join(p, name), base64, 'base64', NOOP); + F.$bundling && Fs.writeFile(U.join(p, name), base64, 'base64', NOOP); response.files[name] = 1; } From 28475df16fd896aedaaa72fefb19a47aff75ca01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Jul 2018 08:39:15 +0200 Subject: [PATCH 0507/1669] New changes. --- changes.txt | 1 + index.js | 14 +++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index 88cfa1e5b..fa70985c1 100755 --- a/changes.txt +++ b/changes.txt @@ -55,6 +55,7 @@ - added: config `nosql-logger` (default `true`) enables simple logs when re-indexing and cleaning - added: config `security.txt` for auto-generating security.txt content (more in docs) - added: config `default-proxy` for default web proxy server +- added: config `allow-cache-cluster` (default `true`) allow/disallow cache synchronization - added: `GUID()` a global alias for `U.GUID()` - added: `VIEW()` a global alias for `F.view()` - added: `SchemaBuilderEntity.$response([index])` returns a specific response from an operation in `async` queue diff --git a/index.js b/index.js index 954d775dc..2c043af1b 100755 --- a/index.js +++ b/index.js @@ -731,6 +731,7 @@ function Framework() { 'allow-performance': false, 'allow-custom-titles': false, 'allow-cache-snapshot': false, + 'allow-cache-cluster': true, 'allow-debug': false, 'allow-head': false, 'allow-filter-errors': true, @@ -9850,7 +9851,7 @@ FrameworkCache.prototype.stop = function() { FrameworkCache.prototype.clear = function(sync) { this.items = {}; - F.isCluster && sync !== false && process.send(CLUSTER_CACHE_CLEAR); + F.isCluster && sync !== false && F.config['allow-cache-cluster'] && process.send(CLUSTER_CACHE_CLEAR); this.savePersist(); return this; }; @@ -9889,7 +9890,7 @@ FrameworkCache.prototype.set2 = function(name, value, expire, sync) { FrameworkCache.prototype.set = FrameworkCache.prototype.add = function(name, value, expire, sync, persist) { var type = typeof(expire); - if (F.isCluster && sync !== false) { + if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { CLUSTER_CACHE_SET.key = name; CLUSTER_CACHE_SET.value = value; CLUSTER_CACHE_SET.expire = expire; @@ -9964,7 +9965,7 @@ FrameworkCache.prototype.remove = function(name, sync) { this.items[name] = undefined; } - if (F.isCluster && sync !== false) { + if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { CLUSTER_CACHE_REMOVE.key = name; process.send(CLUSTER_CACHE_REMOVE); } @@ -9990,7 +9991,7 @@ FrameworkCache.prototype.removeAll = function(search, sync) { count++; } - if (F.isCluster && sync !== false) { + if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { CLUSTER_CACHE_REMOVEALL.key = search; process.send(CLUSTER_CACHE_REMOVEALL); } @@ -16669,11 +16670,6 @@ process.on('message', function(msg, h) { msg.TYPE === 'cache-remove' && F.cache.remove(msg.key, false); msg.TYPE === 'cache-remove-all' && F.cache.removeAll(msg.key, false); msg.TYPE === 'cache-clear' && F.cache.clear(false); - msg.TYPE === 'nosql-lock' && F.databases[msg.name] && F.databases[msg.name].lock(); - msg.TYPE === 'nosql-unlock' && F.databases[msg.name] && F.databases[msg.name].unlock(); - msg.TYPE === 'nosql-meta' && F.databases[msg.name] && F.databases[msg.name].$meta(); - msg.TYPE === 'nosql-counter-lock' && F.databases[msg.name] && (F.databases[msg.name].counter.locked = true); - msg.TYPE === 'nosql-counter-unlock' && F.databases[msg.name] && (F.databases[msg.name].counter.locked = false); msg.TYPE === 'req' && F.cluster.req(msg); msg.TYPE === 'res' && msg.target === F.id && F.cluster.res(msg); msg.TYPE === 'emit' && F.$events[msg.name] && F.emit(msg.name, msg.data); From ec7852665692abbb31165d69a06e5b9963a444e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Jul 2018 08:40:34 +0200 Subject: [PATCH 0508/1669] Updated unit-test. --- test/test-utils.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-utils.js b/test/test-utils.js index 23ff8004d..592810d0c 100755 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -627,6 +627,7 @@ function other() { var b = U.atob(a); assert.ok(b === input, 'U.atob() / U.btoa()'); + assert.ok(U.decryptUID(U.encryptUID(100)) === 100, 'U.encryptUID() + U.decryptUID()'); } function Utils_Ls2_StringFilter() { From e0696b444dfe93cea49c9efc1936ab149037b6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Jul 2018 09:28:56 +0200 Subject: [PATCH 0509/1669] Updated executable file. --- bin/totaljs | 168 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 137 insertions(+), 31 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index cd8150b5c..f17802003 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -10,8 +10,27 @@ var $type = 1; var isDirectory = false; function display_help() { + console.log('--------------------------------------------------------'); + console.log('TEMPLATES'); + console.log('--------------------------------------------------------'); + console.log(''); + console.log('without arguments : creates emptyproject'); + console.log('flow : creates emptyproject-flow'); + console.log('dashboard : creates emptyproject-dashboard'); + console.log('flowboard : creates emptyproject-flowboard'); + console.log('spa : creates emptyproject-jcomponent'); + console.log('pwa : creates emptyproject-pwa'); + console.log('rest : creates emptyproject-restservice'); + console.log('cms : downloads Total.js CMS'); + console.log('eshop : downloads Total.js Eshop'); + console.log('superadmin : downloads Total.js SuperAdmin'); + console.log('openplatform : downloads Total.js OpenPlatform'); + console.log('helpdesk : downloads Total.js HelpDesk'); + console.log(''); + console.log('--------------------------------------------------------'); + console.log('TOOLS'); + console.log('--------------------------------------------------------'); console.log(''); - console.log('without arguments : creates empty-project'); console.log('-translate : creates a resource file with the localized text from views'); console.log('-translate "TEXT" : creates an identificator for the resource'); console.log('-translate filename : parses and creates a resource file from the text file'); @@ -21,7 +40,6 @@ function display_help() { console.log('-merge source target : merges first resource into the second "-merge source target"'); console.log('-clean source : cleans a resource file "-clean source"'); console.log('-minify filename : minifies .js, .css or .html file into filename.min.[extension]'); - console.log('-v or -version : Total.js version'); console.log('-install : run "totaljs -install help" to see what can be installed'); console.log('8000 : starts a server'); console.log(''); @@ -433,40 +451,52 @@ function main() { if (cmd === '-i' || cmd === '-install') { - var libs = ['jc', 'jc.min', 'jcta', 'jcta.min', 'jctajr', 'jctajr.min', 'ta', 'jr', 'jr.jc']; + var libs = ['jc', 'jc.min', 'jcta', 'jcta.min', 'jctajr', 'jctajr.min', 'ta', 'jr', 'jr.jc', 'spa', 'spa.min']; var tmp = process.argv[i + 1] || ''; if (tmp === 'help') { - return console.log('Following libs can be installed: jc, jc.min, jcta.min, jctajr.min, ta, jr, jr.jc'); + return console.log('Following libs can be installed: jc, jc.min, jcta.min, jctajr.min, ta, jr, jr.jc, spa'); } - if (!tmp || libs.indexOf(tmp) < 0) + if (!tmp || libs.indexOf(tmp) < 0) return console.log('Unknown library: "' + tmp + '"'); - + console.log(''); console.log('Installing: ' + tmp); var url = ''; switch(tmp) { - case 'jc': - url = 'https://rawgit.com/totaljs/jComponent/master/jc.js'; + case 'jc': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/jc.js'; + break; + case 'jc.min': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/jc.min.js'; break; - case 'jc.min': - url = 'https://rawgit.com/totaljs/jComponent/master/jc.min.js'; + case 'jcta.min': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/jcta.min.js'; break; - case 'jcta.min': - url = 'https://rawgit.com/totaljs/jComponent/master/jcta.min.js'; + case 'jctajr.min': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/jctajr.min.js'; break; - case 'jctajr.min': - url = 'https://rawgit.com/totaljs/jComponent/master/jctajr.min.js'; + case 'spa': + case 'spa.min': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/spa.min.js'; break; - case 'ta': + case 'spa@14': + case 'spa.min@14': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/spa.min@14.js'; + break; + case 'spa@15': + case 'spa.min@15': + url = 'https://rawgit.com/totaljs/components/master/0dependencies/spa.min@15.js'; + break; + case 'ta': url = 'https://rawgit.com/totaljs/Tangular/master/Tangular.js'; break; - case 'jr': + case 'jr': url = 'https://rawgit.com/totaljs/jRouting/master/jrouting.js'; break; - case 'jr.jc': + case 'jr.jc': url = 'https://rawgit.com/totaljs/jRouting/master/jrouting.jcomponent.js'; break; } @@ -497,6 +527,66 @@ function main() { return; } + if (cmd === '-cms' || cmd === 'cms') { + git(dir, 'cms'); + return; + } + + if (cmd === '-eshop' || cmd === 'eshop') { + git(dir, 'eshop'); + return; + } + + if (cmd === '-superadmin' || cmd === 'superadmin') { + git(dir, 'superadmin'); + return; + } + + if (cmd === '-messenger' || cmd === 'messenger') { + git(dir, 'messenger'); + return; + } + + if (cmd === '-helpdesk' || cmd === 'helpdesk') { + git(dir, 'helpdesk'); + return; + } + + if (cmd === '-openplatform' || cmd === 'openplatform') { + git(dir, 'openplatform'); + return; + } + + if (cmd === '-flow' || cmd === 'flow') { + git(dir, 'emptyproject-flow'); + return; + } + + if (cmd === '-dashboard' || cmd === 'dashboard') { + git(dir, 'emptyproject-dashboard'); + return; + } + + if (cmd === '-flowboard' || cmd === 'flowboard') { + git(dir, 'emptyproject-flowboard'); + return; + } + + if (cmd === '-pwa' || cmd === 'pwa') { + git(dir, 'emptyproject-pwa'); + return; + } + + if (cmd === '-spa' || cmd === 'spa') { + git(dir, 'emptyproject-jcomponent'); + return; + } + + if (cmd === '-rest' || cmd === 'rest') { + git(dir, 'emptyproject-restservice'); + return; + } + if (cmd === '-diff') { diff(process.argv[i + 1] || '', process.argv[i + 2] || ''); return; @@ -669,7 +759,7 @@ function main() { texts[key] = command.command; if (!resource[key]) { - output.push(key + ';"' + command.command.replace(/\"/g, '""') + '";'); + output.push(key + ';"' + command.command.replace(/"/g, '""') + '";'); resource[key] = true; count++; } @@ -701,23 +791,39 @@ function main() { } } - console.log('Downloading "empty-project" from www.totaljs.com'); + git(dir, 'emptyproject'); +} + +function git(dir, type) { + + var done = function() { + console.log('Installed: {0}'.format(type)); + console.log(); + }; - var url = 'https://cdn.totaljs.com/empty-project.package'; + U.ls(dir, function(fol, fil) { - Utils.download(url, ['get'], function(err, response) { - var filename = path.join(dir, 'total.package'); - var stream = fs.createWriteStream(filename); - response.pipe(stream); - stream.on('finish', function() { - console.log('Unpacking file.'); - exec('totalpackage unpack total.package', function() { - fs.unlink(filename, NOOP); - console.log('Done.'); - console.log(''); + if (fol.length || fil.length) { + console.log('Directory "{0}"" is not empty.'.format(dir)); + console.log(); + return; + } + + F.path.mkdir(dir); + exec('git clone https://github.com/totaljs/{0}.git {1}'.format(type, dir), function(err) { + F.path.mkdir(path.join(dir, '/node_modules/')); + F.rmdir(path.join(dir, '.git'), function() { + F.unlink(path.join(dir, '.gitignore'), function() { + F.path.exists(path.join(dir, 'package.json'), function(e) { + if (e) + exec('npm install total.js --save', done); + else + exec('npm install', done); + }); + }); }); }); }); } -main(); +main(); \ No newline at end of file From f993c8e9fcdec0060758ff6781dfa636649a4d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Jul 2018 16:12:10 +0200 Subject: [PATCH 0510/1669] Added new features. --- changes.txt | 1 + index.js | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index fa70985c1..abee981c5 100755 --- a/changes.txt +++ b/changes.txt @@ -96,6 +96,7 @@ - added: `U.decryptUID(value, key)` for encrypting number/string values - added: `U.encryptUID(value, key)` for decrypting of number/string values - added: `F.config['secret-uid']` as a hidden secret for encrypting/decrypting values +- added: `F.dir(path)` for changing of root directory - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index 2c043af1b..9081e6815 100755 --- a/index.js +++ b/index.js @@ -306,7 +306,7 @@ global.$$$ = global.GETSCHEMA = (group, name, fn, timeout) => framework_builders global.CREATE = (group, name) => framework_builders.getschema(group, name).default(); global.SCRIPT = (body, value, callback, param) => F.script(body, value, callback, param); global.SINGLETON = (name, def) => SINGLETONS[name] || (SINGLETONS[name] = (new Function('return ' + (def || '{}')))()); -global.FUNCTION = (name) => F.functions[name]; +global.FUNCTION = (name) => F.functions[name] || NOOP; global.ROUTING = (name) => F.routing(name); global.SCHEDULE = (date, each, fn, param) => F.schedule(date, each, fn, param); global.FINISHED = framework_internal.onFinished; @@ -970,6 +970,11 @@ Framework.prototype = { var framework = new Framework(); global.framework = global.F = module.exports = framework; +F.dir = function(path) { + F.directory = path; + directory = path; +}; + F.prototypes = function(fn) { var proto = {}; proto.Chunker = framework_utils.Chunker.prototype; @@ -4609,7 +4614,7 @@ F.$restart = function() { global.G = F.global = {}; F.resources = {}; F.connections = {}; - F.functions = {}; + global.FUNCTIONS = F.functions = {}; F.themes = {}; F.uptodates = null; F.versions = null; From 5a22e826d913f5f6612632fddd894b991b81206e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 17 Jul 2018 21:50:57 +0200 Subject: [PATCH 0511/1669] Fixed stopping instance. --- cluster.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cluster.js b/cluster.js index 2c1179e29..46327dae9 100644 --- a/cluster.js +++ b/cluster.js @@ -159,7 +159,8 @@ exports.restart = function(index) { fork.removeAllListeners(); fork.disconnect(); exec(index); - } + } else + exec(index); } }; @@ -241,11 +242,13 @@ function exec(index) { var fork = Cluster.fork(); fork.$id = index.toString(); fork.on('message', message); + fork.on('exit', () => FORKS[index] = null); - if (FORKS[index]) - FORKS[index] = fork; - else + if (FORKS[index] === undefined) FORKS.push(fork); + else + FORKS[index] = fork; + (function(fork) { setTimeout(function() { OPTIONS.options.id = fork.$id; From b27ae68161251ac424490a9056c5818854648cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 09:32:35 +0200 Subject: [PATCH 0512/1669] Added `NOSQL().memory()` and `TABLE().memory()`. --- changes.txt | 1 + nosql.js | 92 +++++++++++++++++++++++++++++++++++++++++++++++++- nosqlstream.js | 16 +++++---- 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/changes.txt b/changes.txt index abee981c5..a06388456 100755 --- a/changes.txt +++ b/changes.txt @@ -97,6 +97,7 @@ - added: `U.encryptUID(value, key)` for decrypting of number/string values - added: `F.config['secret-uid']` as a hidden secret for encrypting/decrypting values - added: `F.dir(path)` for changing of root directory +- added: `NOSQL()/TABLE().memory(count, [size])` for memory consumption, more in docs - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/nosql.js b/nosql.js index 71a383933..7546223e1 100755 --- a/nosql.js +++ b/nosql.js @@ -615,6 +615,13 @@ function Database(name, filename, readonly) { const TP = Table.prototype; const DP = Database.prototype; +TP.memory = DP.memory = function(count, size) { + var self = this; + count && (self.buffercount = count); // def: 15 - count of stored documents in memory while reading/writing + size && (self.buffersize = size * 1024); // def: 32 - size of buffer in kB + return self; +}; + TP.view = DP.view = function() { throw new Error('NoSQL Views are not supported in this version.'); }; @@ -1401,6 +1408,12 @@ DP.$update = function() { var indexer = 0; var fs = new NoSQLStream(self.filename); + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); @@ -1680,6 +1693,12 @@ DP.$reader2 = function(filename, items, callback, reader) { var fs = new NoSQLStream(self.filename); + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); @@ -1863,6 +1882,12 @@ DP.$reader3 = function() { var fs = new NoSQLStream(self.filename); + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); @@ -2043,6 +2068,12 @@ DP.$streamer = function() { var count = 0; var fs = new NoSQLStream(self.filename); + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var docs = JSON.parse('[' + fs.docs + ']', jsonparser); for (var j = 0; j < docs.length; j++) { @@ -2289,6 +2320,12 @@ DP.$remove = function() { var indexer = 0; var backup = false; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + for (var i = 0; i < length; i++) { var fil = filter[i]; fil.compare = fil.builder.compile(); @@ -2394,6 +2431,12 @@ DP.$clean = function() { var fs = new NoSQLStream(self.filename); var writer = Fs.createWriteStream(self.filename + '-tmp'); + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.divider = NEWLINE; fs.ondocuments = function() { @@ -5360,6 +5403,12 @@ SP.scan = function(beg, end, mapreduce, callback, reverse) { stats.current = item.date; stats.index = index; + if (self.buffersize) + reader.buffersize = self.buffersize; + + if (self.buffercount) + reader.buffercount = self.buffercount; + reader.ondocuments = function() { var docs = JSON.parse('[' + reader.docs + ']', jsonparser); for (var j = 0; j < docs.length; j++) { @@ -5653,10 +5702,15 @@ TP.extend = function(schema, callback) { var count = 0; var fs = new NoSQLStream(self.filename); var data = {}; - var tmp = self.filename + '-tmp'; var writer = Fs.createWriteStream(tmp); + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + writer.write(meta, 'utf8'); writer.on('finish', function() { Fs.rename(tmp, self.filename, function() { @@ -5894,6 +5948,12 @@ TP.$reader = function() { fs.divider = '\n'; data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); @@ -6096,6 +6156,12 @@ TP.$reader3 = function() { fs.divider = '\n'; data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); @@ -6298,6 +6364,12 @@ TP.$update = function() { var data = {}; data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); @@ -6456,6 +6528,12 @@ TP.$remove = function() { fs.divider = '\n'; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + for (var i = 0; i < length; i++) { var fil = filter[i]; fil.compare = fil.builder.compile(true); @@ -6559,6 +6637,12 @@ TP.$clean = function() { fs.divider = NEWLINE; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { writer.write(fs.docs + NEWLINE); }; @@ -6642,6 +6726,12 @@ TP.$streamer = function() { data.keys = self.$keys; fs.divider = '\n'; + if (self.buffersize) + fs.buffersize = self.buffersize; + + if (self.buffercount) + fs.buffercount = self.buffercount; + fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); for (var a = count ? 0 : 1; a < lines.length; a++) { diff --git a/nosqlstream.js b/nosqlstream.js index 1762f0ddd..6665e067e 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -44,6 +44,8 @@ function NoSQLStream(filename) { this.buffer = null; this.divider = ','; this.remchar = '-'; + this.buffercount = BUFFERDOCS; + this.buffersize = BUFFERSIZE; // this.canceled = false; // this.docs = ''; // this.docscount = 0; @@ -95,7 +97,7 @@ NoSQLStream.prototype.readhelpers = function() { self.docscount++; self.indexer++; - if (self.docscount >= BUFFERDOCS) { + if (self.docscount >= self.buffercount) { if (self.ondocuments() === false) self.canceled = true; @@ -158,7 +160,7 @@ NoSQLStream.prototype.readhelpers = function() { self.docs += (self.docs ? self.divider : '') + tmp.trim(); self.docscount++; - if (self.docscount >= BUFFERDOCS) { + if (self.docscount >= self.buffercount) { if (self.ondocuments() === false) self.canceled = true; self.docs = ''; @@ -215,7 +217,7 @@ NoSQLStream.prototype.readhelpers = function() { self.docscount++; self.indexer++; - if (self.docscount >= BUFFERDOCS) { + if (self.docscount >= self.buffercount) { if (self.ondocuments() === false) self.canceled = true; @@ -325,7 +327,7 @@ NoSQLStream.prototype.writehelpers = function() { self.docs += (self.docs ? self.divider : '') + tmp; self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); self.docscount++; - if (self.docsbuffer.length >= BUFFERDOCS) { + if (self.docsbuffer.length >= self.buffercount) { if (self.ondocuments() === false) self.canceled = true; @@ -568,7 +570,7 @@ NoSQLStream.prototype.read = function() { self.close(); } else { - size = size < BUFFERSIZE ? size : BUFFERSIZE; + size = size < self.buffersize ? size : self.buffersize; var buffer = framework_utils.createBufferSize(size); Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readbuffer); } @@ -602,7 +604,7 @@ NoSQLStream.prototype.readreverse2 = function() { } else { var size = self.stats.size - self.bytesread; - size = size < BUFFERSIZE ? size : BUFFERSIZE; + size = size < self.buffersize ? size : self.buffersize; self.position -= size; var buffer = framework_utils.createBufferSize(size); Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readreversebuffer); @@ -631,7 +633,7 @@ NoSQLStream.prototype.readupdate = function() { self.flush(); } else { - size = size < BUFFERSIZE ? size : BUFFERSIZE; + size = size < self.buffersize ? size : self.buffersize; var buffer = framework_utils.createBufferSize(size); Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readwritebuffer); } From be53cec7b915c415dcdd9c900a88a6fba92fe738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 10:01:45 +0200 Subject: [PATCH 0513/1669] Updated `CONFIG()`. --- index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 9081e6815..8ddc6c5bd 100755 --- a/index.js +++ b/index.js @@ -294,7 +294,9 @@ global.NOBIN = global.NOSQLBINARY = (name) => F.nosql(name).binary; global.NOSQLSTORAGE = (name) => F.nosql(name).storage; global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); -global.CONFIG = (name) => F.config[name]; +global.CONFIG = function(name, val) { + return arguments.length === 1 ? F.config[name] : (F.config[name] = val); +}; global.UPTODATE = (type, url, options, interval, callback) => F.uptodate(type, url, options, interval, callback); global.INSTALL = (type, name, declaration, options, callback) => F.install(type, name, declaration, options, callback); global.UNINSTALL = (type, name, options) => F.uninstall(type, name, options); From d534b8c4a215dead52331d7e75dcd8e568781693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 10:10:45 +0200 Subject: [PATCH 0514/1669] Fixed `.memory()`. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 7546223e1..78ecbf745 100755 --- a/nosql.js +++ b/nosql.js @@ -617,7 +617,7 @@ const DP = Database.prototype; TP.memory = DP.memory = function(count, size) { var self = this; - count && (self.buffercount = count); // def: 15 - count of stored documents in memory while reading/writing + count && (self.buffercount = count + 1); // def: 15 - count of stored documents in memory while reading/writing size && (self.buffersize = size * 1024); // def: 32 - size of buffer in kB return self; }; From d469035ec502c811bd9d0da8e8c7e75560024e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 10:14:38 +0200 Subject: [PATCH 0515/1669] Added `database.one2()`. --- nosql.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nosql.js b/nosql.js index 78ecbf745..cb630840e 100755 --- a/nosql.js +++ b/nosql.js @@ -1129,6 +1129,15 @@ DP.one = function() { return builder; }; +DP.one2 = function() { + var self = this; + var builder = new DatabaseBuilder(self); + builder.first(); + self.pending_reader2.push({ builder: builder, count: 0 }); + setImmediate(next_operation, self, 11); + return builder; +}; + DP.top = function(max) { var self = this; var builder = new DatabaseBuilder(self); @@ -5787,6 +5796,16 @@ TP.one = function() { return builder; }; +TP.one2 = function() { + var self = this; + self.readonly && self.throwReadonly(); + var builder = new DatabaseBuilder(self); + builder.first(); + self.pending_reader2.push({ builder: builder, count: 0 }); + setImmediate(next_operation, self, 11); + return builder; +}; + TP.top = function(max) { var self = this; self.readonly && self.throwReadonly(); From d972e05e8f1caedbc8a0215228996c02c6b1aa9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 11:38:36 +0200 Subject: [PATCH 0516/1669] Improved code. --- index.js | 71 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/index.js b/index.js index 8ddc6c5bd..acb47ebab 100755 --- a/index.js +++ b/index.js @@ -335,13 +335,10 @@ global.UID = function(type) { return UIDGENERATOR.date + index.padLeft(4, '0') + UIDGENERATOR.instance + (index % 2 ? 1 : 0); }; -global.ROUTE = (a, b, c, d, e) => F.route(a, b, c, d, e); -global.WEBSOCKET = (a, b, c, d) => F.websocket(a, b, c, d); -global.FILE = (a, b, c) => F.file(a, b, c); -global.REDIRECT = (a, b, c, d) => F.redirect(a, b, c, d); global.AUTH = function(fn) { F.onAuthorize = fn; }; + global.WEBSOCKETCLIENT = function(callback) { var ws = require('./websocketclient').create(); callback && callback.call(ws, ws); @@ -1363,7 +1360,7 @@ F.stop = F.kill = function(signal) { return F; }; -F.redirect = function(host, newHost, withPath, permanent) { +global.REDIRECT = F.redirect = function(host, newHost, withPath, permanent) { var external = host.startsWith('http://') || host.startsWith('https'); if (external) { @@ -1811,7 +1808,8 @@ global.GROUP = F.group = function() { * @param {Number} timeout Response timeout. * @return {Framework} */ -F.web = F.route = function(url, funcExecute, flags, length, language) { + +global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, language) { var name; var tmp; @@ -2174,11 +2172,11 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (type === 'string') { viewname = funcExecute; - funcExecute = (function(name, sitemap, language) { + funcExecute = (function(name, sitemap, language, workflow) { var themeName = U.parseTheme(name); if (themeName) name = prepare_viewname(name); - return function() { + return function(id) { if (language && !this.language) this.language = language; sitemap && this.sitemap(sitemap.id, language); @@ -2188,15 +2186,24 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { this.themeName = themeName; if (!this.route.workflow) return this.view(name); + var self = this; - this.$exec(this.route.workflow, null, function(err, response) { - if (err) - self.content(err); + if (this.route.workflow instanceof Object) { + workflow.view = name; + if (workflow.id instanceof Array) + controller_json_workflow_multiple.call(self, id); else - self.view(name, response); - }); + controller_json_workflow.call(self, id); + } else { + this.$exec(this.route.workflow, null, function(err, response) { + if (err) + self.content(err); + else + self.view(name, response); + }); + } }; - })(viewname, sitemap, language); + })(viewname, sitemap, language, workflow); } else if (typeof(funcExecute) !== 'function') { viewname = (sitemap && sitemap.url !== '/' ? sitemap.id : workflow ? '' : url) || ''; @@ -2211,17 +2218,27 @@ F.web = F.route = function(url, funcExecute, flags, length, language) { if (!viewname || viewname === '/') viewname = 'index'; - funcExecute = (function(name, sitemap, language, action) { - return function() { - if (language && !this.language) - this.language = language; - sitemap && this.sitemap(sitemap.id, language); - name[0] === '~' && this.theme(''); - if (!this.route.workflow) - return this.view(name); + funcExecute = (function(name, sitemap, language) { + return function(id) { var self = this; - if (action) { + if (language && !self.language) + self.language = language; + + sitemap && self.sitemap(sitemap.id, language); + + if (name[0] === '~') + self.themeName = ''; + + if (!self.route.workflow) + return self.view(name); + + if (self.route.workflow instanceof Object) { + workflow.view = name; + if (workflow.id instanceof Array) + controller_json_workflow_multiple.call(self, id); + else + controller_json_workflow.call(self, id); } else { self.$exec(self.route.workflow, null, function(err, response) { if (err) @@ -2783,7 +2800,7 @@ function merge_middleware(a, b, first) { * @param {Object} options Optional, additional options for middleware. * @return {Framework} */ -F.websocket = function(url, funcInitialize, flags, length) { +global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { var tmp; @@ -3066,7 +3083,7 @@ F.initwebsocket = function() { * @param {String Array} middleware * @return {Framework} */ -F.file = function(fnValidation, fnExecute, flags) { +global.FILE = F.file = function(fnValidation, fnExecute, flags) { var a; @@ -16964,7 +16981,7 @@ function controller_json_workflow(id) { self[w.type](w.name, self.callback()); else { if (w.type) - self[w.type](self.callback()); + self[w.type](self.callback(w.view)); else self.throw500('Operation @' + w.id + ' not found.'); } @@ -17004,7 +17021,7 @@ function controller_json_workflow_multiple(id) { w.async = op; } - var async = self.$async(self.callback(), w.index); + var async = self.$async(self.callback(w.view), w.index); for (var i = 0; i < w.async.length; i++) { var a = w.async[i]; if (a.id) From 6c8d9f96eb29ddcaa22ec95c766207041cbe8d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 15:46:00 +0200 Subject: [PATCH 0517/1669] New improvements. --- changes.txt | 2 ++ index.js | 22 ++++++++++++- internal.js | 10 ++++++ nosql.js | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index a06388456..8f73a7814 100755 --- a/changes.txt +++ b/changes.txt @@ -98,6 +98,8 @@ - added: `F.config['secret-uid']` as a hidden secret for encrypting/decrypting values - added: `F.dir(path)` for changing of root directory - added: `NOSQL()/TABLE().memory(count, [size])` for memory consumption, more in docs +- added: `HttpFile.fs(storage_name, [custom], [callback])` saves a file into the FileStorage +- added: `res.filefs(storage_name, id, [download], [headers], [callback])` returns file from FileStorage - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index acb47ebab..58bfe423a 100755 --- a/index.js +++ b/index.js @@ -15510,6 +15510,26 @@ function extend_response(PROTO) { return this.$file(); }; + /** + * Responds with a file from FileStorage + * @param {String} name A name of FileStorage + * @param {String/Number} id + * @param {String} download Optional, a download name. + * @param {Object} headers Optional, additional headers. + * @param {Function} done Optional, callback. + * @return {Framework} + */ + PROTO.filefs = function(name, id, download, headers, callback) { + var self = this; + var options = {}; + options.id = id; + options.download = download; + options.headers = headers; + options.done = callback; + FILESTORAGE(name).res(self, options, $file_notmodified); + return self; + }; + /** * Responds with a stream * @param {String} contentType @@ -16315,7 +16335,7 @@ function $file_notmodified(res, name) { if (res.getHeader('Last-Modified')) delete headers['Last-Modified']; else - headers['Last-Modified'] = name[2]; + headers['Last-Modified'] = name instanceof Array ? name[2] : name; if (res.getHeader('Expires')) delete headers.Expires; diff --git a/internal.js b/internal.js index c4acb0bb7..1fdeb070a 100755 --- a/internal.js +++ b/internal.js @@ -899,6 +899,16 @@ HttpFile.prototype.image = function(im) { return framework_image.init(this.path, im, this.width, this.height); }; +HttpFile.prototype.fs = function(storagename, custom, callback) { + + if (typeof(custom) === 'function') { + callback = custom; + custom = null; + } + + return FILESTORAGE(storagename).insert(this.filename, Fs.createReadStream(this.path), custom, callback); +}; + // ********************************************************************************* // ================================================================================= // JS CSS + AUTO-VENDOR-PREFIXES diff --git a/nosql.js b/nosql.js index cb630840e..cb05377ec 100755 --- a/nosql.js +++ b/nosql.js @@ -4877,6 +4877,96 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { return cacheid; }; +Binary.prototype.meta = function(id, callback, count) { + + var self = this; + + if (count > 3) { + callback(new Error('File not found.')); + return self; + } + + var isnew = false; + + if (id > 0) + isnew = true; + else if (id[0] === 'B' || id[0] === 'b') { + id = +id.substring(id.length - DIRECTORYLENGTH); + isnew = true; + } else if (id.indexOf('#') === -1) + id = self.db.name + '#' + id; + + var filename; + + if (isnew) { + filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + self.ext); + } else + filename = framework_utils.join(self.directory, id + self.ext); + + var stream = Fs.createReadStream(filename, BINARYREADMETA); + stream.on('error', err => callback(err)); + stream.on('data', function(buffer) { + var json = buffer.toString('utf8').replace(REGCLEAN, ''); + if (json) { + callback(null, JSON.parse(json, jsonparser)); + CLEANUP(stream); + } else + setTimeout(readfileattempt, 100, self, id, callback, count || 1); + }); + + return self; +}; + +Binary.prototype.res = function(res, options, notmodified) { + + var self = this; + var isnew = false; + var id = options.id || ''; + + if (id > 0) + isnew = true; + else if (id[0] === 'B' || id[0] === 'b') { + id = +id.substring(id.length - DIRECTORYLENGTH); + isnew = true; + } else if (id.indexOf('#') === -1) + id = self.db.name + '#' + id; + + var filename; + + if (isnew) + filename = Path.join(self.$directory(id), id.toString().padLeft(DIRECTORYLENGTH, '0') + self.ext); + else + filename = framework_utils.join(self.directory, id + self.ext); + + var stream = Fs.createReadStream(filename, BINARYREADMETA); + stream.on('error', () => res.throw404()); + stream.on('data', function(buffer) { + var json = buffer.toString('utf8').replace(REGCLEAN, ''); + if (json) { + var obj = JSON.parse(json, jsonparser); + var utc = obj.date ? new Date(+obj.date.substring(0, 4), +obj.date.substring(4, 6), +obj.date.substring(6, 8)).toUTCString() : ''; + if (!options.download && res.req.headers['if-modified-since'] === utc) { + res.extention = U.getExtension(obj.name); + notmodified(res, utc); + } else { + res.options.type = obj.type; + res.options.stream = Fs.createReadStream(filename, BINARYREADDATA); + if (!options.download) { + if (!options.headers) + options.headers = {}; + options.headers['Last-Modified'] = utc; + } else + res.options.download = options.download; + res.options.headers = options.headers; + res.options.done = options.done; + res.options.compress = options.nocompress ? false : true; + res.$stream(); + } + } else + res.throw404(); + }); +}; + Binary.prototype.read = function(id, callback, count) { var self = this; From 8eaec674b9e41cc83ff6bfecb1142fa016a52f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 16:15:46 +0200 Subject: [PATCH 0518/1669] New improvements. --- builders.js | 54 +-- changes.txt | 23 ++ index.js | 594 +-------------------------------- internal.js | 158 +-------- test/controllers/default.js | 5 - test/test-framework-debug.js | 2 - test/test-framework-release.js | 2 - test/test-framework-test.js | 4 - 8 files changed, 51 insertions(+), 791 deletions(-) diff --git a/builders.js b/builders.js index 81779cc33..289c9966e 100755 --- a/builders.js +++ b/builders.js @@ -37,7 +37,7 @@ const Qs = require('querystring'); var schemas = {}; var operations = {}; -var transforms = { pagination: {}, error: {}, transformbuilder: {}, restbuilder: {} }; +var transforms = { pagination: {}, error: {}, restbuilder: {} }; function SchemaBuilder(name) { this.name = name; @@ -4051,56 +4051,6 @@ UrlBuilder.prototype.toOne = function(keys, delimiter) { return builder.join(delimiter || '&'); }; -function TransformBuilder() {} - -TransformBuilder.transform = function(name, obj) { - - OBSOLETE('TransformBuilder', 'Builders.TransformBuilder will be removed in next versions.'); - - var index = 2; - - if (obj === undefined) { - obj = name; - name = transforms['transformbuilder_default']; - index = 1; - } - - var current = transforms['transformbuilder'][name]; - if (!current) { - F.error('Transformation "' + name + '" not found.', 'TransformBuilder.transform()'); - return obj; - } - - var sum = arguments.length - index; - if (sum <= 0) - return current.call(obj, obj); - - var arr = new Array(sum + 1); - var indexer = 1; - arr[0] = obj; - for (var i = index; i < arguments.length; i++) - arr[indexer++] = arguments[i]; - return current.apply(obj, arr); -}; - -/** - * STATIC: Create a transformation - * @param {String} name - * @param {Function} fn - * @param {Boolean} isDefault Default transformation for all TransformBuilders. - */ -TransformBuilder.addTransform = function(name, fn, isDefault) { - transforms['transformbuilder'][name] = fn; - isDefault && TransformBuilder.setDefaultTransform(name); -}; - -TransformBuilder.setDefaultTransform = function(name) { - if (name) - transforms['transformbuilder_default'] = name; - else - delete transforms['transformbuilder_default']; -}; - function RESTBuilder(url) { this.$url = url; @@ -4909,14 +4859,12 @@ exports.ErrorBuilder = ErrorBuilder; exports.Pagination = Pagination; exports.Page = Page; exports.UrlBuilder = UrlBuilder; -exports.TransformBuilder = TransformBuilder; exports.SchemaOptions = SchemaOptions; exports.OperationOptions = OperationOptions; exports.RESTBuilderResponse = RESTBuilderResponse; global.RESTBuilder = RESTBuilder; global.RESTBuilderResponse = RESTBuilderResponse; global.ErrorBuilder = ErrorBuilder; -global.TransformBuilder = TransformBuilder; global.Pagination = Pagination; global.Page = Page; global.UrlBuilder = global.URLBuilder = UrlBuilder; diff --git a/changes.txt b/changes.txt index 8f73a7814..7ec8142c9 100755 --- a/changes.txt +++ b/changes.txt @@ -152,6 +152,29 @@ - replaced: config `default-maximum-file-descriptors` to `default-maxopenfiles` - replaced: `controller.proxy()` functionality (the name remains) via `controller.proxy2()` functionality +- removed: `F.responseFile()` +- removed: `F.responsePipe()` +- removed: `F.responseImage()` +- removed: `F.responseImageWithoutCache()` +- removed: `F.responseStream()` +- removed: `F.responseBinary()` +- removed: `F.responseContent()` +- removed: `F.responseRedirect()` +- removed: `F.response400()` +- removed: `F.response401()` +- removed: `F.response404()` +- removed: `F.response408()` +- removed: `F.response431()` +- removed: `F.response500()` +- removed: `F.response501()` +- removed: `F.responseStatic()` +- removed: `F.setModified()` +- removed: `F.notModified()` +- removed: `F.responseCode()` +- removed: `F.noCache()` +- removed: `controller.$modified()` +- removed: `controller.$etag()` + - improved: `debug` mode timing with improved consumption - improved: performance (+20%) NoSQL embedded database - improved: reading performance (+5%) in `U.streamer()` diff --git a/index.js b/index.js index 58bfe423a..7c53e5314 100755 --- a/index.js +++ b/index.js @@ -123,11 +123,6 @@ HEADERS.sse['Pragma'] = 'no-cache'; HEADERS.sse['Expires'] = '-1'; HEADERS.sse[HEADER_TYPE] = 'text/event-stream'; HEADERS.sse['X-Powered-By'] = 'Total.js'; -HEADERS.mmr = {}; -HEADERS.mmr[HEADER_CACHE] = 'private, no-cache, no-store, max-age=0'; -HEADERS.mmr['Pragma'] = 'no-cache'; -HEADERS.mmr['Expires'] = '-1'; -HEADERS.mmr['X-Powered-By'] = 'Total.js'; HEADERS.file_lastmodified = {}; HEADERS.file_lastmodified['Access-Control-Allow-Origin'] = '*'; HEADERS.file_lastmodified[HEADER_CACHE] = 'public, max-age=11111111'; @@ -255,7 +250,6 @@ HEADERS.authorization = { user: '', password: '', empty: true }; HEADERS.fsStreamRead = { flags: 'r', mode: '0666', autoClose: true }; HEADERS.fsStreamReadRange = { flags: 'r', mode: '0666', autoClose: true, start: 0, end: 0 }; HEADERS.workers = { cwd: '' }; -HEADERS.mmrpipe = { end: false }; HEADERS.responseLocalize = {}; HEADERS.responseNotModified = {}; HEADERS.responseNotModified[HEADER_CACHE] = 'public, max-age=11111111'; @@ -492,29 +486,6 @@ global.OFF = function() { return arguments.length > 1 ? F.removeListener.apply(F, arguments) : F.removeAllListeners.apply(F, arguments); }; -global.MAKE = global.TRANSFORM = function(transform, fn) { - - if (typeof(transform) === 'function') { - var tmp = fn; - fn = transform; - transform = tmp; - } - - var obj; - - if (typeof(fn) === 'function') { - obj = {}; - fn.call(obj, obj); - } else - obj = fn; - - return transform ? TransformBuilder.transform.apply(obj, arguments) : obj; -}; - -global.NEWTRANSFORM = function() { - return TransformBuilder.addTransform.apply(F, arguments); -}; - global.NEWSCHEMA = function(group, name, make) { if (typeof(name) === 'function') { @@ -785,8 +756,7 @@ function Framework() { mapping: {}, packages: {}, blocks: {}, - resources: {}, - mmr: {} + resources: {} }; this.owners = []; @@ -847,7 +817,6 @@ function Framework() { websocketPing: 0, websocketCleaner: 0, obsolete: 0, - restart: 0, mail: 0 }, @@ -866,7 +835,6 @@ function Framework() { path: 0, upload: 0, schema: 0, - mmr: 0, blocked: 0, 'delete': 0, mobile: 0, @@ -891,7 +859,6 @@ function Framework() { forward: 0, notModified: 0, sse: 0, - mmr: 0, errorBuilder: 0, error400: 0, error401: 0, @@ -999,7 +966,6 @@ F.prototypes = function(fn) { proto.RESTBuilderResponse = framework_builders.RESTBuilderResponse.prototype; proto.SchemaBuilder = framework_builders.SchemaBuilder.prototype; proto.SchemaOptions = framework_builders.SchemaOptions.prototype; - proto.TransformBuilder = framework_builders.TransformBuilder.prototype; proto.UrlBuilder = framework_builders.UrlBuilder.prototype; proto.WebSocket = WebSocket.prototype; proto.WebSocketClient = WebSocketClient.prototype; @@ -2449,13 +2415,6 @@ function flags_to_object(flags) { return obj; } -F.mmr = function(url, process) { - url = framework_internal.preparePath(U.path(url)); - F.routes.mmr[url] = { exec: process }; - F._request_check_POST = true; - return F; -}; - /** * Get routing by name * @param {String} name @@ -4574,150 +4533,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe }; F.restart = function() { - if (!F.isRestarted) { - F.isRestarted = true; - F.emit('restart'); - setTimeout(() => F.$restart(), 1000); - } - return F; -}; - -F.$restart = function() { - - console.log('----------------> RESTART ' + new Date().format('yyyy-MM-dd HH:mm:ss')); - - F.server.setTimeout(0); - F.server.timeout = 0; - F.server.close(function() { - - Object.keys(F.modules).forEach(function(key) { - var item = F.modules[key]; - item && item.uninstall && item.uninstall(); - }); - - Object.keys(F.models).forEach(function(key) { - var item = F.models[key]; - item && item.uninstall && item.uninstall(); - }); - - Object.keys(F.controllers).forEach(function(key) { - var item = F.controllers[key]; - item && item.uninstall && item.uninstall(); - }); - - Object.keys(F.workers).forEach(function(key) { - var item = F.workers[key]; - if (item && item.kill) { - item.removeAllListeners(); - item.kill('SIGTERM'); - } - }); - - Object.keys(F.connections).forEach(function(key) { - var item = F.connections[key]; - if (item) { - item.removeAllListeners(); - item.close(); - } - }); - - framework_builders.restart(); - framework_image.restart(); - framework_mail.restart(); - U.restart(); - framework_internal.restart(); - - F.cache.clear(); - F.cache.stop(); - F.$events = {}; - global.G = F.global = {}; - F.resources = {}; - F.connections = {}; - global.FUNCTIONS = F.functions = {}; - F.themes = {}; - F.uptodates = null; - F.versions = null; - F.schedules = []; - F.isLoaded = false; - F.isRestarted = false; - F.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; - PERF = {}; - - F.routes = { - sitemap: null, - web: [], - system: {}, - files: [], - cors: [], - corsall: false, - websockets: [], - middleware: {}, - redirects: {}, - resize: {}, - request: [], - views: {}, - merge: {}, - mapping: {}, - packages: {}, - blocks: {}, - resources: {}, - mmr: {} - }; - - F.temporary = { - path: {}, - notfound: {}, - processing: {}, - range: {}, - views: {}, - versions: {}, - dependencies: {}, - other: {}, - internal: {}, - owners: {}, - ready: {} - }; - - F.modificators = null; - F.helpers = {}; - F.modules = {}; - F.models = {}; - F.sources = {}; - F.controllers = {}; - F.dependencies = {}; - F.isomorphic = {}; - F.errors = []; - F.problems = []; - F.changes = []; - F.traces = []; - F.workers = {}; - F.convertors = []; - F.convertors2 = null; - F.databases = {}; - - F._request_check_redirect = false; - F._request_check_referer = false; - F._request_check_POST = false; - F._request_check_robot = false; - F._request_check_mobile = false; - F._length_middleware = 0; - F._length_request_middleware = 0; - F._length_files = 0; - F._length_wait = 0; - F._length_themes = 0; - F._length_cors = 0; - F._length_subdomain_web = 0; - F._length_subdomain_websocket = 0; - F.isVirtualDirectory = false; - F.isTheme = false; - F.stats.other.restart++; - - setTimeout(() => F.removeAllListeners(), 2000); - setTimeout(function() { - var init = F.temporary.init; - F.mode(init.isHTTPS ? require('https') : http, init.name, init.options); - }, 1000); - }); + OBSOLETE('F.restart()', 'This function is not supported'); return F; }; @@ -5693,8 +5509,7 @@ F.usage = function(detailed) { websocket: F.routes.websockets.length, file: F.routes.files.length, middleware: Object.keys(F.routes.middleware).length, - redirect: redirects.length, - mmr: Object.keys(F.routes.mmr).length + redirect: redirects.length }; output.stats = F.stats; @@ -6067,13 +5882,6 @@ function compile_content(extension, content, filename) { return content; } -// OBSOLETE -F.responseStatic = function(req, res, done) { - res.options.callback = done; - res.continue(); - return F; -}; - F.restore = function(filename, target, callback, filter) { var buffer_key = U.createBuffer(':'); @@ -6425,29 +6233,6 @@ F.isProcessing = function(filename) { return !!F.temporary.processing[filename]; }; -/** - * Disable HTTP cache for current request/response - * @param {Request} req Request - * @param {Response} res (optional) Response - * @return {Framework} - */ -F.noCache = function(req) { - OBSOLETE('F.noCache()', 'Use req.noCache() or res.noCache() --> they have same functionality.'); - req.noCache(); - return F; -}; - -// OBSOLETE -F.responseFile = function(req, res, filename, downloadName, headers, done, key) { - res.$key = key; - res.options.filename = filename; - res.options.download = downloadName; - res.options.headers = headers; - res.options.callback = done; - res.$file(); - return F; -}; - /** * Clears file information in release mode * @param {String/Request} url @@ -6465,192 +6250,6 @@ F.touch = function(url) { return F; }; -// OBSOLETE -F.responsePipe = function(req, res, url, headers, timeout, callback) { - res.pipe(url, headers, timeout, callback); - return F; -}; - -// OBSOLETE -F.responseCustom = function(req, res) { - res.$custom(); - return F; -}; - -// OBSOLETE -F.responseImage = function(req, res, filename, make, headers, done) { - - if (typeof(filename) === 'object') - res.options.stream = filename; - else - res.options.filename = filename; - - res.options.headers = headers; - res.options.callback = done; - res.options.make = make; - res.$image(); - return F; -}; - -// OBSOLETE -F.responseImageWithoutCache = function(req, res, filename, make, headers, done) { - - if (typeof(filename) === 'object') - res.options.stream = filename; - else - res.options.filename = filename; - - res.options.headers = headers; - res.options.callback = done; - res.options.make = make; - res.options.cache = false; - res.$image(); -}; - -// OBSOLETE -F.responseStream = function(req, res, type, stream, download, headers, done, nocompress) { - res.options.type = type; - res.options.stream = stream; - res.options.download = download; - res.options.headers = headers; - res.options.compress = nocompress ? false : true; - res.options.callback = done; - res.$stream(); - return F; -}; - -// OBSOLETE -F.responseBinary = function(req, res, type, buffer, encoding, download, headers, done) { - res.options.type = type; - res.options.body = buffer; - res.options.encoding = encoding; - res.options.download = download; - res.options.headers = headers; - res.options.callback = done; - res.$binary(); - return F; -}; - -F.setModified = function(req, res, value) { - if (typeof(value) === 'string') - res.setHeader('Etag', value + F.config['etag-version']); - else - res.setHeader('Last-Modified', value.toUTCString()); - return F; -}; - -F.notModified = function(req, res, compare, strict) { - - var type = typeof(compare); - if (type === 'boolean') { - var tmp = compare; - compare = strict; - strict = tmp; - type = typeof(compare); - } - - var isEtag = type === 'string'; - var val = req.headers[isEtag ? 'if-none-match' : 'if-modified-since']; - - if (isEtag) { - if (val !== (compare + F.config['etag-version'])) - return false; - } else { - - if (!val) - return false; - - var date = compare === undefined ? new Date().toUTCString() : compare.toUTCString(); - if (strict) { - if (new Date(Date.parse(val)) === new Date(date)) - return false; - } else { - if (new Date(Date.parse(val)) < new Date(date)) - return false; - } - } - - - var headers; - - if (isEtag) { - headers = HEADERS.notModifiedEtag; - headers['Etag'] = val; - } else { - headers = HEADERS.notModifiedLastModifiedDate; - headers['Last-Modified'] = val; - } - - res.success = true; - res.writeHead(304, headers); - res.end(); - - F.stats.response.notModified++; - response_end(res); - return true; -}; - -F.responseCode = function(req, res, code, problem) { - res.options.code = code; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response400 = function(req, res, problem) { - res.options.code = 400; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response401 = function(req, res, problem) { - res.options.code = 401; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response403 = function(req, res, problem) { - res.options.code = 403; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response404 = function(req, res, problem) { - res.options.code = 404; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response408 = function(req, res, problem) { - res.options.code = 408; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response431 = function(req, res, problem) { - res.options.code = 431; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - -F.response500 = function(req, res, error) { - res.throw500(error); - return F; -}; - -F.response501 = function(req, res, problem) { - res.options.code = 501; - problem && (res.options.problem = problem); - res.$throw(); - return F; -}; - F.response503 = function(req, res) { res.options.code = 503; res.options.headers = HEADERS.response503; @@ -6659,25 +6258,6 @@ F.response503 = function(req, res) { return F; }; -// OBSOLETE -F.responseContent = function(req, res, code, body, type, compress, headers) { - res.options.code = code; - res.options.body = body; - res.options.type = type; - res.options.compress = compress === undefined || compress === true; - res.options.headers = headers; - res.$text(); - return F; -}; - -// OBSOLETE -F.responseRedirect = function(req, res, url, permanent) { - res.options.url = url; - res.options.permanent = permanent; - res.$redirect(); - return F; -}; - global.LOAD = F.load = function(debug, types, pwd) { if (pwd && pwd[0] === '.' && pwd.length < 4) @@ -6793,7 +6373,7 @@ global.LOAD = F.load = function(debug, types, pwd) { * @param {Object} options * @return {Framework} */ -F.initialize = function(http, debug, options, restart) { +F.initialize = function(http, debug, options) { if (!options) options = {}; @@ -6908,7 +6488,7 @@ F.initialize = function(http, debug, options, restart) { F.usagesnapshot(); } - if (!process.connected || restart) + if (!process.connected) F.console(); setTimeout(function() { @@ -7061,17 +6641,11 @@ F.mode = function(http, name, options) { break; } - var restart = false; - if (F.temporary.init) - restart = true; - else - F.temporary.init = { name: name, isHTTPS: typeof(http.STATUS_CODES) === 'undefined', options: options }; - F.config.trace = debug; F.consoledebug('startup'); F.$startup(function() { F.consoledebug('startup (done)'); - F.initialize(http, debug, options, restart); + F.initialize(http, debug, options); }); return F; }; @@ -7116,17 +6690,11 @@ F.custom = function(mode, http, request, response, options) { break; } - var restart = false; - if (F.temporary.init) - restart = true; - else - F.temporary.init = { name: mode, isHTTPS: false, options: options }; - F.config.trace = debug; F.consoledebug('startup'); F.$startup(function() { F.consoledebug('startup (done)'); - F.initialize(http, debug, options, restart); + F.initialize(http, debug, options); }); return F; @@ -7511,11 +7079,6 @@ F.$requestcontinue = function(req, res, headers) { req.$type = 2; multipart = ''; break; - case 'lace': - req.$type = 4; - flags.push('mmr'); - req.$flags += 'e'; - break; default: if (multipart) { // 'undefined' DATA @@ -7599,8 +7162,6 @@ F.$requestcontinue = function(req, res, headers) { if (multipart) { if (isCORS) F.$cors(req, res, cors_callback_multipart, multipart); - else if (req.$type === 4) - F.$requestcontinue_mmr(req, res, multipart); else req.$total_multipart(multipart); } else { @@ -7632,22 +7193,9 @@ function cors_callback1(req) { } function cors_callback_multipart(req, res, multipart) { - if (req.$type === 4) - F.$requestcontinue_mmr(req, res, multipart); - else - req.$total_multipart(multipart); + req.$total_multipart(multipart); } -F.$requestcontinue_mmr = function(req, res, header) { - var route = F.routes.mmr[req.url]; - F.stats.request.mmr++; - if (route) { - F.path.verify('temp'); - framework_internal.parseMULTIPART_MIXED(req, header, F.config['directory-temp'], route.exec); - } else - req.$total_status(404); -}; - F.$cors = function(req, res, fn, arg) { var isAllowed = F.routes.corsall; @@ -11241,43 +10789,6 @@ Controller.prototype.mail = function(address, subject, view, model, callback) { return message; }; -/* - Check if ETag or Last Modified has modified - @compare {String or Date} - @strict {Boolean} :: if strict then use equal date else use great than date (default: false) - - if @compare === {String} compare if-none-match - if @compare === {Date} compare if-modified-since - - return {Boolean}; -*/ -Controller.prototype.notModified = function(compare, strict) { - return F.notModified(this.req, this.res, compare, strict); -}; - -/* - Set last modified header or Etag - @value {String or Date} - - if @value === {String} set ETag - if @value === {Date} set LastModified - - return {Controller}; -*/ -Controller.prototype.setModified = function(value) { - F.setModified(this.req, this.res, value); - return this; -}; - -/** - * Sets expire headers - * @param {Date} date - */ -Controller.prototype.setExpires = function(date) { - date && this.res.setHeader('Expires', date.toUTCString()); - return this; -}; - Controller.prototype.$template = function(name, model, expire, key) { return this.$viewToggle(true, name, model, expire, key); }; @@ -11725,48 +11236,6 @@ Controller.prototype.$isValue = function(bool, charBeg, charEnd, value) { return charBeg + value + charEnd; }; -Controller.prototype.$modified = function(value) { - - var self = this; - var type = typeof(value); - var date; - - if (type === 'number') { - date = new Date(value); - } else if (type === 'string') { - - var d = value.split(' '); - - date = d[0].split('-'); - var time = (d[1] || '').split(':'); - - var year = U.parseInt(date[0] || ''); - var month = U.parseInt(date[1] || '') - 1; - var day = U.parseInt(date[2] || '') - 1; - - if (month < 0) - month = 0; - - if (day < 0) - day = 0; - - var hour = U.parseInt(time[0] || ''); - var minute = U.parseInt(time[1] || ''); - var second = U.parseInt(time[2] || ''); - - date = new Date(year, month, day, hour, minute, second, 0); - } else if (U.isDate(value)) - date = value; - - date && self.setModified(date); - return ''; -}; - -Controller.prototype.$etag = function(value) { - this.setModified(value); - return ''; -}; - Controller.prototype.$options = function(arr, selected, name, value) { var type = typeof(arr); @@ -12341,9 +11810,7 @@ Controller.prototype.json = function(obj, headers, beautify, replacer) { }; Controller.prototype.success = function(is, value) { - if (is === undefined) - is = true; - return this.json(SUCCESS(is, value)); + return this.json(SUCCESS(is === undefined ? true : is, value)); }; /** @@ -12873,45 +12340,6 @@ Controller.prototype.sse = function(data, eventname, id, retry) { return self; }; -Controller.prototype.mmr = function(name, stream, callback) { - - var self = this; - var res = self.res; - - if (typeof(stream) === 'function') { - callback = stream; - stream = name; - } - - if (!stream) - stream = name; - - if (!self.isConnected || (!self.type && res.success) || (self.type && self.type !== 2)) { - callback = null; - return self; - } - - if (!self.type) { - self.type = 2; - self.boundary = '----totaljs' + U.GUID(10); - self.req.$total_success(); - self.req.on('close', () => self.close()); - res.success = true; - HEADERS.mmr[HEADER_TYPE] = 'multipart/x-mixed-replace; boundary=' + self.boundary; - res.writeHead(self.status || 200, HEADERS.mmr); - } - - res.write('--' + self.boundary + NEWLINE + HEADER_TYPE + ': ' + U.getContentType(U.getExtension(name)) + NEWLINE + NEWLINE); - F.stats.response.mmr++; - - if (typeof(stream) === 'string') - stream = Fs.createReadStream(stream); - - stream.pipe(res, HEADERS.mmrpipe); - CLEANUP(stream, () => callback && callback()); - return self; -}; - /** * Close a response * @param {Boolean} end @@ -14695,10 +14123,6 @@ function extend_request(PROTO) { return this; }; - PROTO.notModified = function(compare, strict) { - return F.notModified(this, this.res, compare, strict); - }; - /** * Read a cookie from current request * @param {String} name Cookie name. diff --git a/internal.js b/internal.js index 1fdeb070a..1e9c5a237 100755 --- a/internal.js +++ b/internal.js @@ -322,128 +322,6 @@ function uploadparser_done() { this.buffer_parser.end(); } -exports.parseMULTIPART_MIXED = function(req, contentType, tmpDirectory, onFile) { - - var boundary = contentType.split(';')[1]; - if (!boundary) { - F.reqstats(false, false); - F.stats.request.error400++; - req.res.writeHead(400); - req.res.end(); - return; - } - - // For unexpected closing - req.once('close', () => !req.$upload && req.clear()); - - var parser = new MultipartParser(); - var close = 0; - var stream; - var tmp; - var counter = 0; - var path = framework_utils.combine(tmpDirectory, (F.id ? 'i-' + F.id + '_' : '') + 'uploadedmixed-'); - - boundary = boundary.substring(boundary.indexOf('=', 2) + 1); - req.buffer_exceeded = false; - req.buffer_has = true; - req.buffer_parser = parser; - - parser.initWithBoundary(boundary); - - parser.onPartBegin = function() { - // Temporary data - tmp = new HttpFile(); - tmp.$step = 0; - tmp.$is = false; - tmp.length = 0; - }; - - parser.onHeaderValue = function(buffer, start, end) { - - if (req.buffer_exceeded) - return; - - var header = buffer.slice(start, end).toString(ENCODING); - if (tmp.$step === 1) { - var index = header.indexOf(';'); - if (index === -1) - tmp.type = header.trim(); - else - tmp.type = header.substring(0, index).trim(); - tmp.$step = 2; - return; - } - - if (tmp.$step !== 0) - return; - - header = parse_multipart_header(header); - - tmp.$step = 1; - tmp.$is = header[1] !== null; - tmp.name = header[0]; - - if (tmp.$is) { - tmp.filename = header[1]; - tmp.path = path + (INDEXMIXED++) + '.bin'; - - stream = Fs.createWriteStream(tmp.path, WRITESTREAM); - stream.once('close', () => close--); - stream.once('error', () => close--); - close++; - } else - destroyStream(stream); - }; - - parser.onPartData = function(buffer, start, end) { - - if (req.buffer_exceeded) - return; - - var data = buffer.slice(start, end); - var length = data.length; - - if (!tmp.$is) - return; - - if (tmp.length) { - stream.write(data); - tmp.length += length; - return; - } - - stream.write(data); - tmp.length += length; - onFile(req, tmp, counter++); - }; - - parser.onPartEnd = function() { - - if (stream) { - stream.end(); - stream = null; - } - - if (req.buffer_exceeded || !tmp.$is) - return; - - tmp.$is = undefined; - tmp.$step = undefined; - }; - - parser.onEnd = function() { - if (close) { - setImmediate(parser.onEnd); - } else { - onFile(req, null); - F.responseContent(req, req.res, 200, EMPTYBUFFER, 'text/plain', false); - } - }; - - req.on('data', uploadparser); - req.on('end', uploadparser_done); -}; - function parse_multipart_header(header) { var arr = new Array(2); @@ -780,7 +658,9 @@ function HttpFile() { this.rem = true; } -HttpFile.prototype.rename = HttpFile.prototype.move = function(filename, callback) { +var HFP = HttpFile.prototype; + +HFP.rename = HFP.move = function(filename, callback) { var self = this; Fs.rename(self.path, filename, function(err) { @@ -794,7 +674,7 @@ HttpFile.prototype.rename = HttpFile.prototype.move = function(filename, callbac return self; }; -HttpFile.prototype.copy = function(filename, callback) { +HFP.copy = function(filename, callback) { var self = this; @@ -811,38 +691,38 @@ HttpFile.prototype.copy = function(filename, callback) { return self; }; -HttpFile.prototype.$$rename = HttpFile.prototype.$$move = function(filename) { +HFP.$$rename = HFP.$$move = function(filename) { var self = this; return function(callback) { return self.rename(filename, callback); }; }; -HttpFile.prototype.$$copy = function(filename) { +HFP.$$copy = function(filename) { var self = this; return function(callback) { return self.copy(filename, callback); }; }; -HttpFile.prototype.readSync = function() { +HFP.readSync = function() { return Fs.readFileSync(this.path); }; -HttpFile.prototype.read = function(callback) { +HFP.read = function(callback) { var self = this; Fs.readFile(self.path, callback); return self; }; -HttpFile.prototype.$$read = function() { +HFP.$$read = function() { var self = this; return function(callback) { self.read(callback); }; }; -HttpFile.prototype.md5 = function(callback) { +HFP.md5 = function(callback) { var self = this; var md5 = Crypto.createHash('md5'); var stream = Fs.createReadStream(self.path); @@ -866,40 +746,40 @@ HttpFile.prototype.md5 = function(callback) { return self; }; -HttpFile.prototype.$$md5 = function() { +HFP.$$md5 = function() { var self = this; return function(callback) { self.md5(callback); }; }; -HttpFile.prototype.stream = function(options) { +HFP.stream = function(options) { return Fs.createReadStream(this.path, options); }; -HttpFile.prototype.pipe = function(stream, options) { +HFP.pipe = function(stream, options) { return Fs.createReadStream(this.path, options).pipe(stream, options); }; -HttpFile.prototype.isImage = function() { +HFP.isImage = function() { return this.type.indexOf('image/') !== -1; }; -HttpFile.prototype.isVideo = function() { +HFP.isVideo = function() { return this.type.indexOf('video/') !== -1; }; -HttpFile.prototype.isAudio = function() { +HFP.isAudio = function() { return this.type.indexOf('audio/') !== -1; }; -HttpFile.prototype.image = function(im) { +HFP.image = function(im) { if (im === undefined) im = F.config['default-image-converter'] === 'im'; return framework_image.init(this.path, im, this.width, this.height); }; -HttpFile.prototype.fs = function(storagename, custom, callback) { +HFP.fs = function(storagename, custom, callback) { if (typeof(custom) === 'function') { callback = custom; @@ -2403,9 +2283,7 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'selected': case 'disabled': case 'checked': - case 'etag': case 'header': - case 'modified': case 'options': case 'readonly': case 'canonical': diff --git a/test/controllers/default.js b/test/controllers/default.js index d73e692df..43c86d0cf 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -464,14 +464,9 @@ function viewIndex() { assert.ok(self.hash('sha1', '123456', false) === '7c4a8d09ca3762af61e59520943dc26494f8941b', 'controller.hash()'); - self.setModified('123456'); - var date = new Date(); date.setFullYear(1984); - self.setModified(date); - self.setExpires(date); - assert.ok(self.routeScript('p.js') === '/js/p.js', name + 'routeScript()'); assert.ok(self.routeStyle('p.css') === '/css/p.css', name + 'routeStyle()'); assert.ok(self.routeImage('p.jpg') === '/img/p.jpg', name + 'routeImage()'); diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 833a3d715..cbcade096 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -68,8 +68,6 @@ function test_controller_functions(next) { utils.request(url, ['get'], function(error, data, code, headers) { error && assert.ok(false, 'test_controller_functions: ' + error.toString()); assert.ok(code === 404, 'controller: statusCode ' + code); - assert.ok(headers['etag'] === '1234561', 'controller: setModified(etag)'); - assert.ok(headers['last-modified'].toString().indexOf('1984') !== -1, 'controller: setModified(date)'); next(); }); } diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 6b28ef1bf..e47ac4ad0 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -68,8 +68,6 @@ function test_controller_functions(next) { utils.request(url, ['get'], function(error, data, code, headers) { error && assert.ok(false, 'test_controller_functions: ' + error.toString()); assert.ok(code === 404, 'controller: statusCode ' + code); - assert.ok(headers['etag'] === '1234561', 'controller: setModified(etag)'); - assert.ok(headers['last-modified'].toString().indexOf('1984') !== -1, 'controller: setModified(date)'); next(); }); } diff --git a/test/test-framework-test.js b/test/test-framework-test.js index 9a3ea898d..a17eaf8cc 100755 --- a/test/test-framework-test.js +++ b/test/test-framework-test.js @@ -71,10 +71,6 @@ function test_controller_functions(next) { assert.ok(false, 'test_controller_functions: ' + error.toString()); assert.ok(code === 404, 'controller: statusCode ' + code); - assert.ok(headers['etag'] === '123456:1', 'controller: setModified(etag)'); - assert.ok(headers['last-modified'].toString().indexOf('1984') !== -1, 'controller: setModified(date)'); - assert.ok(headers['expires'].toString().indexOf('1984') !== -1, 'controller: setExpires(date)'); - next(); }); } From 225e3f827d9b681171fc38130a074fdc36d233e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 19:33:53 +0200 Subject: [PATCH 0519/1669] Improved memory usage. --- graphdb.js | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/graphdb.js b/graphdb.js index fe5e30535..1917c48f3 100644 --- a/graphdb.js +++ b/graphdb.js @@ -112,6 +112,7 @@ function GraphDB(name) { self.next(value); }; + F.grapdbinstance = true; self.open(); } diff --git a/index.js b/index.js index 7c53e5314..fe4ef349f 100755 --- a/index.js +++ b/index.js @@ -1321,7 +1321,7 @@ F.stop = F.kill = function(signal) { F.server.close(); } - var extenddelay = require('./graphdb').getImportantOperations() > 0; + var extenddelay = F.grapdbinstance && require('./graphdb').getImportantOperations() > 0; setTimeout(() => process.exit(signal), global.TEST || extenddelay ? 2000 : 300); return F; }; From 400918325d5eb69cf7ccc43df43987a8fea4a1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 19:52:50 +0200 Subject: [PATCH 0520/1669] Updated beta. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fe0868e20..1b2e69c60 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-87", + "version": "3.0.0-88", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From f5aab53346ad5f3e139b4e50a30b0253a0587ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 19:58:49 +0200 Subject: [PATCH 0521/1669] Added back `F.responseFile()`. --- index.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/index.js b/index.js index fe4ef349f..ca65ed850 100755 --- a/index.js +++ b/index.js @@ -6250,6 +6250,18 @@ F.touch = function(url) { return F; }; +// OBSOLETE +F.responseFile = function(req, res, filename, downloadName, headers, done, key) { + OBSOLETE('F.responseFile()', 'This method is obsolete, use res.file()'); + res.$key = key; + res.options.filename = filename; + res.options.download = downloadName; + res.options.headers = headers; + res.options.callback = done; + res.$file(); + return F; +}; + F.response503 = function(req, res) { res.options.code = 503; res.options.headers = HEADERS.response503; From 221153df2a3fbf6163c27f2ae35e5d79c7960030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 22:48:14 +0200 Subject: [PATCH 0522/1669] Added new stats. --- changes.txt | 1 + index.js | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 7ec8142c9..d405c02e0 100755 --- a/changes.txt +++ b/changes.txt @@ -100,6 +100,7 @@ - added: `NOSQL()/TABLE().memory(count, [size])` for memory consumption, more in docs - added: `HttpFile.fs(storage_name, [custom], [callback])` saves a file into the FileStorage - added: `res.filefs(storage_name, id, [download], [headers], [callback])` returns file from FileStorage +- added: new stats `F.stats.performance` contains count of `request` and `file` per minute - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` diff --git a/index.js b/index.js index ca65ed850..66d9dccec 100755 --- a/index.js +++ b/index.js @@ -808,11 +808,17 @@ function Framework() { other: {}, internal: {}, // controllers/modules names for the routing owners: {}, - ready: {} + ready: {}, + service: { redirect: 0, request: 0, file: 0 } }; this.stats = { + performance: { + request: 0, + file: 0 + }, + other: { websocketPing: 0, websocketCleaner: 0, @@ -6788,6 +6794,19 @@ F.service = function(count) { var releasegc = false; + if (F.temporary.service.request) + F.temporary.service.request++; + else + F.temporary.service.request = 1; + + if (F.temporary.service.file) + F.temporary.service.file++; + else + F.temporary.service.file = 1; + + F.stats.performance.request = F.stats.request.request ? F.stats.request.request / F.temporary.service.request : 0; + F.stats.performance.file = F.stats.request.file ? F.stats.request.file / F.temporary.service.file : 0; + // clears temporary memory for non-exist files F.temporary.notfound = {}; From ec13667fd9dc93fa43e0d761ee3809a4a59f94c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 19 Jul 2018 23:23:36 +0200 Subject: [PATCH 0523/1669] Added `res.filenosql()` and `httpfile.nosql()`. --- changes.txt | 2 ++ index.js | 11 +++++++++++ internal.js | 10 ++++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index d405c02e0..e2e8bcbc8 100755 --- a/changes.txt +++ b/changes.txt @@ -99,7 +99,9 @@ - added: `F.dir(path)` for changing of root directory - added: `NOSQL()/TABLE().memory(count, [size])` for memory consumption, more in docs - added: `HttpFile.fs(storage_name, [custom], [callback])` saves a file into the FileStorage +- added: `HttpFile.nosql(db_name, [custom], [callback])` saves a file into the FileStorage - added: `res.filefs(storage_name, id, [download], [headers], [callback])` returns file from FileStorage +- added: `res.filenosql(db_name, id, [download], [headers], [callback])` returns file from FileStorage - added: new stats `F.stats.performance` contains count of `request` and `file` per minute - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory diff --git a/index.js b/index.js index 66d9dccec..192030188 100755 --- a/index.js +++ b/index.js @@ -14985,6 +14985,17 @@ function extend_response(PROTO) { return self; }; + PROTO.filenosql = function(name, id, download, headers, callback) { + var self = this; + var options = {}; + options.id = id; + options.download = download; + options.headers = headers; + options.done = callback; + NOSQL(name).binary.res(self, options, $file_notmodified); + return self; + }; + /** * Responds with a stream * @param {String} contentType diff --git a/internal.js b/internal.js index 1e9c5a237..513fa7a18 100755 --- a/internal.js +++ b/internal.js @@ -780,15 +780,21 @@ HFP.image = function(im) { }; HFP.fs = function(storagename, custom, callback) { - if (typeof(custom) === 'function') { callback = custom; custom = null; } - return FILESTORAGE(storagename).insert(this.filename, Fs.createReadStream(this.path), custom, callback); }; +HFP.nosql = function(name, custom, callback) { + if (typeof(custom) === 'function') { + callback = custom; + custom = null; + } + return NOSQL(name).binary.insert(this.filename, Fs.createReadStream(this.path), custom, callback); +}; + // ********************************************************************************* // ================================================================================= // JS CSS + AUTO-VENDOR-PREFIXES From 00972ef3734b99218e3994763dcfb3b42a52edb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 08:09:48 +0200 Subject: [PATCH 0524/1669] Fixed bug with persistent items in cache. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 192030188..b49361abf 100755 --- a/index.js +++ b/index.js @@ -9418,7 +9418,7 @@ FrameworkCache.prototype.savePersist = function() { for (var i = 0, length = keys.length; i < length; i++) { var key = keys[i]; var item = self.items[key]; - if (item.persist) + if (item && item.persist) obj[key] = item; } From 329d7cac0f3f7f390360a8946955e55dd3cd240a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 08:10:41 +0200 Subject: [PATCH 0525/1669] Updated beta vesion. --- package.json | 2 +- test/test-tmp.js | 226 ----------------------------------------------- 2 files changed, 1 insertion(+), 227 deletions(-) diff --git a/package.json b/package.json index 1b2e69c60..70460531c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-88", + "version": "3.0.0-89", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/test/test-tmp.js b/test/test-tmp.js index 15712cb79..25f47af6a 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,228 +1,2 @@ require('../index'); -const Fs = require('fs'); - -var insert = 1; -var link = 0; -var update = 0; -var read = 0; -var remove = 0; - -if (insert) { - try { - Fs.unlinkSync(F.path.databases('skuska.gdb')); - } catch (e) {} -} - -var db = GRAPHDB('skuska'); - -db.ready(function() { - - db.class('user', 'name:string|age:number'); - db.class('order', 'id:string|price:number'); - db.class('tags', 'name:string'); - db.relation('like', true); - - // this.insert('Si kokot?', console.log); - //this.count(console.log); - //this.read(20303, console.log); - - /* - setTimeout(function() { - db.connect('like', 8, 9); - setTimeout(function() { - db.read(2, console.log); - // db.read(6, console.log); - // db.read(13, console.log); - // var opt = {}; - // opt.relation = 'like'; - // opt.class = 'fet'; - // db.graph(8, opt, function(err, doc) { - // console.log(JSON.stringify(doc, null, ' ')); - // }); - }, 500); - }, 500); - */ - - setTimeout(function() { - var count = 0; - - //db.findRelationCount('order', (err, count) => console.log('order', count)); - // db.findRelationCount('user', (err, count) => console.log('user', count)); - // db.findRelationCount('tags', (err, count) => console.log('tags', count)); - - // db.findRelation(db.$classes.order.nodeid, function(id, next) { - - // for (var i = 0; i < id.length; i++) { - // var a = ordersid[id[i].ID]; - - // if (a == null) - // console.log('MISSING', id[i]); - // else - // ordersid[id[i].ID]++; - // } - - // if (next) - // next(); - // else - // console.log(ordersid); - // }); - - //db.read(34, (err, doc) => console.log(34, doc)); - //db.read(35, (err, doc) => console.log(35, doc)); - - // db.read(34, (err, doc) => console.log(34, doc.length)); - // db.read(35, (err, doc) => console.log(35, doc.length)); - // db.read(36, (err, doc) => console.log(36, doc.length)); - // db.read(37, (err, doc) => console.log(37, doc.length)); - - // db.find('user').callback((err, docs) => console.log('users', docs.length)); - // db.find('order').callback((err, docs) => console.log('orders', docs.length)); - // db.find('tags').callback((err, docs) => console.log('tags', docs.length)); - - }, 500); - - // db.find('user').take(100).callback((err, docs, count) => console.log('users', docs.length, count)); - // db.find('order').take(100).callback((err, docs, count) => console.log('orders', docs.length, count)); - // db.find('tags').take(100).callback((err, docs, count) => console.log('tags', docs.length, count)); - - // setTimeout(function() { - // db.find2('user').take(100).callback(console.log); - // }, 100); - - // var sum = 0; - // db.findRelation(db.$classes.user.nodeid, function(id, next) { - // sum += id.length; - // console.log(id); - // if (next) - // next(); - // else - // console.log('--->', sum); - // }); - - //console.log(db); - - if (insert) { - - var max = 10; - - (max).async(function(i, next) { - if (i && i % 1000 === 0) - console.log('user', i); - db.insert('user', { name: GUID(5), age: i + 20 }, next); - }, function() { - console.log('user done'); - }); -/* - (max).async(function(i, next) { - if (i && i % 100000 === 0) - console.log('order', i); - db.insert('order', { id: UID(), price: (i + 1) }, next); - }, function() { - console.log('order done'); - }); - - (max).async(function(i, next) { - - if (i && i % 100000 === 0) - console.log('tags', i); - - db.insert('tags', { name: GUID(5) }, next); - }, function() { - console.log('tags done'); - });*/ - - } - - // db.read(1, console.log); - - // db.read(11, console.log); - // null [ { RELATION: 1, TYPE: 1, ID: 5, INDEX: 0 } ] 0 - - // db.connect('like', 1, 5); - - // db.read(1000001, (err, links) => console.log(1000001, links)); - // db.read(1000002, (err, links) => console.log(1000002, links)); - // db.read(1000003, (err, links) => console.log(1000003, links)); - // db.read(1000004, (err, links) => console.log(1000004, links)); - // db.read(1000005, (err, links) => console.log(1000005, links)); - // db.read(1000006, (err, links) => console.log(1000006, links)); - - //db.link(100, 1000, 1, 0); - //db.link(100, 10000, 1, 0); - //db.link(100000, 200000, 1, 1); - //db.link(100, 1000000, 1, 0); - - //db.read(1000004, console.log); - - if (link) { - // 11 - //GRAPHDB('skuska').setLinkId(null, 4, 2, console.log); - //GRAPHDB('skuska').setLinkId(11, 3, 6, console.log); - //GRAPHDB('skuska').read(11, console.log); - - //GRAPHDB('skuska').read(1001, console.log); - // 100 knows 1000 (but not vice versa) - //GRAPHDB('skuska').link(10, 30, 1, 0, console.log); - //GRAPHDB('skuska').link(20, 30, 1, 0, console.log); - //return; - //return; - - // 100 knows 100000 (but not vice versa) - //GRAPHDB('skuska').link(100, 10000, 1, 0, console.log); - //return; - - //GRAPHDB('skuska').read(100001, console.log); - //return; - - var opt = {}; - - opt.type = 1; - // opt.relation = 1; - - db.graph(100, opt, function(err, doc) { - console.log(JSON.stringify(doc, null, ' ')); - }); - - //db.read(1000005, console.log); - } - - if (remove) { - //console.log(db); - //db.read(51, console.log); - // GRAPHDB('skuska').clean([10, 51], console.log); - setTimeout(function() { - //db.remove(100, console.log); - }, 1000); - } - - if (read) { - db.read(50, console.log); - } - - if (update) { - db.update(2, { id: 2, guid: GUID(100), kokotaris: 99 }, console.log); - } - - // GRAPHDB('skuska').insert({ index: 999999 }, console.log); - - //GRAPHDB('skuska').remove(100, console.log); - - // GRAPHDB('skuska').read(2, console.log); - //GRAPHDB('skuska').link(2, 9, console.log); -}); - -/* -KEYVALUE('skuska').traverse(340304, function(err, val, index) { - console.log(val); - if (index === 6) - return false; -}); -*/ -/* - -KEYVALUE('skuska').ready(function() { - for (var i = 0; i < 1000000; i++) - KEYVALUE('skuska').put({ id: i + 1, guid: GUID(100) }); -}); -*/ \ No newline at end of file From 8c72bf1df6670414fcca980f45cb7e2e43960d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 09:54:24 +0200 Subject: [PATCH 0526/1669] Fixed compressing 3rd party compilers. --- internal.js | 41 ++++++++++++++++++++++--------------- test/controllers/default.js | 3 +-- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/internal.js b/internal.js index 513fa7a18..7c30acde2 100755 --- a/internal.js +++ b/internal.js @@ -80,11 +80,9 @@ const REG_CSS_11 = /\$[a-z0-9-_]+/gi; const REG_CSS_12 = /(margin|padding):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; -const EMPTYBUFFER = framework_utils.createBufferSize(0); const ALLOWEDMARKUP = { G: 1, M: 1, R: 1, repository: 1, model: 1, config: 1, global: 1, resource: 1, RESOURCE: 1, CONFIG: 1, author: 1, root: 1, functions: 1, NOW: 1, F: 1 }; var INDEXFILE = 0; -var INDEXMIXED = 0; global.$STRING = function(value) { return value != null ? value.toString() : ''; @@ -1856,7 +1854,7 @@ function view_parse(content, minify, filename, controller) { builder += '+' + DELIMITER + (new Function('self', 'return self.$import(' + cmd[0] + '!' + cmd.substring(1) + ')'))(controller) + DELIMITER; } else if (cmd7 === 'compile' && cmd.lastIndexOf(')') === -1) { - builderTMP = builder + '+(F.onCompileView.call(self,\'' + (cmd8[7] === ' ' ? cmd.substring(8) : '') + '\','; + builderTMP = builder + '+(F.onCompileView.call(self,\'' + (cmd8[7] === ' ' ? cmd.substring(8).trim() : '') + '\','; builder = ''; sectionName = cmd.substring(8); isCOMPILATION = true; @@ -2501,12 +2499,25 @@ function removeComments(html) { function compressView(html, minify) { var cache = []; + var beg = 0; + var end; while (true) { - var beg = html.indexOf('@{'); + beg = html.indexOf('@{compile ', beg - 1); if (beg === -1) break; - var end = html.indexOf('}', beg + 2); + end = html.indexOf('@{end}', beg + 6); + if (end === -1) + break; + cache.push(html.substring(beg, end + 6)); + html = html.substring(0, beg) + '#@' + (cache.length - 1) + '#' + html.substring(end + 6); + } + + while (true) { + beg = html.indexOf('@{', beg); + if (beg === -1) + break; + end = html.indexOf('}', beg + 2); if (end === -1) break; cache.push(html.substring(beg, end + 1)); @@ -2845,16 +2856,17 @@ function compressHTML(html, minify, isChunk) { var indexer = 0; var length = tags.length; var chars = 65; + var tagBeg, tagEnd, beg, end, len, key, value; for (var i = 0; i < length; i++) { var o = tags[i]; - var tagBeg = '<' + o; - var tagEnd = ' 90) chars = 65; - var value = html.substring(beg, end + len); + value = html.substring(beg, end + len); if (!i) { end = value.indexOf('>'); @@ -3289,11 +3301,6 @@ function existsSync(filename) { } } -exports.restart = function() { - INDEXMIXED = 0; - INDEXFILE = 0; -}; - function markup(body) { body = body.ROOT(); var command = view_find_command(body, 0, true); diff --git a/test/controllers/default.js b/test/controllers/default.js index 43c86d0cf..6e74ebade 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -530,7 +530,6 @@ function viewViews() { //console.log('\n\n\n'); //self.framework.stop(); //return; - assert.ok(output.contains('#COMPONENTVIEWPETER#'), name + 'components rendering'); assert.ok(output.contains('#
@{{ vue_command }}
#'), name + 'VUE command'); assert.ok(output.contains('#mobilefalse#'), name + 'mobile'); @@ -539,7 +538,7 @@ function viewViews() { assert.ok(output.contains('HELPER:1-10'), name + 'inline helper + foreach 1'); assert.ok(output.contains('HELPER:2-21'), name + 'inline helper + foreach 2'); assert.ok(output.contains('
SECTION
'), name + 'section'); - assert.ok(output.contains('COMPILE_TANGULARCOMPILED'), name + 'onCompileView with name'); + assert.ok(output.contains('COMPILE_TANGULAR\nCOMPILED'), name + 'onCompileView with name'); assert.ok(output.contains('COMPILE_WITHOUTCOMPILED'), name + 'onCompileView without name'); assert.ok(output.contains('
4
4
FOREACH
'), name + 'foreach'); assert.ok(output.contains('
3
3
C:10
C:11
C:12
'), name + 'foreach - nested'); From 7de3e2c264d74f6b41d171cc5671e28af5085b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 10:15:41 +0200 Subject: [PATCH 0527/1669] Fixed callbacks. --- index.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index b49361abf..482f1ecc0 100755 --- a/index.js +++ b/index.js @@ -16017,7 +16017,17 @@ function response_end(res) { !res.req.isStaticFile && F.$events['request-end'] && F.emit('request-end', res.req, res); res.req.clear(true); res.controller && res.req.$total_success(); - res.options.callback && res.options.callback(); + + if (res.options.callback) { + res.options.callback(); + res.options.callback = null; + } + + if (res.options.done) { + res.options.done(); + res.options.done = null; + } + // res.options = EMPTYOBJECT; res.controller = null; } From 5deaf4c0f1e1e184317bcfb1ae342df82329e51f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 13:31:00 +0200 Subject: [PATCH 0528/1669] Added cluster locking. --- cluster.js | 14 ++++++- index.js | 4 +- nosql.js | 107 +++++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 111 insertions(+), 14 deletions(-) diff --git a/cluster.js b/cluster.js index 46327dae9..95567a852 100644 --- a/cluster.js +++ b/cluster.js @@ -24,6 +24,7 @@ * @version 3.0.0 */ +const Fs = require('fs'); const Cluster = require('cluster'); const CLUSTER_REQ = { TYPE: 'req' }; const CLUSTER_RES = { TYPE: 'res' }; @@ -192,13 +193,24 @@ function master(count, mode, options, callback) { console.log('Mode : ' + mode); console.log('====================================================\n'); + // Remove all DB locks + Fs.readdir(F.path.databases(), function(err, files) { + if (!err) { + var reglock = /(\.nosql-lock|\.table-lock|\.nosql-counter2-lock)$/; + for (var i = 0; i < files.length; i++) { + var file = files[i]; + reglock.test(file) && Fs.unlinkSync(F.path.databases(file)); + } + } + }); + THREADS = count; var can = function(cb) { if (CONTINUE) cb(); else - setTimeout(can, 1000, cb); + setTimeout(can, 500, cb); }; count.async(function(i, next) { diff --git a/index.js b/index.js index 482f1ecc0..e06ca8d47 100755 --- a/index.js +++ b/index.js @@ -6744,7 +6744,7 @@ F.console = function() { }; F.usagesnapshot = function(filename) { - Fs.writeFile(filename || F.path.root('usage.log'), JSON.stringify(F.usage(true), null, ' '), NOOP); + Fs.writeFile(filename || F.path.root('usage' + (F.id ? ('-' + F.id) : '') + '.log'), JSON.stringify(F.usage(true), null, ' '), NOOP); return F; }; @@ -16193,6 +16193,8 @@ process.on('message', function(msg, h) { msg.TYPE === 'req' && F.cluster.req(msg); msg.TYPE === 'res' && msg.target === F.id && F.cluster.res(msg); msg.TYPE === 'emit' && F.$events[msg.name] && F.emit(msg.name, msg.data); + msg.TYPE === 'nosql-meta' && NOSQL(msg.name).meta(msg.key, msg.value, true); + msg.TYPE === 'table-meta' && TABLE(msg.name).meta(msg.key, msg.value, true); } F.$events.message && F.emit('message', msg, h); }); diff --git a/nosql.js b/nosql.js index cb05377ec..4caedf375 100755 --- a/nosql.js +++ b/nosql.js @@ -71,6 +71,7 @@ const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' } const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' }; const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; const TABLERECORD = { '+': 1, '-': 1, '*': 1 }; +const CLUSTERMETA = {}; const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) { return a.removeDiacritics().localeCompare(b.removeDiacritics()); @@ -83,6 +84,33 @@ var JSONBUFFER = process.argv.findIndex(n => n.endsWith('nosqlworker.js')) === - var FORK; var FORKCALLBACKS; +function clusterlock(db, method) { + Fs.open(db.filenameLock, 'wx', function(err, fd) { + + if (err) { + setTimeout(clusterlock, 100, db, method); + return; + } + + Fs.write(fd, F.id.toString(), function(err) { + err && F.error('NoSQLStream.lock.write()', err); + Fs.close(fd, function(err) { + err && F.error('NoSQLStream.lock.close()', err); + db.locked = true; + db[method](); + }); + }); + }); + +} + +function clusterunlock(db) { + if (db.locked) { + db.locked = false; + Fs.unlink(db.filenameLock, NOOP); + } +} + function promise(fn) { var self = this; return new Promise(function(resolve, reject) { @@ -172,11 +200,11 @@ exports.worker = function() { break; case 'update': var obj = FORKCALLBACKS[msg.id]; - obj && obj.builder.$callback(msg.err, msg.response, msg.repository); + obj && obj.builder.$callback && obj.builder.$callback(msg.err, msg.response, msg.repository); break; case 'remove': var obj = FORKCALLBACKS[msg.id]; - obj && obj.builder.$callback(msg.err, msg.response, msg.repository); + obj && obj.builder.$callback && obj.builder.$callback(msg.err, msg.response, msg.repository); break; case 'backup': case 'restore': @@ -505,6 +533,7 @@ function Table(name, filename) { t.filenameCounter = filename + EXTENSION_TABLE + EXTENSION_COUNTER; t.filenameMeta = filename + EXTENSION_TABLE + '-meta'; t.directory = Path.dirname(filename); + t.filenameLock = t.filename + '-lock'; t.name = name; t.$name = '$' + name; t.pending_reader = []; @@ -576,6 +605,7 @@ function Database(name, filename, readonly) { self.directory = Path.dirname(filename); if (!readonly) { + self.filenameLock = self.filename + '-lock'; self.filenameCounter = self.readonly ? filename.format('counter', '-') : filename + EXTENSION + EXTENSION_COUNTER; self.filenameLog = self.readonly || readonly === true ? '' : filename + EXTENSION_LOG; self.filenameBackup = self.readonly || readonly === true ? '' : filename + EXTENSION_BACKUP; @@ -713,15 +743,30 @@ TP.set = DP.set = function(name, value) { return this.meta(name, value); }; -TP.meta = DP.meta = function(name, value) { +TP.meta = DP.meta = function(name, value, nosave) { var self = this; + if (value === undefined) return self.metadata ? self.metadata[name] : undefined; + if (!self.metadata) self.metadata = {}; + self.metadata[name] = value; clearTimeout(self.timeoutmeta); - self.timeoutmeta = setTimeout(() => self.$meta(true), 500); + + if (nosave) + self.timeoutmeta = setTimeout(() => self.$meta(true), 500); + + if (F.isCluster && !nosave) { + CLUSTERMETA.id = F.id; + CLUSTERMETA.TYPE = (self instanceof Table ? 'table' : 'nosql') + '-meta'; + CLUSTERMETA.name = self.name; + CLUSTERMETA.key = name; + CLUSTERMETA.value = value; + process.send(CLUSTERMETA); + } + return self; }; @@ -1169,10 +1214,15 @@ DP.next = function(type) { if (type && NEXTWAIT[this.step]) return; + if (F.isCluster && type === 0 && this.locked) + clusterunlock(this); + if (!this.$writting && !this.$reading) { if (this.step !== 12 && this.pending_clear.length) { - if (INMEMORY[this.name]) + if (!this.readonly && F.isCluster) + clusterlock(this, '$clear'); + else if (INMEMORY[this.name]) this.$clear_inmemory(); else this.$clear(); @@ -1180,7 +1230,10 @@ DP.next = function(type) { } if (this.step !== 13 && this.pending_clean.length) { - this.$clean(); + if (!this.readonly && F.isCluster) + clusterlock(this, '$clean'); + else + this.$clean(); return; } @@ -1206,7 +1259,9 @@ DP.next = function(type) { } if (this.step !== 2 && !this.$writting && this.pending_update.length) { - if (INMEMORY[this.name]) + if (!this.readonly && F.isCluster) + clusterlock(this, '$update'); + else if (INMEMORY[this.name]) this.$update_inmemory(); else this.$update(); @@ -1214,6 +1269,8 @@ DP.next = function(type) { } if (this.step !== 3 && !this.$writting && this.pending_remove.length) { + if (!this.readonly && F.isCluster) + clusterlock(this, '$remove'); if (INMEMORY[this.name]) this.$remove_inmemory(); else @@ -3385,6 +3442,7 @@ function Counter(db) { t.TIMEOUT = 30000; t.db = db; t.cache; + t.filenameLock = db.filenameCounter + '-lock'; t.key = (db instanceof Table ? 'table' : 'nosql') + db.name.hash(); t.type = 0; // 1 === saving, 2 === reading t.$events = {}; @@ -4313,6 +4371,15 @@ function counter_parse_days_all(output, value, year, opt) { } CP.save = function() { + var self = this; + if (F.isCluster) + clusterlock(self, '$save'); + else + self.$save(); + return self; +}; + +CP.$save = function() { var self = this; self.db.readonly && self.db.throwReadonly(); @@ -4322,7 +4389,7 @@ CP.save = function() { return self; } - var filename = self.db.filename + EXTENSION_COUNTER; + var filename = self.db.filenameCounter; var reader = Fs.createReadStream(filename); var writer = Fs.createWriteStream(filename + '-tmp'); var dt = NOW.format('MMdd') + '='; @@ -4428,6 +4495,7 @@ CP.save = function() { CLEANUP(writer, function() { Fs.rename(filename + '-tmp', filename, function() { + F.isCluster && clusterunlock(self); clearTimeout(self.timeout); self.timeout = 0; self.type = 0; @@ -5911,15 +5979,24 @@ TP.next = function(type) { if (!this.ready || (type && NEXTWAIT[this.step])) return; + if (F.isCluster && type === 0 && this.locked) + clusterunlock(this); + if (!this.$writting && !this.$reading) { if (this.step !== 12 && this.pending_clear.length) { - this.$clear(); + if (!this.readonly && F.isCluster) + clusterlock(this, '$clear'); + else + this.$clear(); return; } if (this.step !== 13 && this.pending_clean.length) { - this.$clean(); + if (!this.readonly && F.isCluster) + clusterlock(this, '$clean'); + else + this.$clean(); return; } @@ -5942,12 +6019,18 @@ TP.next = function(type) { } if (this.step !== 2 && !this.$writting && this.pending_update.length) { - this.$update(); + if (!this.readonly && F.isCluster) + clusterlock(this, '$update'); + else + this.$update(); return; } if (this.step !== 3 && !this.$writting && this.pending_remove.length) { - this.$remove(); + if (!this.readonly && F.isCluster) + clusterlock(this, '$remove'); + else + this.$remove(); return; } } From 9b85f549ee160d2d61184cc63e2cf156bf1d235c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 13:35:44 +0200 Subject: [PATCH 0529/1669] Fixed NoSQL Binary for cluster. --- nosql.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nosql.js b/nosql.js index 4caedf375..5b72b4cdf 100755 --- a/nosql.js +++ b/nosql.js @@ -4706,6 +4706,8 @@ Binary.prototype.insert = function(name, buffer, custom, callback) { var id; + F.isCluster && self.$refresh(); + if (self.meta.free.length) { id = self.meta.free.shift(); } else { @@ -4759,6 +4761,7 @@ Binary.prototype.insertstream = function(id, name, type, stream, callback, custo id = +id.substring(id.length - DIRECTORYLENGTH); } } else { + F.isCluster && self.$refresh(); isnew = true; if (self.meta.free.length) { id = self.meta.free.shift(); From 7786df077af00d1c266d05a100a9b3872bede845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 14:33:57 +0200 Subject: [PATCH 0530/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 70460531c..3a9507de0 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-89", + "version": "3.0.0-90", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 5c4600aed1b832d651d489313c548851c33c2cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 20 Jul 2018 16:53:32 +0200 Subject: [PATCH 0531/1669] Disabled NoSQL worker in cluster. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 5b72b4cdf..28477e07b 100755 --- a/nosql.js +++ b/nosql.js @@ -133,7 +133,7 @@ exports.pid = function() { exports.worker = function() { - if (FORK) + if (FORK || F.isCluster) return; // Clears unhandled callbacks From 09a4ca99b8ce9a94e7e813c3e3dc9be60ac861a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 21 Jul 2018 14:23:59 +0200 Subject: [PATCH 0532/1669] Improved code. --- index.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index e06ca8d47..299e6af2e 100755 --- a/index.js +++ b/index.js @@ -701,7 +701,7 @@ function Framework() { 'allow-performance': false, 'allow-custom-titles': false, 'allow-cache-snapshot': false, - 'allow-cache-cluster': true, + 'allow-cache-cluster': false, 'allow-debug': false, 'allow-head': false, 'allow-filter-errors': true, @@ -13302,17 +13302,12 @@ WebSocket.prototype.find = function(id) { for (var i = 0; i < length; i++) { var connection = self.connections[self._keys[i]]; - - if (!isFn) { - if (connection.id === id) + if (isFn) { + if (id(connection, connection.id)) return connection; - continue; - } - - if (id(connection, connection.id)) + } else if (connection.id === id) return connection; } - return null; }; From 02a3c027a67fd484aab900965b2dc8ed39585f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 22 Jul 2018 19:03:16 +0200 Subject: [PATCH 0533/1669] Added new improvements. --- index.js | 13 ++++++++++--- nosql.js | 9 ++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 299e6af2e..81f4e260b 100755 --- a/index.js +++ b/index.js @@ -5236,8 +5236,11 @@ F.$onParseXML = function(req) { * @return {Object} */ F.onParseJSON = function(value) { - if (value) - return JSON.parse(value); + if (value) { + try { + return JSON.parse(value); + } catch (e) {} + } }; F.onParseJSON.$def = true; @@ -13847,7 +13850,11 @@ WebSocketClient.prototype.$decode = function() { if (data instanceof Buffer) data = data.toString(ENCODING); F.config['default-websocket-encodedecode'] === true && (data = $decodeURIComponent(data)); - data.isJSON() && this.container.emit('message', this, F.onParseJSON(data, this.req)); + if (data.isJSON()) { + var tmp = F.onParseJSON(data, this.req); + if (tmp !== undefined) + this.container.emit('message', this, tmp); + } break; default: // TEXT diff --git a/nosql.js b/nosql.js index 28477e07b..b77c7c606 100755 --- a/nosql.js +++ b/nosql.js @@ -553,6 +553,7 @@ function Table(name, filename) { t.$free = true; t.$writting = false; t.$reading = false; + t.$allocations = true; t.counter = new Counter(t); t.$meta(); @@ -645,6 +646,7 @@ function Database(name, filename, readonly) { const TP = Table.prototype; const DP = Database.prototype; + TP.memory = DP.memory = function(count, size) { var self = this; count && (self.buffercount = count + 1); // def: 15 - count of stored documents in memory while reading/writing @@ -6950,6 +6952,11 @@ TP.$streamer = function() { return self; }; +TP.allocations = function(enable) { + this.$allocations = enable; + return this; +}; + TP.parseSchema = function() { var self = this; var arr = arguments[0] instanceof Array ? arguments[0] : arguments; @@ -7148,7 +7155,7 @@ TP.stringify = function(doc, insert, byteslen) { } else insert = true; - if (insert && size) + if (insert && size && self.$allocations) output += '|'.padRight(size, '.'); return (esc ? '*' : '+') + output; From 76def746dd932bd5f623eac021b197a48bb2d7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 22 Jul 2018 21:47:01 +0200 Subject: [PATCH 0534/1669] Added back `F.responseImage()` for backward compatibility. --- index.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/index.js b/index.js index 81f4e260b..051bf1157 100755 --- a/index.js +++ b/index.js @@ -6259,6 +6259,22 @@ F.touch = function(url) { return F; }; +// OBSOLETE +F.responseImage = function(req, res, filename, make, headers, done) { + OBSOLETE('F.responseImage()', 'This method is obsolete, use res.img()'); + + if (typeof(filename) === 'object') + res.options.stream = filename; + else + res.options.filename = filename; + + res.options.headers = headers; + res.options.callback = done; + res.options.make = make; + res.$image(); + return F; +}; + // OBSOLETE F.responseFile = function(req, res, filename, downloadName, headers, done, key) { OBSOLETE('F.responseFile()', 'This method is obsolete, use res.file()'); From 548c5cdc4eee925dbde8f837fa0c9eed19bae33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 22 Jul 2018 22:11:00 +0200 Subject: [PATCH 0535/1669] Updated `maxKeys` for query string parser. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index e2e8bcbc8..8a9ce21b8 100755 --- a/changes.txt +++ b/changes.txt @@ -127,6 +127,7 @@ - updated: `@{import()}` by adding `manifest` value linked to `/manifest.json` - updated: `F.use()` supports `function` instead of `middleware` name - updated: improved crypto algorithm +- updated: decreased a maximum count of keys to `33` from `69` when the query string is parsing - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index 051bf1157..d3f82bc19 100755 --- a/index.js +++ b/index.js @@ -68,7 +68,7 @@ const REG_SKIPERROR = /epipe|invalid\sdistance/i; const REG_UTF8 = /[^\x20-\x7E]+/; const FLAGS_INSTALL = ['get']; const FLAGS_DOWNLOAD = ['get', 'dnscache']; -const QUERYPARSEROPTIONS = { maxKeys: 69 }; +const QUERYPARSEROPTIONS = { maxKeys: 33 }; const EMPTYARRAY = []; const EMPTYOBJECT = {}; const EMPTYREQUEST = { uri: {} }; From 9861dba9188bb15c7a296194ca7003ebdfde04a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 08:03:51 +0200 Subject: [PATCH 0536/1669] Fixed `date` and `callback` in NoSQL. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index b77c7c606..e07ed4da3 100755 --- a/nosql.js +++ b/nosql.js @@ -163,7 +163,7 @@ exports.worker = function() { case 'insert': case 'update': case 'remove': - item.builder && item.builder.$callback(err, EMPTYOBJECT, EMPTYOBJECT); + item.builder && item.builder.$callback && item.builder.$callback(err, EMPTYOBJECT, EMPTYOBJECT); break; case 'clean': case 'clear': @@ -7107,7 +7107,7 @@ TP.stringify = function(doc, insert, byteslen) { break; case 4: // Date // val = val ? val.toISOString() : ''; - val = val ? val.getTime() : ''; + val = val ? (val instanceof Date ? val.getTime() : val.parseDate().getTime()) : ''; !val && (size += 13); break; case 5: // Object From 42b1f840d24d652cc624e97ef2119af7889d1455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 08:06:43 +0200 Subject: [PATCH 0537/1669] Fixed TABLE date serializer. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index e07ed4da3..30252b84d 100755 --- a/nosql.js +++ b/nosql.js @@ -7107,7 +7107,7 @@ TP.stringify = function(doc, insert, byteslen) { break; case 4: // Date // val = val ? val.toISOString() : ''; - val = val ? (val instanceof Date ? val.getTime() : val.parseDate().getTime()) : ''; + val = val ? val instanceof Date ? val.getTime() : val : ''; !val && (size += 13); break; case 5: // Object From 3ffb041f3b8c6e2621955a555ad93216c49cf954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 08:42:50 +0200 Subject: [PATCH 0538/1669] Fixed NoSQL worker and `modify/update.insert()` command. --- nosql.js | 13 +++++++++---- nosqlworker.js | 6 ++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 30252b84d..2b0801e51 100755 --- a/nosql.js +++ b/nosql.js @@ -646,7 +646,6 @@ function Database(name, filename, readonly) { const TP = Table.prototype; const DP = Database.prototype; - TP.memory = DP.memory = function(count, size) { var self = this; count && (self.buffercount = count + 1); // def: 15 - count of stored documents in memory while reading/writing @@ -1592,7 +1591,7 @@ DP.$update = function() { for (var i = 0; i < length; i++) { var item = filter[i]; if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert); + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); var tmp = self.insert(item.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; @@ -1679,7 +1678,7 @@ DP.$update_inmemory = function() { for (var i = 0; i < length; i++) { var item = filter[i]; if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert); + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); var tmp = self.insert(item.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; @@ -3190,10 +3189,14 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { }; DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function() { + var obj = {}; + obj.options = this.$options; obj.code = this.$code; obj.params = this.$params; + obj.insert = this.$insertcallback ? this.$insertcallback.toString() : null; + if (this.$functions) { obj.functions = []; for (var i = 0; i < this.$functions.length; i++) @@ -3207,6 +3210,7 @@ DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = fun }; DatabaseBuilder2.prototype.parse = DatabaseBuilder.prototype.parse = function(data) { + data = JSON.parse(data, jsonparser); this.$options = data.options; this.$code = data.code; @@ -3214,6 +3218,7 @@ DatabaseBuilder2.prototype.parse = DatabaseBuilder.prototype.parse = function(da this.$take = data.options.take; this.$skip = data.options.skip; this.$repository = data.repository; + this.$insertcallback = data.insert ? eval('(' + data.insert + ')') : null; if (data.functions) { for (var i = 0; i < data.functions.length; i++) @@ -6681,7 +6686,7 @@ TP.$update = function() { for (var i = 0; i < length; i++) { var item = filter[i]; if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert); + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); var tmp = self.insert(item.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; diff --git a/nosqlworker.js b/nosqlworker.js index 38e75af36..8ca8d67c2 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -42,10 +42,16 @@ const RESTORAGESTATS = { TYPE: 'storage.stats' }; const RESSTORAGECLEAR = { TYPE: 'storage.clear' }; const RESSTREAM = { TYPE: 'stream' }; +global.NOW = global.DATETIME = new Date(); + function killprocess() { process.exit(0); } +setInterval(function() { + global.NOW = global.DATETIME = new Date(); +}, 30000); + // One day cleaner setInterval(function() { var keys = Object.keys(F.databasescleaner); From eda1b019000096bc87146b3f43114106b54d3dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 16:00:28 +0200 Subject: [PATCH 0539/1669] Fixed meta. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 2b0801e51..965b4717e 100755 --- a/nosql.js +++ b/nosql.js @@ -756,7 +756,7 @@ TP.meta = DP.meta = function(name, value, nosave) { self.metadata[name] = value; clearTimeout(self.timeoutmeta); - if (nosave) + if (!nosave) self.timeoutmeta = setTimeout(() => self.$meta(true), 500); if (F.isCluster && !nosave) { From cb577a53343db786a4b4370438939c4759c966f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 17:51:52 +0200 Subject: [PATCH 0540/1669] Updated beta. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3a9507de0..04fce40e9 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-90", + "version": "3.0.0-91", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4742390081ce3a7f6e3939d73c3ea56114318b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 22:42:08 +0200 Subject: [PATCH 0541/1669] Improved `CORS()`. --- index.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/index.js b/index.js index d3f82bc19..bcfcb16e1 100755 --- a/index.js +++ b/index.js @@ -1616,6 +1616,11 @@ global.CORS = F.cors = function(url, flags, credentials) { return F; } + if (flags === true) { + credentials = true; + flags = null; + } + var route = {}; var origins = []; var methods = []; @@ -1667,6 +1672,9 @@ global.CORS = F.cors = function(url, flags, credentials) { } } + if (!methods.length) + methods = 'POST,PUT,GET,DELETE,PATCH,GET,HEAD'.split(','); + route.isWILDCARD = url.lastIndexOf('*') !== -1; var index = url.indexOf('{'); From 9397b61463657c095b15a9ed5b957ed186827230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 23 Jul 2018 23:10:46 +0200 Subject: [PATCH 0542/1669] Fixed inserting documents. --- nosql.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 965b4717e..2a921a6d3 100755 --- a/nosql.js +++ b/nosql.js @@ -331,6 +331,9 @@ exports.worker = function() { var self = this; var builder; + if (framework_builders.isSchema(doc)) + doc = doc.$clean(); + if (unique) { builder = self.one(); @@ -354,7 +357,7 @@ exports.worker = function() { return builder; } - return send(self, 'insert', framework_builders.isSchema(doc) ? doc.$clean() : doc).builder = new DatabaseBuilder2(self); + return send(self, 'insert', doc).builder = new DatabaseBuilder2(self); }; TP.count = DP.count = function() { @@ -473,7 +476,7 @@ exports.worker = function() { }; SP.insert = function(doc) { - notify(this.db, 'storage.insert', doc); + notify(this.db, 'storage.insert', framework_builders.isSchema(doc) ? doc.$clean() : doc); return this; }; @@ -5398,6 +5401,9 @@ SP.insert = function(doc) { return self; } + if (framework_builders.isSchema(doc)) + doc = doc.$clean(); + self.locked_reader = true; if (self.$mapreduce.length) { @@ -5731,6 +5737,9 @@ TP.insert = function(doc, unique) { var self = this; var builder; + if (framework_builders.isSchema(doc)) + doc = doc.$clean(); + self.readonly && self.throwReadonly(); if (unique) { From 3ec8d15ee4b5a1c3a4ede43fc9cdc21742d09c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 00:04:02 +0200 Subject: [PATCH 0543/1669] Improved `schema.required()` by adding `workflow` arg. --- builders.js | 12 +++++++----- index.js | 42 ++++++++++++++++++++++++++++++++++-------- utils.js | 2 +- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/builders.js b/builders.js index 289c9966e..d9c48501c 100755 --- a/builders.js +++ b/builders.js @@ -28,7 +28,7 @@ const REQUIRED = 'The field "@" is invalid.'; const DEFAULT_SCHEMA = 'default'; -const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true }; +const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true, $$workflow: true }; const REGEXP_CLEAN_EMAIL = /\s/g; const REGEXP_CLEAN_PHONE = /\s|\.|-|\(|\)/g; const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; @@ -1592,7 +1592,7 @@ SchemaBuilderEntity.prototype.default = function() { * @param [callback] * @returns {SchemaInstance} */ -SchemaBuilderEntity.prototype.make = function(model, filter, callback, argument, novalidate) { +SchemaBuilderEntity.prototype.make = function(model, filter, callback, argument, novalidate, workflow) { if (typeof(model) === 'function') { model.call(this, this); @@ -1607,6 +1607,9 @@ SchemaBuilderEntity.prototype.make = function(model, filter, callback, argument, var output = this.prepare(model); + if (workflow) + output.$$workflow = workflow; + if (novalidate) { callback && callback(null, output, argument); return output; @@ -1616,10 +1619,9 @@ SchemaBuilderEntity.prototype.make = function(model, filter, callback, argument, if (builder.hasError()) { this.onError && this.onError(builder, model, 'make'); callback && callback(builder, null, argument); - return output; - } + } else + callback && callback(null, output, argument); - callback && callback(null, output, argument); return output; }; diff --git a/index.js b/index.js index bcfcb16e1..bb773911f 100755 --- a/index.js +++ b/index.js @@ -347,8 +347,23 @@ global.$CREATE = function(schema) { global.$MAKE = function(schema, model, filter, callback, novalidate, argument) { schema = parseSchema(schema); + var o = framework_builders.getschema(schema[0], schema[1]); - return o ? o.make(model, filter, callback, argument, novalidate) : undefined; + var w = null; + + if (filter instanceof Array) { + w = {}; + for (var i = 0; i < filter.length; i++) + w[filter[i]] = i + 1; + filter = null; + } else if (filter instanceof Object) { + if (!(filter instanceof RegExp)) { + filter = null; + w = filter; + } + } + + return o ? o.make(model, filter, callback, argument, novalidate, w) : undefined; }; global.$QUERY = function(schema, options, callback, controller) { @@ -437,14 +452,17 @@ function performschema(type, schema, model, options, callback, controller) { return false; } - o.make(model, function(err, model) { + var workflow = {}; + workflow[type.substring(1)] = 1; + + o.make(model, null, function(err, model) { if (err) { callback && callback(err); } else { model.$$controller = controller; model[type](options, callback); } - }); + }, null, false, workflow); return !!o; } @@ -2327,6 +2345,15 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu console.warn('F.route() skips "binary" flag because the "raw" flag is not defined.'); } + if (workflow && workflow.id) { + workflow.meta = {}; + if (workflow.id instanceof Array) { + for (var i = 0; i < workflow.id.length; i++) + workflow.meta[workflow.id[i]] = i + 1; + } else + workflow.meta[workflow.id] = 1; + } + if (subdomain) F._length_subdomain_web++; @@ -2334,7 +2361,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var r = instance.route; r.hash = hash; r.id = id; - r.name = name; + r.name = name.trim(); r.groups = flags_to_object(groups); r.priority = priority; r.sitemap = sitemap ? sitemap.id : ''; @@ -5297,10 +5324,10 @@ F.$onParseQueryUrl = function(req) { * @param {String} name * @param {Function(err, body)} callback */ -F.onSchema = function(req, group, name, callback, filter, novalidate) { +F.onSchema = function(req, group, name, callback, filter, novalidate, workflow) { var schema = GETSCHEMA(group, name); if (schema) - schema.make(req.body, filter, onSchema_callback, callback, novalidate); + schema.make(req.body, filter, onSchema_callback, callback, novalidate, workflow ? workflow.meta : null); else callback(new Error('Schema "' + group + '/' + name + '" not found.')); }; @@ -14508,7 +14535,6 @@ function extend_request(PROTO) { return next(this, code); var self = this; - F.onSchema(this, this.$total_route.schema[0], this.$total_route.schema[1], function(err, body) { if (err) { @@ -14521,7 +14547,7 @@ function extend_request(PROTO) { next(self, code); } - }, route.schema[2], route.novalidate); + }, route.schema[2], route.novalidate, route.workflow); }; PROTO.$total_authorize = function(isLogged, user, roles) { diff --git a/utils.js b/utils.js index d5178e8dd..5cd64a2f8 100755 --- a/utils.js +++ b/utils.js @@ -2276,7 +2276,7 @@ exports.validate_builder = function(model, error, schema, collection, path, inde continue; var TYPE = collection[schema].schema[name]; - if (TYPE.can && !TYPE.can(model)) + if (TYPE.can && !TYPE.can(model, model.$$workflow || EMPTYOBJECT)) continue; var value = model[name]; From c66be71a4963d12495b708d0475262930d53937a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 00:07:03 +0200 Subject: [PATCH 0544/1669] Added new changes. --- changes.txt | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 8a9ce21b8..1694e8525 100755 --- a/changes.txt +++ b/changes.txt @@ -123,11 +123,14 @@ - updated: JavaScript compressor, now optimizes multiple `var` declarations - updated: `CORS()` without arguments for all routes, methods and origins - updated: `CORS()` tries to join multiple same preferences to one +- updated: `CORS(path)` without additional arguments allows all HTTP methods - updated: `U.keywords()` for Chinese/Japan characters - updated: `@{import()}` by adding `manifest` value linked to `/manifest.json` - updated: `F.use()` supports `function` instead of `middleware` name - updated: improved crypto algorithm - updated: decreased a maximum count of keys to `33` from `69` when the query string is parsing +- updated: extended `schema.required(name, (model, workflow) => workflow.update)`, more in docs. +- updated: `$MAKE(schema, model, [filter/workflows], ...)` supports `workflows` (array or object) instead of filter for `schema.required()` - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/package.json b/package.json index 04fce40e9..2efee3a42 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-91", + "version": "3.0.0-92", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 627ea8e99c1389928c8ebbdd8e39b038d8fce818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 12:30:59 +0200 Subject: [PATCH 0545/1669] Addded `CATCH()`. --- changes.txt | 1 + index.js | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/changes.txt b/changes.txt index 1694e8525..7d2ad0437 100755 --- a/changes.txt +++ b/changes.txt @@ -83,6 +83,7 @@ - added: `schema.define()(DEFAULT_VALUE)` added `DEFAULT_VALUE` - added: `TESTUSER([user])` for faking of `F.onAuthorize` delegate, targeted for unit-testing only - added: `G` as a global alias for `F.global` +- added: `CATCH([name])` is an improved `F.error()` without arguments - added: a simple support for `.heic` and `.heif` image format - added: `controller.sitemap_url2()` - added: `controller.sitemap_name2()` diff --git a/index.js b/index.js index bb773911f..a0835434e 100755 --- a/index.js +++ b/index.js @@ -537,6 +537,12 @@ global.CLEANUP = function(stream, callback) { }); }; +global.CATCH = function(name) { + return name == null ? F.error() : function(err) { + err && F.error(err, name); + }; +}; + global.SUCCESS = function(success, value) { if (typeof(success) === 'function') { From f18eed2503d8d91eb34e17504456d08365a2b2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 12:45:20 +0200 Subject: [PATCH 0546/1669] Renamed `CATCH()` to `ERROR()`. --- changes.txt | 2 +- index.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 7d2ad0437..60c60ceaa 100755 --- a/changes.txt +++ b/changes.txt @@ -83,7 +83,7 @@ - added: `schema.define()(DEFAULT_VALUE)` added `DEFAULT_VALUE` - added: `TESTUSER([user])` for faking of `F.onAuthorize` delegate, targeted for unit-testing only - added: `G` as a global alias for `F.global` -- added: `CATCH([name])` is an improved `F.error()` without arguments +- added: `ERROR([name])` is an improved `F.error()` without arguments - added: a simple support for `.heic` and `.heif` image format - added: `controller.sitemap_url2()` - added: `controller.sitemap_name2()` diff --git a/index.js b/index.js index a0835434e..12bc3f6b4 100755 --- a/index.js +++ b/index.js @@ -329,6 +329,12 @@ global.UID = function(type) { return UIDGENERATOR.date + index.padLeft(4, '0') + UIDGENERATOR.instance + (index % 2 ? 1 : 0); }; +global.ERROR = function(name) { + return name == null ? F.errorcallback : function(err) { + err && F.error(err, name); + }; +}; + global.AUTH = function(fn) { F.onAuthorize = fn; }; @@ -537,12 +543,6 @@ global.CLEANUP = function(stream, callback) { }); }; -global.CATCH = function(name) { - return name == null ? F.error() : function(err) { - err && F.error(err, name); - }; -}; - global.SUCCESS = function(success, value) { if (typeof(success) === 'function') { From 488e932b1543715627c084c328d5f4c1505b490e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 21:41:28 +0200 Subject: [PATCH 0547/1669] Added missing `code` and `message` when closing. --- index.js | 10 ++++++++-- websocketclient.js | 9 +++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 12bc3f6b4..cbd73573d 100755 --- a/index.js +++ b/index.js @@ -13752,6 +13752,12 @@ WebSocketClient.prototype.$ondata = function(data) { case 0x08: // close + this.closemessage = current.buffer.slice(4).toString('utf8'); + this.closecode = current.buffer[2] << 8 | current.buffer[3]; + + if (this.closemessage && F.config['default-websocket-encodedecode']) + this.closemessage = $decodeURIComponent(this.closemessage); + this.close(); break; @@ -13998,7 +14004,7 @@ WebSocketClient.prototype.$onclose = function() { this.container.$remove(this._id); this.container.$refresh(); - this.container.$events.close && this.container.emit('close', this); + this.container.$events.close && this.container.emit('close', this, this.closecode, this.closemessage); this.socket.removeAllListeners(); F.$events['websocket-end'] && F.emit('websocket-end', this.container, this); }; @@ -14080,7 +14086,7 @@ WebSocketClient.prototype.ping = function() { WebSocketClient.prototype.close = function(message, code) { if (!this.isClosed) { this.isClosed = true; - this.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + this.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); } return this; }; diff --git a/websocketclient.js b/websocketclient.js index 506ae66e7..5cbb89827 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -289,7 +289,12 @@ WebSocketClient.prototype.$ondata = function(data) { break; case 0x08: - // close + this.closemessage = current.buffer.slice(4).toString('utf8'); + this.closecode = current.buffer[2] << 8 | current.buffer[3]; + + if (this.closemessage && this.options.encodedecode) + this.closemessage = $decodeURIComponent(this.closemessage); + this.close(true); break; @@ -515,7 +520,7 @@ WebSocketClient.prototype.$onclose = function() { this.deflatechunks = null; } - this.$events.close && this.emit('close'); + this.$events.close && this.emit('close', this.closecode, this.closemessage); this.socket && this.socket.removeAllListeners(); }; From f02a5eaa812254db9233fda53e0515fb72b007fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 21:57:44 +0200 Subject: [PATCH 0548/1669] Updated NPM version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2efee3a42..15781a9b9 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-92", + "version": "3.0.0-93", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 058bfb81fd0940b98adb10ed9531fb1866daa3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 23:37:37 +0200 Subject: [PATCH 0549/1669] Improver `controller.proxy()`. --- index.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index cbd73573d..525c450c4 100755 --- a/index.js +++ b/index.js @@ -12494,7 +12494,7 @@ Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callbac flags.push(req.method); flags.push('dnscache'); - if (type === CT_JSON) + if ((/\/json/i).test(type)) flags.push('json'); var c = req.method[0]; @@ -12537,13 +12537,12 @@ Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callbac if (err) { callback && callback(err); self.invalid().push(err); - return; + } else { + self.status = code; + callback && callback(err, data, code, headers); + self.content(data, (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, '')); } - self.status = code; - callback && callback(err, data, code, headers); - self.content(data, (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, '')); - }, null, h, ENCODING, timeout || 10000); }; From 2a30953f15bb3f3cef7ff6c20a46ca0c6aacf512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 24 Jul 2018 23:39:54 +0200 Subject: [PATCH 0550/1669] Added `OBSOLETE()` for `res.proxy()`. --- index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.js b/index.js index 525c450c4..2588ed499 100755 --- a/index.js +++ b/index.js @@ -15105,6 +15105,8 @@ function extend_response(PROTO) { PROTO.proxy = function(url, headers, timeout, callback) { + OBSOLETE('res.proxy()', 'You need to use controller.proxy()'); + var res = this; if (res.success || res.headersSent) From dc880223d838a41e64e80ae6fd65e316cfdec68f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 25 Jul 2018 11:51:00 +0200 Subject: [PATCH 0551/1669] Fixed proxy host. --- utils.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 5cd64a2f8..ccd57e1f5 100755 --- a/utils.js +++ b/utils.js @@ -682,11 +682,11 @@ ProxyAgent.prototype.createSocket = function(options, callback) { PROXYOPTIONS.headers['Proxy-Authorization'] = proxy._auth; var req = self.request(PROXYOPTIONS); - - req.setTimeout(3000); + req.setTimeout(10000); req.on('response', proxyagent_response); req.on('connect', function(res, socket) { if (res.statusCode === 200) { + socket.$req = req; callback(socket); } else { var err = new Error('Proxy could not be established (maybe a problem in auth), code: ' + res.statusCode); @@ -743,6 +743,7 @@ function request_call(uri, options) { opt.path = uri.href; opt.headers = uri.headers; opt.method = uri.method; + opt.headers.host = uri.host; if (options.proxy._auth) opt.headers['Proxy-Authorization'] = options.proxy._auth; } else From 7595b68db651287b4331b3bf5eb093068febf5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 25 Jul 2018 11:58:10 +0200 Subject: [PATCH 0552/1669] Removed proxy for `localhost` and `127.0.0.1`. --- utils.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils.js b/utils.js index ccd57e1f5..623a2f388 100755 --- a/utils.js +++ b/utils.js @@ -631,6 +631,9 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) proxy = parseProxy(F.config['default-proxy']); + if (proxy && (uri.hostname === 'localhost' || uri.hostname === '127.0.0.1')) + proxy = null; + options.proxy = proxy; if (proxy && uri.protocol === 'https:') { From d3cf912edd715f1a9c1c34a4edec1646d8865eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 07:47:30 +0200 Subject: [PATCH 0553/1669] Fixed max. length of filename in NoSQL binary. --- nosql.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nosql.js b/nosql.js index 2a921a6d3..ae768110c 100755 --- a/nosql.js +++ b/nosql.js @@ -4665,6 +4665,9 @@ Binary.prototype.insert = function(name, buffer, custom, callback) { custom = null; } + if (name.length > 80) + name = name.substring(0, 80) + name.substring(name.lastIndexOf('.')); + if (!buffer) { var reader = Fs.createReadStream(name); CLEANUP(reader); @@ -4870,6 +4873,9 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { custom = null; } + if (name.length > 80) + name = name.substring(0, 80) + name.substring(name.lastIndexOf('.')); + if (!buffer) { if (isfn) { From 3de7428a1915abbdf693412e7111064bfe08dc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 09:38:42 +0200 Subject: [PATCH 0554/1669] Added support for joining `NOSQL` with `TABLE` and vice versa. --- nosql.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index ae768110c..caae110f6 100755 --- a/nosql.js +++ b/nosql.js @@ -2763,7 +2763,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { } var isTable = self.db instanceof Table; - var db = isTable ? TABLE(join.name) : NOSQL(join.name); + var db = isTable || join.table ? TABLE(join.name) : NOSQL(join.name); if (join.scalartype) { join.items = []; @@ -2886,6 +2886,18 @@ DatabaseBuilder.prototype.join = function(field, name) { if (!self.$join) self.$join = {}; + var table = self instanceof Table; + + if (name instanceof Database) + name = name.name; + else if (name instanceof Table) { + table = true; + name = name.name; + } else if (name[0] === '#') { + table = true; + name = name.substring(1); + } + var key = name + '.' + field; var join = self.$join[key]; if (join) @@ -2894,6 +2906,7 @@ DatabaseBuilder.prototype.join = function(field, name) { var item = self.$join[key] = {}; item.field = field; item.name = name; + item.table = table; item.builder = join = new DatabaseBuilder(self.db); join.on = function(a, b) { From 8a293fe45969cab9b0a336a8c1d35dfbd0a5e367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 09:42:15 +0200 Subject: [PATCH 0555/1669] Fixed joining. --- nosql.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index caae110f6..1dcb03bb9 100755 --- a/nosql.js +++ b/nosql.js @@ -2763,7 +2763,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { } var isTable = self.db instanceof Table; - var db = isTable || join.table ? TABLE(join.name) : NOSQL(join.name); + var db = join.table ? TABLE(join.name) : NOSQL(join.name); if (join.scalartype) { join.items = []; @@ -2888,14 +2888,12 @@ DatabaseBuilder.prototype.join = function(field, name) { var table = self instanceof Table; - if (name instanceof Database) + if (name instanceof Database) { name = name.name; - else if (name instanceof Table) { + table = false; + } else if (name instanceof Table) { table = true; name = name.name; - } else if (name[0] === '#') { - table = true; - name = name.substring(1); } var key = name + '.' + field; From e6a1997e7926efc087513feafdca1d0c4de4a955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 11:10:16 +0200 Subject: [PATCH 0556/1669] Added new features. --- changes.txt | 4 +++- index.js | 28 ++++++++++++++++++++++++++-- nosql.js | 16 ++++++++++++++-- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index 60c60ceaa..03473705c 100755 --- a/changes.txt +++ b/changes.txt @@ -102,7 +102,9 @@ - added: `HttpFile.fs(storage_name, [custom], [callback])` saves a file into the FileStorage - added: `HttpFile.nosql(db_name, [custom], [callback])` saves a file into the FileStorage - added: `res.filefs(storage_name, id, [download], [headers], [callback])` returns file from FileStorage -- added: `res.filenosql(db_name, id, [download], [headers], [callback])` returns file from FileStorage +- added: `res.filenosql(db_name, id, [download], [headers], [callback])` returns file from NoSQL binary +- added: `res.imagefs(storage_name, id, image_make_fn, [headers], [callback])` returns file from FileStorage +- added: `res.imagenosql(db_name, id, image_make_fn, [headers], [callback])` returns file from NoSQL binary - added: new stats `F.stats.performance` contains count of `request` and `file` per minute - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory diff --git a/index.js b/index.js index 2588ed499..a581b6bf5 100755 --- a/index.js +++ b/index.js @@ -15059,6 +15059,30 @@ function extend_response(PROTO) { return self; }; + PROTO.imagefs = function(name, id, make, headers, callback) { + var self = this; + var options = {}; + options.id = id; + options.image = true; + options.make = make; + options.headers = headers; + options.done = callback; + FILESTORAGE(name).res(self, options, $file_notmodified); + return self; + }; + + PROTO.imagenosql = function(name, id, make, headers, callback) { + var self = this; + var options = {}; + options.id = id; + options.image = true; + options.make = make; + options.headers = headers; + options.done = callback; + NOSQL(name).binary.res(self, options, $file_notmodified); + return self; + }; + /** * Responds with a stream * @param {String} contentType @@ -15355,7 +15379,7 @@ function extend_response(PROTO) { } // Is package? - if (options.filename[0] === '@') + if (options.filename && options.filename[0] === '@') options.filename = F.path.package(options.filename.substring(1)); var name = F.temporary.path[req.$key]; @@ -15441,7 +15465,7 @@ function extend_response(PROTO) { if (res.getHeader('Last-Modified')) delete headers['Last-Modified']; - else + else if (!headers['Last-Modified']) headers['Last-Modified'] = name[2]; headers.Etag = ETAG + F.config['etag-version']; diff --git a/nosql.js b/nosql.js index 1dcb03bb9..eac3caa32 100755 --- a/nosql.js +++ b/nosql.js @@ -5037,28 +5037,40 @@ Binary.prototype.res = function(res, options, notmodified) { filename = framework_utils.join(self.directory, id + self.ext); var stream = Fs.createReadStream(filename, BINARYREADMETA); + stream.on('error', () => res.throw404()); stream.on('data', function(buffer) { var json = buffer.toString('utf8').replace(REGCLEAN, ''); if (json) { var obj = JSON.parse(json, jsonparser); var utc = obj.date ? new Date(+obj.date.substring(0, 4), +obj.date.substring(4, 6), +obj.date.substring(6, 8)).toUTCString() : ''; + if (!options.download && res.req.headers['if-modified-since'] === utc) { res.extention = U.getExtension(obj.name); notmodified(res, utc); } else { + res.options.type = obj.type; res.options.stream = Fs.createReadStream(filename, BINARYREADDATA); + if (!options.download) { if (!options.headers) options.headers = {}; options.headers['Last-Modified'] = utc; } else res.options.download = options.download; + res.options.headers = options.headers; res.options.done = options.done; - res.options.compress = options.nocompress ? false : true; - res.$stream(); + + if (options.image) { + res.options.make = options.make; + res.options.cache = options.cache !== false; + res.$image(); + } else { + res.options.compress = options.nocompress ? false : true; + res.$stream(); + } } } else res.throw404(); From e2cd251765837c36428e83f161c73b0b49ebc9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 11:20:07 +0200 Subject: [PATCH 0557/1669] Fixed HTTP cache. --- index.js | 2 +- nosql.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index a581b6bf5..9a4afa9c5 100755 --- a/index.js +++ b/index.js @@ -15465,7 +15465,7 @@ function extend_response(PROTO) { if (res.getHeader('Last-Modified')) delete headers['Last-Modified']; - else if (!headers['Last-Modified']) + else if (!res.options.lastmodified) headers['Last-Modified'] = name[2]; headers.Etag = ETAG + F.config['etag-version']; diff --git a/nosql.js b/nosql.js index eac3caa32..17f679742 100755 --- a/nosql.js +++ b/nosql.js @@ -5052,6 +5052,7 @@ Binary.prototype.res = function(res, options, notmodified) { res.options.type = obj.type; res.options.stream = Fs.createReadStream(filename, BINARYREADDATA); + res.options.lastmodified = true; if (!options.download) { if (!options.headers) From 46f25cf8480c8fcf4da8b44585ad2d3b7cf87320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 15:13:53 +0200 Subject: [PATCH 0558/1669] Improved operations. --- builders.js | 55 +++++++++++++++++++++++++++++++++++++++++++++++++--- changes.txt | 2 ++ index.js | 5 +++++ package.json | 2 +- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/builders.js b/builders.js index d9c48501c..ecedd6515 100755 --- a/builders.js +++ b/builders.js @@ -4578,9 +4578,11 @@ global.NEWOPERATION = function(name, fn) { return this; }; -global.OPERATION = function(name, value, callback, param) { +global.OPERATION = function(name, value, callback, param, controller) { if (typeof(value) === 'function') { + controller = param; + param = callback; callback = value; value = EMPTYOBJECT; } @@ -4590,7 +4592,7 @@ global.OPERATION = function(name, value, callback, param) { if (fn) { if (fn.$newversion) { - var self = new OperationOptions(error, value, param); + var self = new OperationOptions(error, value, param, controller); if (callback && callback !== NOOP) { self.callback = function(value) { if (arguments.length > 1) { @@ -4626,12 +4628,59 @@ global.OPERATION = function(name, value, callback, param) { } }; -function OperationOptions(error, value, options) { +function OperationOptions(error, value, options, controller) { + + if (!controller && options instanceof global.Controller) { + controller = options; + options = EMPTYOBJECT; + } else if (options === undefined) + options = EMPTYOBJECT; + + this.controller = controller; this.model = this.value = value; this.error = error; this.options = options; } +OperationOptions.prototype = { + + get user() { + return this.controller ? this.controller.user : null; + }, + + get session() { + return this.controller ? this.controller.session : null; + }, + + get language() { + return (this.controller ? this.controller.language : '') || ''; + }, + + get ip() { + return this.controller ? this.controller.ip : null; + }, + + get id() { + return this.controller ? this.controller.id : null; + }, + + get params() { + return this.controller ? this.controller.params : null; + }, + + get files() { + return this.controller ? this.controller.files : null; + }, + + get body() { + return this.controller ? this.controller.body : null; + }, + + get query() { + return this.controller ? this.controller.query : null; + } +}; + OperationOptions.prototype.DB = function() { return F.database(this.error); }; diff --git a/changes.txt b/changes.txt index 03473705c..cc8a80044 100755 --- a/changes.txt +++ b/changes.txt @@ -106,6 +106,7 @@ - added: `res.imagefs(storage_name, id, image_make_fn, [headers], [callback])` returns file from FileStorage - added: `res.imagenosql(db_name, id, image_make_fn, [headers], [callback])` returns file from NoSQL binary - added: new stats `F.stats.performance` contains count of `request` and `file` per minute +- added: new method `controller.operation(name, value, [callback], [options])` for evaluating of operation - updated: (IMPORTANT) NoSQL binary divides files to independent directories for 1000 files per directory - updated: `GROUP()` by adding a new argument `url_prefix` @@ -134,6 +135,7 @@ - updated: decreased a maximum count of keys to `33` from `69` when the query string is parsing - updated: extended `schema.required(name, (model, workflow) => workflow.update)`, more in docs. - updated: `$MAKE(schema, model, [filter/workflows], ...)` supports `workflows` (array or object) instead of filter for `schema.required()` +- updated: `OPERATION()` by adding `controller` - fixed: mail attachments - fixed: mail `message.manually()` diff --git a/index.js b/index.js index 9a4afa9c5..3ff9b59bf 100755 --- a/index.js +++ b/index.js @@ -10100,6 +10100,11 @@ Controller.prototype.$operation = function(name, helper, callback) { return self; }; +Controller.prototype.operation = function(name, value, callback, options) { + OPERATION(name, value, callback, options, this); + return this; +}; + Controller.prototype.$operation2 = function(name, helper, callback) { if (callback == null && typeof(helper) === 'function') { diff --git a/package.json b/package.json index 15781a9b9..ed98c08c1 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-93", + "version": "3.0.0-94", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From fe66741724b75e75801aa84f594980434f23237d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 15:20:53 +0200 Subject: [PATCH 0559/1669] Improved code. --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index 3ff9b59bf..71547eae3 100755 --- a/index.js +++ b/index.js @@ -13110,6 +13110,11 @@ WebSocket.prototype = { } }; +WebSocket.prototype.operation = function(name, value, callback, options) { + OPERATION(name, value, callback, options, this); + return this; +}; + WebSocket.prototype.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { From 2df7a4ae7ec8e8e6ff7d664704ba365701cffb80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 26 Jul 2018 16:16:56 +0200 Subject: [PATCH 0560/1669] Updated unit-test. --- test/test-builders.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test-builders.js b/test/test-builders.js index a32b6e253..ff22c9faf 100755 --- a/test/test-builders.js +++ b/test/test-builders.js @@ -635,6 +635,25 @@ function test_Operations() { assert.ok(err.hasError('bug'), 'OPERATIONS: ErrorHandling 1'); assert.ok(response === undefined, 'OPERATIONS: ErrorHandling 2'); }); + + NEWOPERATION('testC', function($) { + assert.ok($.controller === EMPTYCONTROLLER, 'OPERATIONS: Controller 1'); + $.callback(true); + }); + + NEWOPERATION('testD', function($) { + assert.ok($.options.ok === 100, 'OPERATIONS: Custom options + controller'); + assert.ok($.controller === EMPTYCONTROLLER, 'OPERATIONS: Controller 2'); + $.callback(false); + }); + + OPERATION('testC', 1, function(err, response) { + assert.ok(response === true, 'OPERATIONS: controller 1 response'); + }, EMPTYCONTROLLER); + + OPERATION('testD', 2, function(err, response) { + assert.ok(response === false, 'OPERATIONS: controller 2 response'); + }, { ok: 100 }, EMPTYCONTROLLER); } test_PageBuilder(); From 43c1f7244cb3fee7a55cfaef07886506fc0ce4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Jul 2018 09:02:24 +0200 Subject: [PATCH 0561/1669] Updated code. --- index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.js b/index.js index 71547eae3..733e6267c 100755 --- a/index.js +++ b/index.js @@ -3457,6 +3457,7 @@ global.MODULE = F.module = function(name) { * @return {Framework} */ F.modify = function(fn) { + OBSOLETE('F.modify()', 'This method will be removed from in versions.'); if (!F.modificators) F.modificators = []; F.modificators.push(fn); @@ -15926,6 +15927,7 @@ function $file_nocompress(stream, next, res) { next(); framework_internal.destroyStream(stream); }); + response_end(res); } From d7017c5650c7087cd6e3ff8186b1ebaba66da753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Jul 2018 10:02:03 +0200 Subject: [PATCH 0562/1669] Fixed a critical bug with joining. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 17f679742..7a97df695 100755 --- a/nosql.js +++ b/nosql.js @@ -2886,7 +2886,7 @@ DatabaseBuilder.prototype.join = function(field, name) { if (!self.$join) self.$join = {}; - var table = self instanceof Table; + var table = self.db instanceof Table; if (name instanceof Database) { name = name.name; From 1f7571d7308a20481150d61bae0292904e30f006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Jul 2018 11:40:14 +0200 Subject: [PATCH 0563/1669] Improved code. --- index.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/index.js b/index.js index 733e6267c..6867c1c70 100755 --- a/index.js +++ b/index.js @@ -1286,11 +1286,7 @@ function scriptNow() { return new Date(); } -F.database = function(name) { - return F.nosql(name); -}; - -global.NOSQL = F.nosql = function(name) { +F.database = global.NOSQL = F.nosql = function(name) { var db = F.databases[name]; if (db) return db; From 65d37e60454d498819e874a8e07cce1c4d22a9ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Jul 2018 12:14:41 +0200 Subject: [PATCH 0564/1669] Added hidden property `DatabaseBuilder.$cmd` (must be array). --- nosql.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nosql.js b/nosql.js index 7a97df695..768791a92 100755 --- a/nosql.js +++ b/nosql.js @@ -2711,6 +2711,7 @@ function DatabaseBuilder(db) { this.$repository = {}; this.$counter = 0; this.$keys = []; + this.$cmd = null; } DatabaseBuilder.prototype.promise = promise; @@ -2901,6 +2902,8 @@ DatabaseBuilder.prototype.join = function(field, name) { if (join) return join; + self.$cmd && self.$cmd.push({ type: 'join', arg: arguments }); + var item = self.$join[key] = {}; item.field = field; item.name = name; @@ -2961,6 +2964,7 @@ DatabaseBuilder.prototype.filter = function(fn) { if (self.$scope) code = 'if(!$is){' + code + '}'; + self.$cmd && self.$cmd.push({ type: 'fitler', arg: arguments }); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; @@ -2983,6 +2987,7 @@ DatabaseBuilder.prototype.contains = function(name) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; + self.$cmd && self.$cmd.push({ type: 'contains', arg: arguments }); return self; }; @@ -2994,6 +2999,7 @@ DatabaseBuilder.prototype.empty = function(name) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; + self.$cmd && self.$cmd.push({ type: 'empty', arg: arguments }); return self; }; @@ -3041,6 +3047,7 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key, operator)); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'where', arg: arguments }); return self; }; @@ -3058,6 +3065,7 @@ DatabaseBuilder.prototype.query = function(code) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; + self.$cmd && self.$cmd.push({ type: 'query', arg: arguments }); return self; }; @@ -3079,6 +3087,7 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'month', arg: arguments }); return self; }; @@ -3100,6 +3109,7 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'day', arg: arguments }); return self; }; @@ -3122,6 +3132,7 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'year', arg: arguments }); return self; }; @@ -3157,6 +3168,7 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key)); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'search', arg: arguments }); return self; }; @@ -3169,6 +3181,7 @@ DatabaseBuilder.prototype.regexp = function(name, value) { self.$keys && self.$keys.push(name); self.$code.push(code.format(name, value.toString())); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'regexp', arg: arguments }); return self; }; @@ -3199,6 +3212,7 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { code = 'if(!$is){' + code + '}'; self.$code.push(code.format(name, key, count || 1)); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'fulltext', arg: arguments }); return self; }; @@ -3304,6 +3318,7 @@ DatabaseBuilder.prototype.random = function() { DatabaseBuilder.prototype.sort = function(name, desc) { this.$options.sort = { name: name, asc: desc ? false : true }; + self.$cmd && self.$cmd.push({ type: 'sort', arg: arguments }); return this; }; @@ -3313,6 +3328,7 @@ DatabaseBuilder.prototype.repository = function(key, value) { if (value === undefined) return this.$repository[key]; this.$repository[key] = value; + self.$cmd && self.$cmd.push({ type: 'repository', arg: arguments }); return this; }; @@ -3337,6 +3353,7 @@ DatabaseBuilder.prototype.in = function(name, value) { code = 'if(!$is){' + code + '}'; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'in', arg: arguments }); return self; }; @@ -3350,6 +3367,7 @@ DatabaseBuilder.prototype.notin = function(name, value) { code = 'if(!$is){' + code + '}'; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'notin', arg: arguments }); return self; }; @@ -3365,24 +3383,28 @@ DatabaseBuilder.prototype.between = function(name, a, b) { self.$params[keyb] = b; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'between', arg: arguments }); return self; }; DatabaseBuilder.prototype.or = function() { this.$code.push('$is=false;'); this.$scope = 1; + self.$cmd && self.$cmd.push({ type: 'or' }); return this; }; DatabaseBuilder.prototype.end = function() { this.$scope = 0; this.$code.push('if(!$is)return;'); + self.$cmd && self.$cmd.push({ type: 'end' }); return this; }; DatabaseBuilder.prototype.and = function() { this.$code.push('$is=false;'); this.$scope = 0; + self.$cmd && self.$cmd.push({ type: 'and' }); return this; }; @@ -3413,6 +3435,7 @@ DatabaseBuilder.prototype.fields = function() { } self.$keys && self.$keys.push(name); } + self.$cmd && self.$cmd.push({ type: 'fields', arg: arguments }); return self; }; @@ -3435,6 +3458,7 @@ DatabaseBuilder.prototype.code = function(code) { if (self.$keys) self.$keys = null; + self.$cmd && self.$cmd.push({ type: 'code', arg: arguments }); return self; }; From 7885095ff9ca97a22d9af1a549099f818295ac37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 27 Jul 2018 12:18:02 +0200 Subject: [PATCH 0565/1669] Fixed context. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index 768791a92..407168f36 100755 --- a/nosql.js +++ b/nosql.js @@ -3390,21 +3390,21 @@ DatabaseBuilder.prototype.between = function(name, a, b) { DatabaseBuilder.prototype.or = function() { this.$code.push('$is=false;'); this.$scope = 1; - self.$cmd && self.$cmd.push({ type: 'or' }); + this.$cmd && this.$cmd.push({ type: 'or' }); return this; }; DatabaseBuilder.prototype.end = function() { this.$scope = 0; this.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'end' }); + this.$cmd && this.$cmd.push({ type: 'end' }); return this; }; DatabaseBuilder.prototype.and = function() { this.$code.push('$is=false;'); this.$scope = 0; - self.$cmd && self.$cmd.push({ type: 'and' }); + this.$cmd && this.$cmd.push({ type: 'and' }); return this; }; From a1e8083f8f95d3b90718ea084ac859146ae0b99c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Jul 2018 00:21:21 +0200 Subject: [PATCH 0566/1669] Fixed critical bugs with filters. --- nosql.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index 407168f36..dec30178f 100755 --- a/nosql.js +++ b/nosql.js @@ -3317,19 +3317,21 @@ DatabaseBuilder.prototype.random = function() { }; DatabaseBuilder.prototype.sort = function(name, desc) { - this.$options.sort = { name: name, asc: desc ? false : true }; + var self = this; + self.$options.sort = { name: name, asc: desc ? false : true }; self.$cmd && self.$cmd.push({ type: 'sort', arg: arguments }); - return this; + return self; }; DatabaseBuilder.prototype.repository = function(key, value) { + var self = this; if (key === undefined) - return this.$repository; + return self.$repository; if (value === undefined) - return this.$repository[key]; - this.$repository[key] = value; + return self.$repository[key]; + self.$repository[key] = value; self.$cmd && self.$cmd.push({ type: 'repository', arg: arguments }); - return this; + return self; }; DatabaseBuilder.prototype.compile = function(noTrimmer) { From f371a32dfeddb1cf57b502439ba66100d65b90c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Jul 2018 10:44:20 +0200 Subject: [PATCH 0567/1669] New improvements. --- changes.txt | 2 ++ nosql.js | 72 +++++++++++++++++++++++++++++++++++------------------ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/changes.txt b/changes.txt index cc8a80044..346be8ae0 100755 --- a/changes.txt +++ b/changes.txt @@ -41,6 +41,8 @@ - added: `DatabaseBuilder.query(code)` can contain a raw JS condition in the form e.g. `doc.age > 18 && doc.age < 33` - added: `DatabaseBuilder.regexp(name, regexp)` RegExp search in strings - added: `DatabaseBuilder.fulltext(name, regexp, [weight])` full text search in strings, more info in docs. +- added: `DatabaseBuilder.hour(name, [compare], value)` creates a condition for hours +- added: `DatabaseBuilder.minute(name, [compare], value)` creates a condition for minutes - added: `Database.find2()` performs faster and reverse reading of documents (from end to begin of the file) - added: `Database.stream(fn, [repository], [callback(err, repository, count)])` for streaming documents - added: `Database.lock(callback(next))` locks all internal DB operations diff --git a/nosql.js b/nosql.js index dec30178f..947258c97 100755 --- a/nosql.js +++ b/nosql.js @@ -2711,7 +2711,6 @@ function DatabaseBuilder(db) { this.$repository = {}; this.$counter = 0; this.$keys = []; - this.$cmd = null; } DatabaseBuilder.prototype.promise = promise; @@ -2902,8 +2901,6 @@ DatabaseBuilder.prototype.join = function(field, name) { if (join) return join; - self.$cmd && self.$cmd.push({ type: 'join', arg: arguments }); - var item = self.$join[key] = {}; item.field = field; item.name = name; @@ -2964,7 +2961,6 @@ DatabaseBuilder.prototype.filter = function(fn) { if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$cmd && self.$cmd.push({ type: 'fitler', arg: arguments }); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; @@ -2987,7 +2983,6 @@ DatabaseBuilder.prototype.contains = function(name) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; - self.$cmd && self.$cmd.push({ type: 'contains', arg: arguments }); return self; }; @@ -2999,7 +2994,6 @@ DatabaseBuilder.prototype.empty = function(name) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; - self.$cmd && self.$cmd.push({ type: 'empty', arg: arguments }); return self; }; @@ -3047,7 +3041,6 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key, operator)); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'where', arg: arguments }); return self; }; @@ -3065,7 +3058,6 @@ DatabaseBuilder.prototype.query = function(code) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; - self.$cmd && self.$cmd.push({ type: 'query', arg: arguments }); return self; }; @@ -3087,7 +3079,6 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'month', arg: arguments }); return self; }; @@ -3109,7 +3100,6 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'day', arg: arguments }); return self; }; @@ -3132,7 +3122,48 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'year', arg: arguments }); + return self; +}; + +DatabaseBuilder.prototype.hour = function(name, operator, value) { + var self = this; + var key = 'dh' + (self.$counter++); + + if (value === undefined) { + value = operator; + operator = '='; + } + + self.$params[key] = value; + + var code = compare_datetype('hour', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + return self; +}; + +DatabaseBuilder.prototype.minute = function(name, operator, value) { + var self = this; + var key = 'dh' + (self.$counter++); + + if (value === undefined) { + value = operator; + operator = '='; + } + + self.$params[key] = value; + + var code = compare_datetype('minute', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -3168,7 +3199,6 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key)); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'search', arg: arguments }); return self; }; @@ -3181,7 +3211,6 @@ DatabaseBuilder.prototype.regexp = function(name, value) { self.$keys && self.$keys.push(name); self.$code.push(code.format(name, value.toString())); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'regexp', arg: arguments }); return self; }; @@ -3212,7 +3241,6 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { code = 'if(!$is){' + code + '}'; self.$code.push(code.format(name, key, count || 1)); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'fulltext', arg: arguments }); return self; }; @@ -3319,7 +3347,6 @@ DatabaseBuilder.prototype.random = function() { DatabaseBuilder.prototype.sort = function(name, desc) { var self = this; self.$options.sort = { name: name, asc: desc ? false : true }; - self.$cmd && self.$cmd.push({ type: 'sort', arg: arguments }); return self; }; @@ -3330,7 +3357,6 @@ DatabaseBuilder.prototype.repository = function(key, value) { if (value === undefined) return self.$repository[key]; self.$repository[key] = value; - self.$cmd && self.$cmd.push({ type: 'repository', arg: arguments }); return self; }; @@ -3355,7 +3381,6 @@ DatabaseBuilder.prototype.in = function(name, value) { code = 'if(!$is){' + code + '}'; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'in', arg: arguments }); return self; }; @@ -3369,7 +3394,6 @@ DatabaseBuilder.prototype.notin = function(name, value) { code = 'if(!$is){' + code + '}'; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'notin', arg: arguments }); return self; }; @@ -3385,28 +3409,24 @@ DatabaseBuilder.prototype.between = function(name, a, b) { self.$params[keyb] = b; self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); - self.$cmd && self.$cmd.push({ type: 'between', arg: arguments }); return self; }; DatabaseBuilder.prototype.or = function() { this.$code.push('$is=false;'); this.$scope = 1; - this.$cmd && this.$cmd.push({ type: 'or' }); return this; }; DatabaseBuilder.prototype.end = function() { this.$scope = 0; this.$code.push('if(!$is)return;'); - this.$cmd && this.$cmd.push({ type: 'end' }); return this; }; DatabaseBuilder.prototype.and = function() { this.$code.push('$is=false;'); this.$scope = 0; - this.$cmd && this.$cmd.push({ type: 'and' }); return this; }; @@ -3437,7 +3457,6 @@ DatabaseBuilder.prototype.fields = function() { } self.$keys && self.$keys.push(name); } - self.$cmd && self.$cmd.push({ type: 'fields', arg: arguments }); return self; }; @@ -3460,7 +3479,6 @@ DatabaseBuilder.prototype.code = function(code) { if (self.$keys) self.$keys = null; - self.$cmd && self.$cmd.push({ type: 'code', arg: arguments }); return self; }; @@ -7297,6 +7315,12 @@ function compare_datetype(type, key, arg, operator) { case 'year': type = 'getFullYear()'; break; + case 'hour': + type = 'getHour()'; + break; + case 'minute': + type = 'getMinute()'; + break; } return '$is=false;$tmp={0};if($tmp){if(!$tmp.getTime){$tmp=new Date($tmp);if(isNaN($tmp))$tmp=0;}if($tmp)$is=($tmp.{3}){2}{1};}'.format(key, arg, operator, type); From d769077d15074f3bad3d1847d6ed9846a537902e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 29 Jul 2018 20:17:01 +0200 Subject: [PATCH 0568/1669] Fixed declaration schema + operation in `ROUTE()`. --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 6867c1c70..835a6338a 100755 --- a/index.js +++ b/index.js @@ -1847,7 +1847,8 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.substring(index + 1).trim(); } - url = url.replace(/\s\*[a-z0-9].*?$/i, function(text) { + + url = url.replace(/(^|\s?)\*[a-z0-9].*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; From 843cb83986982e15dd60887a71c8f3b177f6fe32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Jul 2018 09:06:37 +0200 Subject: [PATCH 0569/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ed98c08c1..54d925082 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-94", + "version": "3.0.0-95", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 70fcf58399f8d159045f4bacd8da443d0e1bfdca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 30 Jul 2018 13:51:09 +0200 Subject: [PATCH 0570/1669] Improved `proxy`. --- index.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index 835a6338a..469ff5036 100755 --- a/index.js +++ b/index.js @@ -12500,19 +12500,15 @@ Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callbac if ((/\/json/i).test(type)) flags.push('json'); - var c = req.method[0]; var tmp; - var keys; - if (c === 'G' || c === 'H' || c === 'O') { - if (url.indexOf('?') === -1) { - tmp = Qs.stringify(self.query); - if (tmp) - url += '?' + tmp; - } + if (url.indexOf('?') === -1) { + tmp = Qs.stringify(self.query); + if (tmp) + url += '?' + tmp; } - keys = Object.keys(req.headers); + var keys = Object.keys(req.headers); for (var i = 0, length = keys.length; i < length; i++) { switch (keys[i]) { case 'x-forwarded-for': From e163ebccec8483a6e9e279d534dcf5559bcfe789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 1 Aug 2018 13:11:24 +0200 Subject: [PATCH 0571/1669] Updated version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 54d925082..2c6a9a5c8 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0-95", + "version": "3.0.0", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0fa9d26985b2d802e386721d9acd2dfa91e31814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 1 Aug 2018 13:20:07 +0200 Subject: [PATCH 0572/1669] Fixed spaces --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 42b941df1..469ff5036 100755 --- a/index.js +++ b/index.js @@ -626,7 +626,7 @@ var PERF = {}; function Framework() { this.$id = null; // F.id ==> property - this.version = 3000; + this.version = 3000; this.version_header = '3.0.0'; this.version_node = process.version.toString(); this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); From 98eb26664c74e60e8f2f2e6ebd16b1717549f9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 1 Aug 2018 13:23:00 +0200 Subject: [PATCH 0573/1669] Updated URL. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2c6a9a5c8..98e5cb010 100755 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "author": { "name": "Peter Sirka", "email": "petersirka@gmail.com", - "url": "http://www.petersirka.eu" + "url": "http://www.petersirka.com" }, "bin": { "total": "./bin/totaljs", From f2de77d8f492578c38303752430e0a7bb2d08b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 11:22:37 +0200 Subject: [PATCH 0574/1669] Fixed `nosql.update()` and `nosql.modify()`. --- changes.txt | 4 ++++ nosql.js | 10 ++++++++-- nosqlworker.js | 8 ++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 346be8ae0..a02a80215 100755 --- a/changes.txt +++ b/changes.txt @@ -1,3 +1,7 @@ +======= 3.0.1 + +- fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function + ======= 3.0.0 - added: (IMPORTANT) bundles diff --git a/nosql.js b/nosql.js index 947258c97..f99c895bb 100755 --- a/nosql.js +++ b/nosql.js @@ -370,11 +370,17 @@ exports.worker = function() { }; TP.update = DP.update = function(doc, insert) { - return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + var val = framework_builders.isSchema(doc) ? doc.$clean() : doc; + if (typeof(val) === 'function') + val = val.toString(); + return send(this, 'update', val, insert).builder = new DatabaseBuilder(this); }; TP.modify = DP.modify = function(doc, insert) { - return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + var val = framework_builders.isSchema(doc) ? doc.$clean() : doc; + if (typeof(val) === 'function') + val = val.toString(); + return send(this, 'modify', val, insert).builder = new DatabaseBuilder(this); }; DP.restore = function(filename, callback) { diff --git a/nosqlworker.js b/nosqlworker.js index 8ca8d67c2..1789cd98d 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -122,6 +122,10 @@ process.on('message', function(msg) { }); break; case 'update': + + if (typeof(msg.arg[0]) === 'string') + msg.arg[0] = eval('(' + msg.arg[0] + ')'); + db.update(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, repository) { RESUPDATE.err = err; RESUPDATE.response = response; @@ -131,6 +135,10 @@ process.on('message', function(msg) { }); break; case 'modify': + + if (typeof(msg.arg[0]) === 'string') + msg.arg[0] = eval('(' + msg.arg[0] + ')'); + db.modify(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, repository) { RESUPDATE.err = err; RESUPDATE.response = response; From f10d3358923ca48c4b9d67ac3ed937b9c48f5c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 11:38:41 +0200 Subject: [PATCH 0575/1669] Added default values for CSS variables. --- changes.txt | 1 + internal.js | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index a02a80215..e074db6e8 100755 --- a/changes.txt +++ b/changes.txt @@ -1,5 +1,6 @@ ======= 3.0.1 +- added: CSS variables supports default values `border-radius: $radius || 10px` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function ======= 3.0.0 diff --git a/internal.js b/internal.js index 7c30acde2..4762f30f8 100755 --- a/internal.js +++ b/internal.js @@ -76,7 +76,7 @@ const REG_CSS_7 = /\s\}/g; const REG_CSS_8 = /\s\{/g; const REG_CSS_9 = /;\}/g; const REG_CSS_10 = /\$[a-z0-9-_]+:.*?;/gi; -const REG_CSS_11 = /\$[a-z0-9-_]+/gi; +const REG_CSS_11 = /\$.*?(;|\})/gi; const REG_CSS_12 = /(margin|padding):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; @@ -2613,8 +2613,23 @@ function variablesCSS(content) { }); content = content.replace(REG_CSS_11, function(text) { - var variable = variables[text]; - return variable ? variable : text; + + var index = text.indexOf('||'); + var variable = ''; + var last = text[text.length - 1]; + var len = text.length; + + if (last === ';' || last === '}') + len = len - 1; + else + last = ''; + + if (index !== -1) + variable = variables[text.substring(0, index).trim()] || text.substring(index + 2, len).trim(); + else + variable = variables[text.substring(0, len).trim()]; + + return variable ? (variable + last) : text; }).trim(); return content; From 6003acf92de276ab544b0e0105364d8883286295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 17:57:27 +0200 Subject: [PATCH 0576/1669] Fixed typo. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index e074db6e8..da750095b 100755 --- a/changes.txt +++ b/changes.txt @@ -1,6 +1,6 @@ ======= 3.0.1 -- added: CSS variables supports default values `border-radius: $radius || 10px` +- added: CSS variables support default values `border-radius: $radius || 10px` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function ======= 3.0.0 From 52b146dd54f83493539a916cde5200115e8478a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 19:13:39 +0200 Subject: [PATCH 0577/1669] Fixed `F.wait()` in test mode. --- changes.txt | 2 ++ test.js | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index da750095b..a6f79c2a1 100755 --- a/changes.txt +++ b/changes.txt @@ -1,7 +1,9 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` + - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function +- fixed: `F.wait()` in the test mode ======= 3.0.0 diff --git a/test.js b/test.js index 55d4a2b3b..58e61b814 100644 --- a/test.js +++ b/test.js @@ -181,13 +181,17 @@ exports.load = function() { T.current = null; next(); }, function() { - T.tests.quicksort('priority'); - F.emit('test-begin', T); - console.log('===================== TESTING ======================'); - console.log(''); - T.running = true; - T.start = Date.now(); - NEXT(); + U.wait(function() { + return F._length_wait === 0; + }, function() { + T.tests.quicksort('priority'); + F.emit('test-begin', T); + console.log('===================== TESTING ======================'); + console.log(''); + T.running = true; + T.start = Date.now(); + NEXT(); + }); }); }); }; From 6c85f59a9b337f06be09c429ef40b18e1d66c1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 11 Aug 2018 10:41:16 +0200 Subject: [PATCH 0578/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 98e5cb010..06d9619b8 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0", + "version": "3.0.1-1", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From a2d7286bb96cb626d79cfdd112eed3c650075e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 15 Aug 2018 10:54:31 +0200 Subject: [PATCH 0579/1669] Fixed `LOCALIZE()`. --- index.js | 26 ++++++++++++++++---------- package.json | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index 469ff5036..2c666b859 100755 --- a/index.js +++ b/index.js @@ -21,7 +21,7 @@ /** * @module Framework - * @version 3.0.0 + * @version 3.0.1 */ 'use strict'; @@ -626,8 +626,8 @@ var PERF = {}; function Framework() { this.$id = null; // F.id ==> property - this.version = 3000; - this.version_header = '3.0.0'; + this.version = 3001; + this.version_header = '3.0.1'; this.version_node = process.version.toString(); this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); @@ -3223,7 +3223,7 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { if (url[0] === '#') url = sitemapurl(url.substring(1)); - url = url.replace('*', ''); + url = url.replace('*.*', ''); if (minify == null) minify = true; @@ -3235,24 +3235,30 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { flags = []; var index; + var ext = false; flags = flags.remove(function(item) { item = item.toLowerCase(); if (item === 'nocompress') minify = false; + if (item[0] === '.') + ext = true; return item === 'compress' || item === 'nocompress' || item === 'minify'; }); var index = url.lastIndexOf('.'); - if (index === -1) - flags.push('.html', '.htm', '.md', '.txt'); - else { - flags.push(url.substring(index).toLowerCase()); - url = url.substring(0, index); + if (!ext) { + if (index === -1) + flags.push('.html', '.htm', '.md', '.txt'); + else { + flags.push(url.substring(index).toLowerCase()); + url = url.substring(0, index).replace('*', ''); + } } - url = framework_internal.preparePath(url); + url = framework_internal.preparePath(url.replace('.*', '')); + F.file(url, function(req, res) { F.onLocale && (req.$language = F.onLocale(req, res, req.isStaticFile)); diff --git a/package.json b/package.json index 06d9619b8..e819243bd 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-1", + "version": "3.0.1-2", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From cc2d5a28a5bf2ca4636fa7f5df728b334f9fadfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 15 Aug 2018 10:55:45 +0200 Subject: [PATCH 0580/1669] Added new fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index a6f79c2a1..c74d4d61e 100755 --- a/changes.txt +++ b/changes.txt @@ -4,6 +4,7 @@ - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode +- fixed: `LOCALIZE()` for nested directories ======= 3.0.0 From 129b5f6a8f8757bbf18b5fd02c4b13e5b41cc7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 16 Aug 2018 08:23:29 +0200 Subject: [PATCH 0581/1669] Added a new test script. --- test/sql.js | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 test/sql.js diff --git a/test/sql.js b/test/sql.js new file mode 100644 index 000000000..05fd36741 --- /dev/null +++ b/test/sql.js @@ -0,0 +1,192 @@ +require('../index'); + +function SQL(query) { + + var self = this; + self.options = {}; + self.builder = new framework_nosql.DatabaseBuilder(null); + self.query = query; + + var type = self.parseType(); + switch (type) { + case 'select': + self.parseLimit(); + self.parseOrder(); + self.parseWhere(); + self.parseJoins(); + self.parseTable(); + self.parseNames(); + break; + case 'update': + self.parseWhere(); + self.parseTable(); + self.parseUpdate(); + break; + case 'insert': + self.parseWhere(); + self.parseInsert(); + break; + case 'delete': + self.parseWhere(); + self.parseTable(); + break; + } + + console.log(self.options); +} + +var SQLP = SQL.prototype; + +SQLP.parseLimit = function() { + var self = this; + var tmp = self.query.match(/take \d+ skip \d+$/i); + !tmp && (tmp = self.query.match(/skip \d+ take \d+$/i)); + !tmp && (tmp = self.query.match(/take \d+$/i)); + !tmp && (tmp = self.query.match(/skip \d+$/i)); + if (!tmp) + return self; + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().toLowerCase().split(' '); + if (arr[0] === 'take') + self.options.take = +arr[1]; + else if (arr[2] === 'take') + self.options.take = +arr[3]; + if (arr[0] === 'skip') + self.options.skip = +arr[1]; + else if (arr[2] === 'skip') + self.options.skip = +arr[3]; + return self; +}; + +SQLP.parseOrder = function() { + var self = this; + var tmp = self.query.match(/order by .*?$/i); + + if (!tmp) + return self; + + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().substring(9).split(','); + + self.options.sort = []; + + for (var i = 0; i < arr.length; i++) { + tmp = arr[i].trim().split(' '); + self.options.sort.push({ name: tmp[0], desc: (tmp[1] || '').toLowerCase() === 'desc' }); + } + + return self; +}; + +SQLP.parseWhere = function() { + var self = this; + var tmp = self.query.match(/where .*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + + tmp = tmp[0].toString().substring(6).replace(/\sAND\s/gi, ' && ').replace(/\sOR\s/gi, ' || ').replace(/[a-z0-9]=/gi, function(text) { + return text + '='; + }); + + self.options.where = tmp; + return self; +}; + +SQLP.parseJoins = function() { + var self = this; + var tmp = self.query.match(/left join.*?$/i); + if (!tmp) { + tmp = self.query.match(/join.*?$/i); + if (!tmp) + return self; + } + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().trim(); + // console.log(tmp); + return self; +}; + +SQLP.parseTable = function() { + var self = this; + var tmp = self.query.match(/from\s.*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(5).trim(); + + var arr = tmp.split(' '); + // console.log(arr); + + return self; +}; + +SQLP.parseNames = function() { + var self = this; + var tmp = self.query.match(/select\s.*?$/i); + if (!tmp) + return self; + + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(6).trim().split(','); + + self.options.fields = []; + + for (var i = 0; i < tmp.length; i++) { + var field = tmp[i].trim(); + var alias = field.match(/as\s.*?$/); + var name = ''; + var type = 0; + + if (alias) { + field = field.replace(alias, ''); + alias = alias.toString().substring(3); + } + + var index = field.indexOf('('); + if (index !== -1) { + switch (field.substring(0, index).toLowerCase()) { + case 'count': + type = 1; + break; + case 'min': + type = 2; + break; + case 'max': + type = 3; + break; + case 'avg': + type = 4; + break; + case 'sum': + type = 5; + break; + case 'distinct': + type = 6; + break; + } + name = field.substring(index + 1, field.lastIndexOf(')')); + } else + name = field; + self.options.fields.push({ alias: alias || name, name: name, type: type }); + } + + return self; +}; + +SQLP.parseUpdate = function() { + var self = this; + return self; +}; + +SQLP.parseInsert = function() { + var self = this; + return self; +}; + +SQLP.parseType = function() { + var self = this; + return self.query.substring(0, self.query.indexOf(' ')).toLowerCase(); +}; + +var sql = new SQL('SELECT COUNT(*) as count, id FROM table a JOIN users ON id=id WHERE a.name="Peter" AND a.age=30 ORDER BY a.name, a.age ASC TAKE 20 SKIP 10'); \ No newline at end of file From d90b8c8620f20c9afc85c552c25619fb542a03b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 20 Aug 2018 08:45:57 +0200 Subject: [PATCH 0582/1669] Added a default `agent` for the requests. --- utils.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.js b/utils.js index 623a2f388..cd566f4c6 100755 --- a/utils.js +++ b/utils.js @@ -879,7 +879,7 @@ function request_response(res, uri, options) { var tmp = Url.parse(loc); tmp.headers = uri.headers; - tmp.agent = false; + // tmp.agent = false; tmp.method = uri.method; res.req.removeAllListeners(); @@ -1147,7 +1147,7 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi var uri = Url.parse(url); uri.method = method; - uri.agent = false; + // uri.agent = false; uri.headers = headers; options.uri = uri; @@ -1257,7 +1257,7 @@ function download_response(res, uri, options) { var tmp = Url.parse(res.headers['location']); tmp.headers = uri.headers; - tmp.agent = false; + // tmp.agent = false; tmp.method = uri.method; res.req.removeAllListeners(); res.req = null; From 0687ceca7cf006e6d749500796ec2eba2dc95f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 21 Aug 2018 12:34:44 +0200 Subject: [PATCH 0583/1669] Added SQL parser. --- test/test-tmp.js | 199 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/test/test-tmp.js b/test/test-tmp.js index 25f47af6a..b03fe94bc 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,2 +1,201 @@ require('../index'); +function SQL(query) { + + var self = this; + self.options = {}; + self.builder = new framework_nosql.DatabaseBuilder(null); + self.query = query; + + var type = self.parseType(); + + switch (type) { + case 'select': + self.parseLimit(); + self.parseOrder(); + self.parseWhere(); + self.parseJoins(); + self.parseTable(); + self.parseNames(); + break; + case 'update': + self.parseWhere(); + self.parseTable(); + self.parseUpdate(); + break; + case 'insert': + self.parseWhere(); + self.parseInsert(); + break; + case 'delete': + self.parseWhere(); + self.parseTable(); + break; + } + + console.log(self.options); +} + +var SQLP = SQL.prototype; + +SQLP.parseLimit = function() { + var self = this; + var tmp = self.query.match(/take \d+ skip \d+$/i); + !tmp && (tmp = self.query.match(/skip \d+ take \d+$/i)); + !tmp && (tmp = self.query.match(/take \d+$/i)); + !tmp && (tmp = self.query.match(/skip \d+$/i)); + if (!tmp) + return self; + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().toLowerCase().split(' '); + if (arr[0] === 'take') + self.options.take = +arr[1]; + else if (arr[2] === 'take') + self.options.take = +arr[3]; + if (arr[0] === 'skip') + self.options.skip = +arr[1]; + else if (arr[2] === 'skip') + self.options.skip = +arr[3]; + return self; +}; + +SQLP.parseOrder = function() { + var self = this; + var tmp = self.query.match(/order by .*?$/i); + + if (!tmp) + return self; + + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().substring(9).split(','); + + self.options.sort = []; + + for (var i = 0; i < arr.length; i++) { + tmp = arr[i].trim().split(' '); + self.options.sort.push({ name: tmp[0], desc: (tmp[1] || '').toLowerCase() === 'desc' }); + } + + return self; +}; + +SQLP.parseWhere = function() { + var self = this; + var tmp = self.query.match(/where .*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + + tmp = tmp[0].toString().substring(6).replace(/\sAND\s/gi, ' && ').replace(/\sOR\s/gi, ' || ').replace(/[a-z0-9]=/gi, function(text) { + return text + '='; + }); + + self.options.where = tmp; + return self; +}; + +SQLP.parseJoins = function() { + var self = this; + var tmp = self.query.match(/left join.*?$/i); + if (!tmp) { + tmp = self.query.match(/join.*?$/i); + if (!tmp) + return self; + } + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().trim(); + + self.options.joins = []; + + tmp = tmp.substring(5).split(/\s?JOIN\s/i); + for (var i = 0; i < tmp.length; i++) { + var join = tmp[i].split(/\son\s/i); + self.options.joins.push({ table: join[0], condition: join[1] }); + } + + return self; +}; + +SQLP.parseTable = function() { + var self = this; + var tmp = self.query.match(/from\s.*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(5).trim(); + + var arr = tmp.split(' '); + // console.log(arr); + + return self; +}; + +SQLP.parseNames = function() { + var self = this; + var tmp = self.query.match(/select\s.*?$/i); + if (!tmp) + return self; + + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(6).trim().split(','); + + self.options.fields = []; + + for (var i = 0; i < tmp.length; i++) { + var field = tmp[i].trim(); + var alias = field.match(/as\s.*?$/); + var name = ''; + var type = 0; + + if (alias) { + field = field.replace(alias, ''); + alias = alias.toString().substring(3); + } + + var index = field.indexOf('('); + if (index !== -1) { + switch (field.substring(0, index).toLowerCase()) { + case 'count': + type = 1; + break; + case 'min': + type = 2; + break; + case 'max': + type = 3; + break; + case 'avg': + type = 4; + break; + case 'sum': + type = 5; + break; + case 'distinct': + type = 6; + break; + } + name = field.substring(index + 1, field.lastIndexOf(')')); + } else + name = field; + self.options.fields.push({ alias: alias || name, name: name, type: type }); + } + + return self; +}; + +SQLP.parseUpdate = function() { + var self = this; + return self; +}; + +SQLP.parseInsert = function() { + var self = this; + return self; +}; + +SQLP.parseType = function() { + var self = this; + return self.query.substring(0, self.query.indexOf(' ')).toLowerCase(); +}; + +var sql = new SQL('SELECT COUNT(*) as count, id FROM table a JOIN users ON users.id=a.id JOIN orders ON orders.id=a.id WHERE a.name="Peter" AND a.age=30 ORDER BY a.name, a.age ASC TAKE 20 SKIP 10'); \ No newline at end of file From 95178316c224e74e1acb6aa7f4c4271e741a80cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 21 Aug 2018 15:07:20 +0200 Subject: [PATCH 0584/1669] Fixed storing files. --- changes.txt | 1 + nosql.js | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index c74d4d61e..813b5e95f 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: CSS variables support default values `border-radius: $radius || 10px` +- fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories diff --git a/nosql.js b/nosql.js index f99c895bb..dbd521169 100755 --- a/nosql.js +++ b/nosql.js @@ -4738,7 +4738,7 @@ Binary.prototype.insert = function(name, buffer, custom, callback) { if (typeof(buffer) === 'string') buffer = framework_utils.createBuffer(buffer, 'base64'); else if (buffer.resume) - return self.insertstream(null, name, type, buffer, callback); + return self.insertstream(null, name, type, buffer, callback, custom); var size = buffer.length; var dimension; @@ -4898,7 +4898,7 @@ Binary.prototype.insertstream = function(id, name, type, stream, callback, custo h.size = writer.bytesWritten; - Fs.open(filepath, 'a', function(err, fd) { + Fs.open(filepath, 'r+', function(err, fd) { if (!err) { var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); @@ -4953,7 +4953,7 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { buffer = framework_utils.createBuffer(buffer, 'base64'); if (buffer.resume) - return self.insertstream(id, name, type, buffer, callback); + return self.insertstream(id, name, type, buffer, callback, custom); var isnew = false; var time = NOW.format('yyyyMMdd'); From 9ee0150f045efd2ae95c85b2cfeba8ae6ea81db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 16:35:30 +0200 Subject: [PATCH 0585/1669] Improved NoSQL filtering. --- nosql.js | 1951 +++++++++++++++--------------------------------------- 1 file changed, 539 insertions(+), 1412 deletions(-) diff --git a/nosql.js b/nosql.js index dbd521169..9a8c670ad 100755 --- a/nosql.js +++ b/nosql.js @@ -872,7 +872,8 @@ DP.update = function(doc, insert) { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; - self.pending_update.push({ builder: builder, doc: data, count: 0, insert: insert === true ? data : insert }); + builder.$options.readertype = 1; + self.pending_update.push({ builder: builder, doc: data, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); return builder; }; @@ -885,6 +886,8 @@ DP.modify = function(doc, insert) { var keys = Object.keys(data); var inc = null; + builder.$options.readertype = 1; + if (keys.length) { for (var i = 0; i < keys.length; i++) { var key = keys[i]; @@ -902,7 +905,7 @@ DP.modify = function(doc, insert) { break; } } - self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, inc: inc, insert: insert === true ? data : insert }); + self.pending_update.push({ builder: builder, doc: data, keys: keys, inc: inc, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); } @@ -1095,16 +1098,11 @@ TP.lock = DP.lock = function(callback) { return self; }; -DP.remove = function(filename) { +DP.remove = function() { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - var backup = filename === undefined ? undefined : filename || self.filenameBackup2; - - if (backup) - backup = new Backuper(backup); - - self.pending_remove.push({ builder: builder, count: 0, backup: backup }); + self.pending_remove.push(builder); setImmediate(next_operation, self, 3); return builder; }; @@ -1117,7 +1115,7 @@ DP.listing = function(builder) { builder = new DatabaseBuilder(self); builder.$options.listing = true; builder.$take = builder.$options.take = 100; - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1128,7 +1126,7 @@ DP.find = function(builder) { builder.db = self; else builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1141,7 +1139,7 @@ DP.find2 = function(builder) { builder = new DatabaseBuilder(self); if (self.readonly) return self.find(builder); - self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -1170,7 +1168,8 @@ DP.scalar = function(type, field) { DP.count = function() { var self = this; var builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, type: 1 }); + builder.$options.readertype = 1; + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1179,7 +1178,7 @@ DP.one = function() { var self = this; var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader.push({ builder: builder, count: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1188,7 +1187,7 @@ DP.one2 = function() { var self = this; var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader2.push({ builder: builder, count: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -1197,7 +1196,7 @@ DP.top = function(max) { var self = this; var builder = new DatabaseBuilder(self); builder.take(max); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1468,21 +1467,12 @@ DP.$update = function() { self.$writting = true; var filter = self.pending_update.splice(0); - var length = filter.length; - var backup = false; - var filters = 0; + var filters = new NoSQLReader(); + var fs = new NoSQLStream(self.filename); var change = false; - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup || fil.builder.$options.backup) - backup = true; - } - - var indexer = 0; - var fs = new NoSQLStream(self.filename); + for (var i = 0; i < filter.length; i++) + filters.add(filter[i].builder, true); if (self.buffersize) fs.buffersize = self.buffersize; @@ -1490,126 +1480,84 @@ DP.$update = function() { if (self.buffercount) fs.buffercount = self.buffercount; - fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - - for (var a = 0; a < docs.length; a++) { - - indexer++; - - var doc = docs[a]; - var is = false; - var rec = fs.docsbuffer[a]; - - for (var i = 0; i < length; i++) { - - var item = filter[i]; - if (item.skip) - continue; - - item.filter.index = indexer; - - var output = item.compare(doc, item.filter, indexer); - if (output) { - - var e = item.keys ? 'modify' : 'update'; - var old = self.$events[e] ? CLONE(output) : 0; - - if (item.filter.options.first) { - item.skip = true; - filters++; - } - - if (item.keys) { - for (var j = 0; j < item.keys.length; j++) { - var key = item.keys[j]; - var val = item.doc[key]; - if (val !== undefined) { - if (typeof(val) === 'function') - output[key] = val(output[key], output); - else if (item.inc && item.inc[key]) { - switch (item.inc[key]) { - case '+': - output[key] = (output[key] || 0) + val; - break; - case '-': - output[key] = (output[key] || 0) - val; - break; - case '*': - output[key] = (output[key] || 0) + val; - break; - case '/': - output[key] = (output[key] || 0) / val; - break; - } - } else - output[key] = val; - } + var update = function(docs, doc, dindex, f, findex) { + + var rec = fs.docsbuffer[dindex]; + var fil = filter[findex]; + var e = fil.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(doc) : 0; + + if (f.first) + f.canceled = true; + + if (fil.keys) { + for (var j = 0; j < fil.keys.length; j++) { + var key = fil.keys[j]; + var val = fil.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[key] = val(doc[key], doc); + else if (fil.inc && fil.inc[key]) { + switch (fil.inc[key]) { + case '+': + doc[key] = (doc[key] || 0) + val; + break; + case '-': + doc[key] = (doc[key] || 0) - val; + break; + case '*': + doc[key] = (doc[key] || 0) + val; + break; + case '/': + doc[key] = (doc[key] || 0) / val; + break; } } else - output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - - self.$events[e] && self.emit(e, output, old); - item.count++; - doc = output; - is = true; + doc[key] = val; } } + } else + docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; - if (is) { + self.$events[e] && self.emit(e, doc, old); + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + }; - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } + var updateflush = function(docs, doc, dindex) { - var upd = JSON.stringify(doc).replace(REGBOOL, JSONBOOL); - if (upd === rec.doc) - continue; + doc = docs[dindex]; - var was = true; + var rec = fs.docsbuffer[dindex]; + var upd = JSON.stringify(doc).replace(REGBOOL, JSONBOOL); + if (upd === rec.doc) + return; - if (!change) - change = true; + !change && (change = true); - if (rec.doc.length === upd.length) { - var b = Buffer.byteLength(upd); - if (rec.length === b) { - fs.write(upd + NEWLINE, rec.position); - was = false; - } - } + var was = true; + !change && (change = true); - if (was) { - var tmp = '-' + rec.doc.substring(1) + NEWLINE; - fs.write(tmp, rec.position); - fs.write2(upd + NEWLINE); - } + if (rec.doc.length === upd.length) { + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + was = false; } + } - if (filters === length) - return false; + if (was) { + var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; + fs.write(tmp, rec.position); + fs.write2(upd + NEWLINE); } }; - fs.$callback = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); - var tmp = self.insert(item.insert); - tmp.$callback = item.builder.$callback; - tmp.$options.log = item.builder.$options.log; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - } + fs.ondocuments = function() { + filters.compare2(JSON.parse('[' + fs.docs + ']', jsonparser), update, updateflush); + }; + fs.$callback = function() { + filters.done(); fs = null; self.$writting = false; self.next(0); @@ -1631,74 +1579,70 @@ DP.$update_inmemory = function() { } var filter = self.pending_update.splice(0); - var length = filter.length; var change = false; + var filters = new NoSQLReader(); - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } + for (var i = 0; i < filter.length; i++) + filters.add(filter[i].builder, true); return self.$inmemory(function() { - var data = self.inmemory['#']; - - for (var a = 0, al = data.length; a < al; a++) { - - var doc = data[a]; - - for (var i = 0; i < length; i++) { - - var item = filter[i]; - var builder = item.builder; - item.filter.index = j; - var output = item.compare(doc, item.filter, j); - if (output) { - - var e = item.keys ? 'modify' : 'update'; - var old = self.$events[e] ? CLONE(output) : 0; - - builder.$options.backup && builder.$backupdoc(doc); - - if (item.keys) { - for (var j = 0, jl = item.keys.length; j < jl; j++) { - var val = item.doc[item.keys[j]]; - if (val !== undefined) { - if (typeof(val) === 'function') - doc[item.keys[j]] = val(doc[item.keys[j]], doc); - else - doc[item.keys[j]] = val; + var old; + + var update = function(docs, doc, dindex, f, findex) { + + var fil = filter[findex]; + var e = fil.keys ? 'modify' : 'update'; + + if (!old) + old = self.$events[e] ? CLONE(doc) : 0; + + if (f.first) + f.canceled = true; + + if (fil.keys) { + for (var j = 0; j < fil.keys.length; j++) { + var key = fil.keys[j]; + var val = fil.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[key] = val(doc[key], doc); + else if (fil.inc && fil.inc[key]) { + switch (fil.inc[key]) { + case '+': + doc[key] = (doc[key] || 0) + val; + break; + case '-': + doc[key] = (doc[key] || 0) - val; + break; + case '*': + doc[key] = (doc[key] || 0) + val; + break; + case '/': + doc[key] = (doc[key] || 0) / val; + break; } - } - } else - doc = typeof(item.doc) === 'function' ? item.doc(doc) : item.doc; - - self.$events[e] && self.emit(e, doc, old); - item.count++; - if (!change) - change = true; + } else + doc[key] = val; + } } - } - } + } else + docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; - self.$save(); + self.$events[e] && self.emit(e, doc, old); + f.builder.$options.backup && f.builder.$backupdoc(old); + return 1; + }; - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); - var tmp = self.insert(item.insert); - tmp.$callback = item.builder.$callback; - tmp.$options.log = item.builder.$options.log; - } else { - var e = item.keys ? 'modify' : 'update'; - item.count && self.$events[e] && self.emit(e, item.doc); - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - } + var updateflush = function(docs, doc) { + !change && (change = true); + self.$events.update && self.emit('update', doc, old); + old = null; + }; + filters.compare2(self.inmemory['#'], update, updateflush); + + change && self.$save(); setImmediate(function() { self.next(0); change && self.$events.change && self.emit('change', 'update'); @@ -1750,24 +1694,8 @@ DP.$reader2 = function(filename, items, callback, reader) { } } - var filter = items; - var length = filter.length; - var first = true; - var indexer = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first || fil.builder.$options.sort) - first = false; - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(items); if (self.buffersize) fs.buffersize = self.buffersize; @@ -1776,144 +1704,11 @@ DP.$reader2 = function(filename, items, callback, reader) { fs.buffercount = self.buffercount; fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - var val; - - for (var j = 0; j < docs.length; j++) { - var json = docs[j]; - indexer++; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = indexer; - var output = item.compare(json, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } - } + return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser)); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else { - if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - } - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - + filters.done(); fs = null; callback(); }; @@ -1929,7 +1724,6 @@ DP.$reader2 = function(filename, items, callback, reader) { DP.$reader3 = function() { var self = this; - self.step = 11; if (!self.pending_reader2.length) { @@ -1939,24 +1733,8 @@ DP.$reader3 = function() { self.$reading = true; - var filter = self.pending_reader2.splice(0); - var length = filter.length; - var first = true; - var indexer = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first) - first = false; - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(self.pending_reader2.splice(0)); if (self.buffersize) fs.buffersize = self.buffersize; @@ -1965,159 +1743,11 @@ DP.$reader3 = function() { fs.buffercount = self.buffercount; fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - var val; - - for (var j = 0; j < docs.length; j++) { - var json = docs[j]; - indexer++; - - var done = true; - - for (var i = 0; i < length; i++) { - - var item = filter[i]; - - if (item.done) - continue; - else if (done) - done = false; - - var builder = item.builder; - item.filter.index = indexer; - - var output = item.compare(json, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (!builder.$inlinesort && !item.done) - item.done = builder.$options.take && builder.$options.take <= item.counter; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } - - if (done) - return false; - } + return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser)); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - + filters.done(); self.$reading = false; fs = null; self.next(0); @@ -2230,235 +1860,57 @@ function nosqlresort(arr, builder, doc) { } DP.$reader2_inmemory = function(items, callback) { + var self = this; + var filters = new NoSQLReader(items); + return self.$inmemory(function() { + filters.compare(self.inmemory['#']); + filters.done(); + callback(); + }); +}; + +DP.$remove = function() { var self = this; - var filter = items; - var length = filter.length; - var name = '#'; + self.step = 3; - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + if (!self.pending_remove.length) { + self.next(0); + return; } - return self.$inmemory(function() { + self.$writting = true; - var data = self.inmemory[name]; - var val; - - for (var j = 0, jl = data.length; j < jl; j++) { - var json = data[j]; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = j; - var output = item.compare(U.clone(json), item.filter, j); - if (!output) - continue; + var fs = new NoSQLStream(self.filename); + var filter = self.pending_remove.splice(0); + var filters = new NoSQLReader(filter); + var change = false; - item.count++; + if (self.buffersize) + fs.buffersize = self.buffersize; - if (!builder.$options.sort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; + if (self.buffercount) + fs.buffercount = self.buffercount; - item.counter++; + var remove = function(docs, d, dindex, f) { + var rec = fs.docsbuffer[dindex]; + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + return 1; + }; - if (item.type) - continue; + var removeflush = function(docs, d, dindex) { + var rec = fs.docsbuffer[dindex]; + !change && (change = true); + self.$events.remove && self.emit('remove', d); + fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); + }; - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = json[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = json[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = json[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = json[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : 0; - if (item.scalarcount) - item.scalarcount++; - else - item.scalarcount = 1; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - - default: - if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - } - } - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.counter : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - else if (builder.$options.sort === EMPTYOBJECT) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - - callback(); - }); -}; - -DP.$remove = function() { - - var self = this; - self.step = 3; - - if (!self.pending_remove.length) { - self.next(0); - return; - } - - self.$writting = true; - - var fs = new NoSQLStream(self.filename); - var filter = self.pending_remove.splice(0); - var length = filter.length; - var change = false; - var indexer = 0; - var backup = false; - - if (self.buffersize) - fs.buffersize = self.buffersize; - - if (self.buffercount) - fs.buffercount = self.buffercount; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup || fil.builder.$options.backup) - backup = true; - } - - fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - - for (var j = 0; j < docs.length; j++) { - - indexer++; - var removed = false; - var doc = docs[j]; - var rec = fs.docsbuffer[j]; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.filter.index = indexer; - var output = item.compare(doc, item.filter, indexer); - if (output) { - removed = true; - doc = output; - break; - } - } - - if (removed) { - - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } - - if (!change) - change = true; - - item.count++; - self.$events.remove && self.emit('remove', doc); - fs.write('-' + rec.doc.substring(1) + NEWLINE, rec.position); - } - } - }; + fs.ondocuments = function() { + filters.compare2(JSON.parse('[' + fs.docs + ']', jsonparser), remove, removeflush); + }; fs.$callback = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - + filters.done(); fs = null; self.$writting = false; self.next(0); @@ -2565,57 +2017,29 @@ DP.$remove_inmemory = function() { return self; } - var filter = self.pending_remove.splice(0); - var length = filter.length; var change = false; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } + var filters = new NoSQLReader(self.pending_remove.splice(0)); return self.$inmemory(function() { + var cache = self.inmemory['#'].slice(0); - var data = self.inmemory['#']; - var arr = []; - - for (var j = 0, jl = data.length; j < jl; j++) { - var json = data[j]; - var removed = false; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.filter.index = j; - if (item.compare(json, item.filter, j)) { - removed = true; - break; - } - } - - if (removed) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(JSON.stringify(json)); - item.count++; - } - change = true; - self.$events.remove && self.emit('remove', json); - } else - arr.push(json); - } - - if (change) { - self.inmemory['#'] = arr; - self.$save(); - } + var remove = function(docs, d, dindex, f) { + f.builder.$options.backup && f.builder.$backupdoc(d); + return 1; + }; - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); - } + var removeflush = function(docs, d) { + !change && (change = true); + self.$events.remove && self.emit('remove', d); + var data = self.inmemory['#']; + var index = data.indexOf(d); + if (index !== -1) + self.inmemory['#'].splice(index, index + 1); + }; + filters.compare2(cache, remove, removeflush); + change && self.$save(); + filters.done(); self.next(0); change && self.$events.change && self.emit('change', 'remove'); }); @@ -3003,6 +2427,16 @@ DatabaseBuilder.prototype.empty = function(name) { return self; }; +DatabaseBuilder.prototype.map = function(name, code) { + var self = this; + var data = { name: name, code: code }; + if (self.$options.mappers) + self.$options.mappers.push(data); + else + self.$options.mappers = [data]; + return self; +}; + DatabaseBuilder.prototype.backup = function(user) { if (this.db.filenameBackup) this.$options.backup = typeof(user) === 'string' ? user : 'unknown'; @@ -3374,6 +2808,18 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$inlinesort = !!(opt.take && opt.sort && opt.sort !== null); self.$limit = (opt.take || 0) + (opt.skip || 0); var key = opt.id ? self.db.name + '_' + opt.id : code.hash(); + + if (opt.mappers) { + var tmp = ''; + self.$mappers = []; + for (var i = 0; i < opt.mappers.length; i++) { + var m = opt.mappers[i]; + tmp += ('doc.{0}=item.builder.$mappers[\'{1}\'](doc,R);'.format(m.name, i)); + self.$mappers.push(new Function('doc', 'repository', code)); + } + self.$mappersexec = new Function('doc', 'R', tmp.join('')); + } + return CACHE[key] ? CACHE[key] : (CACHE[key] = new Function('doc', '$F', 'index', code)); }; @@ -5892,7 +5338,7 @@ TP.remove = function() { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - self.pending_remove.push({ builder: builder, count: 0 }); + self.pending_remove.push(builder); setImmediate(next_operation, self, 3); return builder; }; @@ -5906,7 +5352,7 @@ TP.listing = function(builder) { builder = new DatabaseBuilder(self); builder.$options.listing = true; builder.$take = builder.$options.take = 100; - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -5918,7 +5364,7 @@ TP.find = function(builder) { builder.db = self; else builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -5929,7 +5375,7 @@ TP.find2 = function(builder) { builder.db = self; else builder = new DatabaseBuilder(self); - self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -6038,7 +5484,8 @@ TP.count = function() { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, type: 1 }); + builder.$options.readertype = 1; + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -6048,7 +5495,7 @@ TP.one = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader.push({ builder: builder, count: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -6058,7 +5505,7 @@ TP.one2 = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader2.push({ builder: builder, count: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -6068,7 +5515,7 @@ TP.top = function(max) { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.take(max); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -6202,42 +5649,13 @@ TP.$reader = function() { self.$reading = true; - var filter = self.pending_reader.splice(0); - var length = filter.length; - var first = true; - var indexer = 0; - var keys = {}; - var keyscount = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first || fil.builder.$options.sort) - first = false; - if (fil.builder.$keys == null && keys) - keys = null; - else if (keys) { - if (fil.builder.$options.fields) { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } else - keys = null; - } - - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(self.pending_reader.splice(0)); var data = {}; + var indexer = 0; fs.divider = '\n'; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + data.keys = self.$keys; if (self.buffersize) fs.buffersize = self.buffersize; @@ -6248,146 +5666,19 @@ TP.$reader = function() { fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); - var val; + var arr = []; for (var j = indexer ? 0 : 1; j < lines.length; j++) { - data.line = lines[j].split('|'); data.index = indexer++; - - indexer++; - - var obj = self.parseData(data); - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = indexer; - - var output = item.compare(obj, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } + arr.push(self.parseData(data)); } + + return filters.compare(arr, jsonparser); }; fs.$callback = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - + filters.done(); fs = null; self.$reading = false; self.next(0); @@ -6410,42 +5701,13 @@ TP.$reader3 = function() { self.$reading = true; - var filter = self.pending_reader2.splice(0); - var length = filter.length; - var first = true; - var indexer = 0; - var keys = {}; - var keyscount = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first || fil.builder.$options.sort) - first = false; - if (fil.builder.$keys == null && keys) - keys = null; - else if (keys) { - if (fil.builder.$options.fields) { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } else - keys = null; - } - - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(self.pending_reader2.splice(0)); var data = {}; + var indexer = 0; fs.divider = '\n'; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + data.keys = self.$keys; if (self.buffersize) fs.buffersize = self.buffersize; @@ -6456,152 +5718,23 @@ TP.$reader3 = function() { fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); - var val; + var arr = []; for (var j = 0; j < lines.length; j++) { - data.line = lines[j].split('|'); - if (!TABLERECORD[data.line[0]]) continue; - data.index = indexer++; - indexer++; - - var obj = self.parseData(data); - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = indexer; - - var output = item.compare(obj, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } + arr.push(self.parseData(data)); } + + return filters.compare(arr, jsonparser); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - - self.$reading = false; + filters.done(); fs = null; + self.$reading = false; self.next(0); }; @@ -6621,174 +5754,115 @@ TP.$update = function() { self.$writting = true; + var fs = new NoSQLStream(self.filename); var filter = self.pending_update.splice(0); - var length = filter.length; - var backup = false; - var filters = 0; + var filters = new NoSQLReader(); var change = false; - var keys = {}; - var keyscount = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(true); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - - if (fil.backup || fil.builder.$options.backup) - backup = true; - - if (fil.builder.$keys == null) - keys = null; - else { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } + var indexer = 0; + var data = { keys: self.$keys }; - } + for (var i = 0; i < filter.length; i++) + filters.add(filter[i].builder, true); - var indexer = 0; - var fs = new NoSQLStream(self.filename); fs.divider = '\n'; - var data = {}; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; - if (self.buffersize) fs.buffersize = self.buffersize; if (self.buffercount) fs.buffercount = self.buffercount; - fs.ondocuments = function() { - - var lines = fs.docs.split(fs.divider); - var val; + var update = function(docs, doc, dindex, f, findex) { - for (var a = indexer ? 0 : 1; a < lines.length; a++) { + var rec = fs.docsbuffer[dindex]; - data.line = lines[a].split('|'); - data.length = lines[a].length; - data.index = indexer++; + var fil = filter[findex]; + var e = fil.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(doc) : 0; - var is = false; - var rec = fs.docsbuffer[a]; - var doc = self.parseData(data, data.keys === self.$keys ? EMPTYOBJECT : null); + if (f.first) + f.canceled = true; - for (var i = 0; i < length; i++) { - - var item = filter[i]; - if (item.skip) - continue; - - item.filter.index = indexer; - - var output = item.compare(doc, item.filter, indexer); - if (output) { - - if (item.filter.options.first) { - item.skip = true; - filters++; - } - - if (data.keys !== self.$keys) { - var tmp = data.keys; - data.keys = self.$keys; - output = self.parseData(data, output); - data.keys = tmp; - } - - var e = item.keys ? 'modify' : 'update'; - var old = self.$events[e] ? CLONE(output) : 0; - - if (item.keys) { - for (var j = 0; j < item.keys.length; j++) { - var key = item.keys[j]; - var val = item.doc[key]; - if (val !== undefined) { - if (typeof(val) === 'function') - output[key] = val(output[key], output); - else if (item.inc && item.inc[key]) { - switch (item.inc[key]) { - case '+': - output[key] = (output[key] || 0) + val; - break; - case '-': - output[key] = (output[key] || 0) - val; - break; - case '*': - output[key] = (output[key] || 0) + val; - break; - case '/': - output[key] = (output[key] || 0) / val; - break; - } - } else - output[key] = val; - } + if (fil.keys) { + for (var j = 0; j < fil.keys.length; j++) { + var key = fil.keys[j]; + var val = fil.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[key] = val(doc[key], doc); + else if (fil.inc && fil.inc[key]) { + switch (fil.inc[key]) { + case '+': + doc[key] = (doc[key] || 0) + val; + break; + case '-': + doc[key] = (doc[key] || 0) - val; + break; + case '*': + doc[key] = (doc[key] || 0) + val; + break; + case '/': + doc[key] = (doc[key] || 0) / val; + break; } } else - output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - - self.$events[e] && self.emit(e, output, old); - item.count++; - doc = output; - is = true; + doc[key] = val; } } + } else + docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; - if (is) { + self.$events[e] && self.emit(e, doc, old); + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + }; - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } + var updateflush = function(docs, doc, dindex) { - var upd = self.stringify(doc, null, rec.length); - if (upd === rec.doc) - continue; + doc = docs[dindex]; - if (!change) - change = true; + var rec = fs.docsbuffer[dindex]; + var upd = self.stringify(doc, null, rec.length); - var b = Buffer.byteLength(upd); - if (rec.length === b) { - fs.write(upd + NEWLINE, rec.position); - } else { - var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; - fs.write(tmp, rec.position); - fs.write2(upd + NEWLINE); - } - } + if (upd === rec.doc) + return; + + !change && (change = true); - if (filters === length) - return false; + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + } else { + var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; + fs.write(tmp, rec.position); + fs.write2(upd + NEWLINE); } + }; - fs.$callback = function() { + fs.ondocuments = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); - var tmp = self.insert(item.insert); - tmp.$callback = item.builder.$callback; - tmp.$options.log = item.builder.$options.log; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } + var lines = fs.docs.split(fs.divider); + var arr = []; + + if (!indexer) + arr.push(EMPTYOBJECT); + + for (var a = indexer ? 0 : 1; a < lines.length; a++) { + data.line = lines[a].split('|'); + data.length = lines[a].length; + data.index = indexer++; + arr.push(self.parseData(data, EMPTYOBJECT)); } + filters.compare2(arr, update, updateflush); + }; + + + fs.$callback = function() { + filters.done(); fs = null; self.$writting = false; self.next(0); - change && self.$events.change && self.emit('change', 'update'); }; @@ -6810,12 +5884,9 @@ TP.$remove = function() { var fs = new NoSQLStream(self.filename); var filter = self.pending_remove.splice(0); - var length = filter.length; + var filters = new NoSQLReader(filter); var change = false; var indexer = 0; - var backup = false; - var keys = {}; - var keyscount = 0; fs.divider = '\n'; @@ -6825,78 +5896,37 @@ TP.$remove = function() { if (self.buffercount) fs.buffercount = self.buffercount; - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(true); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup || fil.builder.$options.backup) - backup = true; - if (fil.builder.$keys == null) - keys = null; - else { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } - } + var data = { keys: self.$keys }; - var data = {}; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + var remove = function(docs, d, dindex, f) { + var rec = fs.docsbuffer[dindex + 1]; + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + return 1; + }; + + var removeflush = function(docs, d, dindex) { + var rec = fs.docsbuffer[dindex + 1]; + !change && (change = true); + self.$events.remove && self.emit('remove', d); + fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); + }; fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); + var arr = []; for (var a = indexer ? 0 : 1; a < lines.length; a++) { - data.line = lines[a].split('|'); data.index = indexer++; - - indexer++; - var removed = false; - var doc = self.parseData(data); - var rec = fs.docsbuffer[a]; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.filter.index = indexer; - var output = item.compare(doc, item.filter, indexer); - if (output) { - removed = true; - doc = output; - break; - } - } - - if (removed) { - - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } - - if (!change) - change = true; - - item.count++; - self.$events.remove && self.emit('remove', doc); - fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); - } + arr.push(self.parseData(data)); } + + filters.compare2(arr, remove, removeflush); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - + filters.done(); fs = null; self.$writting = false; self.next(0); @@ -7127,8 +6157,10 @@ TP.parseData = function(data, cache) { var esc = data.line[0] === '*'; var val, alloc; - if (cache && data.keys.length === data.line.length - 2) - alloc = data.line[data.line.length - 1].length - 1; + if (cache && data.keys.length === data.line.length - 2) { + // alloc = data.line[data.line.length - 1].length - 1; + alloc = data.line[data.line.length - 1].length; + } for (var i = 0; i < data.keys.length; i++) { var key = data.keys[i]; @@ -7171,7 +6203,7 @@ TP.parseData = function(data, cache) { } } - alloc && (obj.$$alloc = { size: alloc, length: data.length }); + alloc >= 0 && (obj.$$alloc = { size: alloc, length: data.length }); return obj; }; @@ -7224,7 +6256,7 @@ TP.stringify = function(doc, insert, byteslen) { if (doc.$$alloc) { var l = output.length; var a = doc.$$alloc; - if (l < a.length) { + if (l <= a.length) { var s = (a.length - l) - 1; if (s > 0) { output += '|'.padRight(s, '.'); @@ -7353,132 +6385,227 @@ function jsonparser(key, value) { // item.filter = builder.makefilter(); // item.index = DOCUMENT_COUNTER; -exports.compare = function(item, obj) { +function NoSQLReader(builder) { + var self = this; + self.builders = []; + self.canceled = 0; + builder && self.add(builder); +} + +NoSQLReader.prototype.add = function(builder, noTrimmer) { + var self = this; + if (builder instanceof Array) { + for (var i = 0; i < builder.length; i++) + self.add(builder[i]); + } else { + var item = {}; + item.scalarcount = 0; + item.count = 0; + item.counter = 0; + item.builder = builder; + item.compare = builder.compile(noTrimmer); + item.filter = builder.makefilter(); + item.first = builder.$options.first && !builder.$options.sort; + self.builders.push(item); + } + return self; +}; - var val; - var builder = item.builder; +NoSQLReader.prototype.compare2 = function(docs, custom, done) { + var self = this; + for (var i = 0; i < docs.length; i++) { - item.filter.index = item.index; + var doc = docs[i]; - var output = item.compare(obj, item.filter, item.index); - if (!output) - return; + if (self.builders.length === self.canceled) + return false; - item.count++; + var is = false; - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - return; + for (var j = 0; j < self.builders.length; j++) { - item.counter++; + var item = self.builders[j]; + if (item.canceled) + continue; - if (!builder.$inlinesort && !item.done) - item.done = builder.$options.take && builder.$options.take <= item.counter; + var output = item.compare(doc, item.filter, j); + if (!output) + continue; - if (item.type) - return; + item.is = false; - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; + // @TODO: add "first" checking + if (custom) { + !is && (is = true); + item.count++; + var canceled = item.canceled; + var c = custom(docs, output, i, item, j); + if (!canceled && item.canceled) + self.canceled++; + if (c === 1) + break; + else + continue; } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; + } + + is && done && done(docs, doc, i, self.builders); + } +}; + +NoSQLReader.prototype.compare = function(docs) { + + var self = this; + for (var i = 0; i < docs.length; i++) { + + var doc = docs[i]; + + if (self.builders.length === self.canceled) + return false; + + for (var j = 0; j < self.builders.length; j++) { + + var item = self.builders[j]; + if (item.canceled) + continue; + + var output = item.compare(doc, item.filter, j); + if (!output) + continue; + + var b = item.builder; + item.count++; + + if (!b.$inlinesort && ((b.$options.skip && b.$options.skip >= item.count) || (b.$options.take && b.$options.take <= item.counter))) + continue; + + item.counter++; + + if (!b.$inlinesort && !item.done) { + item.done = b.$options.take && b.$options.take <= item.counter; + if (item.done) + continue; } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; + + if (b.$options.readertype) + continue; + + b.$mappersexec && b.$mappersexec(doc, item.filter); + + var val; + + switch (b.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[b.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[b.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'max': + val = output[b.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'avg': + val = output[b.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[b.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (b.$inlinesort) + nosqlinlinesorter(item, b, output); + else if (item.response) + item.response.push(output); + else + item.response = [output]; + break; } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; + + if (item.first) { + item.canceled = true; + self.canceled++; } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; + } } - - return item.first ? false : true; }; -exports.callback = function(item, err) { +NoSQLReader.prototype.done = function() { - var builder = item.builder; - var output; + var self = this; + for (var i = 0; i < self.builders.length; i++) { - if (builder.$options.scalar || !builder.$options.sort) { + var item = self.builders[i]; + var builder = item.builder; + var output; - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) + if (builder.$options.scalar || !builder.$options.sort) { + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); + continue; + } + + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) output = item.response ? item.response[0] : undefined; else if (builder.$options.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(err, builder, output), item.type === 1 ? item.count : output, item.count); - return; + builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); + builder.done(); } +}; - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(err, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); -}; \ No newline at end of file +exports.NoSQLReader = NoSQLReader; \ No newline at end of file From b193f67883ceb46529c4238c50a8ea574da3d4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 20:58:50 +0200 Subject: [PATCH 0586/1669] Fixed `table.modify()` / `table.update()` --- nosql.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/nosql.js b/nosql.js index 9a8c670ad..e840132f4 100755 --- a/nosql.js +++ b/nosql.js @@ -1103,6 +1103,7 @@ DP.remove = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_remove.push(builder); + builder.$options.readertype = 1; setImmediate(next_operation, self, 3); return builder; }; @@ -5299,6 +5300,7 @@ TP.update = function(doc, insert) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); + builder.$options.readertype = 1; self.pending_update.push({ builder: builder, doc: doc, count: 0, insert: insert === true ? doc : insert }); setImmediate(next_operation, self, 2); return builder; @@ -5328,6 +5330,7 @@ TP.modify = function(doc, insert) { break; } } + builder.$options.readertype = 1; self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, inc: inc, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); } @@ -5339,6 +5342,7 @@ TP.remove = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_remove.push(builder); + builder.$options.readertype = 1; setImmediate(next_operation, self, 3); return builder; }; @@ -6377,14 +6381,6 @@ function jsonparser(key, value) { return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; } -// Item requirements: -// item.first = false; -// item.scalarcount = 0; -// item.builder = builder; -// item.compare = builder.compile(); -// item.filter = builder.makefilter(); -// item.index = DOCUMENT_COUNTER; - function NoSQLReader(builder) { var self = this; self.builders = []; From 0d4639564fa9efe50e9a4ef3bfad88a54697ed58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 21:28:25 +0200 Subject: [PATCH 0587/1669] Fixed file modifications. --- nosql.js | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/nosql.js b/nosql.js index e840132f4..a7e515839 100755 --- a/nosql.js +++ b/nosql.js @@ -6429,20 +6429,21 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { continue; item.is = false; + !is && (is = true); + item.count++; + var canceled = item.canceled; + var c = custom(docs, output, i, item, j); - // @TODO: add "first" checking - if (custom) { - !is && (is = true); - item.count++; - var canceled = item.canceled; - var c = custom(docs, output, i, item, j); - if (!canceled && item.canceled) - self.canceled++; - if (c === 1) - break; - else - continue; - } + if (item.first) { + item.canceled = true; + self.canceled++; + } else if (!canceled && item.canceled) + self.canceled++; + + if (c === 1) + break; + else + continue; } is && done && done(docs, doc, i, self.builders); @@ -6477,11 +6478,8 @@ NoSQLReader.prototype.compare = function(docs) { item.counter++; - if (!b.$inlinesort && !item.done) { + if (!b.$inlinesort && !item.done) item.done = b.$options.take && b.$options.take <= item.counter; - if (item.done) - continue; - } if (b.$options.readertype) continue; From d4c05e827cc4225237e9937ebde6404c4a50955a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 22:19:17 +0200 Subject: [PATCH 0588/1669] New improvements. --- nosql.js | 42 +++++++++++++++++++++++++ nosqlcrawler.js | 83 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 nosqlcrawler.js diff --git a/nosql.js b/nosql.js index a7e515839..29ff8c55e 100755 --- a/nosql.js +++ b/nosql.js @@ -5066,6 +5066,48 @@ SP.listing = function(beg, end, callback) { return self; }; +SP.find = function(beg, end, threads) { + var self = this; + + if (!threads) + threads = 1; + + var builder = new DatabaseBuilder(self); + var filters = new NoSQLReader([builder]); + + self.listing(beg, end, function(err, storage) { + + var count = (storage.length / threads) >> 0; + var opt = { cwd: F.directory }; + var filename = module.filename.replace(/\.js$/, '') + 'crawler.js'; + var counter = 0; + var finish = threads; + + for (var i = 0; i < threads; i++) { + var fork = require('child_process').fork(filename, EMPTYARRAY, opt); + var files = (i === threads - 1) ? storage : storage.splice(0, count); + fork.send({ TYPE: 'init', files: files, builder: builder.stringify() }); + fork.on('message', function(msg) { + counter += msg.count; + msg.response && msg.response.length && filters.compare(msg.response); + finish--; + if (finish === 0) { + filters.builders[0].count = counter; + filters.done(); + } + }); + } + }); + + return builder; +}; + +SP.count = function(beg, end, threads) { + var builder = this.find(beg, end, threads); + builder.$options.readertype = 1; + return builder; +}; + SP.scan = function(beg, end, mapreduce, callback, reverse) { var self = this; diff --git a/nosqlcrawler.js b/nosqlcrawler.js new file mode 100644 index 000000000..bc2998931 --- /dev/null +++ b/nosqlcrawler.js @@ -0,0 +1,83 @@ +// Copyright 2018 (c) Peter Širka +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +/** + * @module NoSQL Crawler + * @version 1.0.0 + */ + +var filename = module.filename.replace(/nosqlcrawler\.js$/, ''); + +require(filename + 'index.js'); +const NoSQLStream = require(filename + 'nosqlstream.js'); + +global.NOW = global.DATETIME = new Date(); + +function killprocess() { + process.exit(0); +} + +setInterval(function() { + global.NOW = global.DATETIME = new Date(); +}, 30000); + +process.on('disconnect', killprocess); +process.on('close', killprocess); +process.on('exit', killprocess); +process.on('message', function(msg) { + + // msg.builder; + // msg.filename or msg.data + + var builder = new framework_nosql.DatabaseBuilder(); + builder.parse(msg.builder); + + var filters = new framework_nosql.NoSQLReader([builder]); + + msg.files.wait(function(item, next) { + find(item.filename, filters, next); + }, function() { + var item = filters.builders[0]; + process.send({ response: item.response, count: item.count }); + setTimeout(() => killprocess(), 1000); + }); + +}); + +function find(filename, filters, next) { + + var fs = new NoSQLStream(filename); + + fs.ondocuments = function() { + return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser)); + }; + + fs.$callback = function() { + fs = null; + next(); + }; + + fs.openread(); +} + +function jsonparser(key, value) { + return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; +} From c9a330339e4edc8e09fd2f368c44534f725a7a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 22:20:45 +0200 Subject: [PATCH 0589/1669] Added new change. --- changes.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes.txt b/changes.txt index 813b5e95f..443b22595 100755 --- a/changes.txt +++ b/changes.txt @@ -1,6 +1,9 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` +- added: NoSQL storage `.find()` + `.count()` with multiple thread support + +- improved: NoSQL reader - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function From 0ab820b973ad59f3bd97f975163c492b3ea32f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 23:35:04 +0200 Subject: [PATCH 0590/1669] Fixed mapping. --- nosql.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 29ff8c55e..ebbb2402f 100755 --- a/nosql.js +++ b/nosql.js @@ -2815,10 +2815,11 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$mappers = []; for (var i = 0; i < opt.mappers.length; i++) { var m = opt.mappers[i]; - tmp += ('doc.{0}=item.builder.$mappers[\'{1}\'](doc,R);'.format(m.name, i)); - self.$mappers.push(new Function('doc', 'repository', code)); + tmp += ('doc.{0}=item.builder.$mappers[{1}](doc,item.filter.repository,item.filter.repository);'.format(m.name, i)); + opt.fields && opt.fields.push(m.name); + self.$mappers.push(new Function('doc', 'repository', 'R', m.code.lastIndexOf('return ') === -1 ? ('return ' + m.code) : m.code)); } - self.$mappersexec = new Function('doc', 'R', tmp.join('')); + self.$mappersexec = new Function('doc', 'item', tmp); } return CACHE[key] ? CACHE[key] : (CACHE[key] = new Function('doc', '$F', 'index', code)); @@ -6526,7 +6527,7 @@ NoSQLReader.prototype.compare = function(docs) { if (b.$options.readertype) continue; - b.$mappersexec && b.$mappersexec(doc, item.filter); + b.$mappersexec && b.$mappersexec(output, item); var val; From d2fdbec5cff31555f1cb8188330cc98b2aa9102c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 00:01:15 +0200 Subject: [PATCH 0591/1669] New improvement. --- nosql.js | 7 +- utils.js | 345 +++++++++++++++++++++++++++++++------------------------ 2 files changed, 203 insertions(+), 149 deletions(-) diff --git a/nosql.js b/nosql.js index ebbb2402f..51f477d4b 100755 --- a/nosql.js +++ b/nosql.js @@ -5074,10 +5074,10 @@ SP.find = function(beg, end, threads) { threads = 1; var builder = new DatabaseBuilder(self); - var filters = new NoSQLReader([builder]); self.listing(beg, end, function(err, storage) { + var filters = new NoSQLReader([builder]); var count = (storage.length / threads) >> 0; var opt = { cwd: F.directory }; var filename = module.filename.replace(/\.js$/, '') + 'crawler.js'; @@ -5109,6 +5109,10 @@ SP.count = function(beg, end, threads) { return builder; }; +SP.scalar = function(type, field) { + return this.find().scalar(type, field); +}; + SP.scan = function(beg, end, mapreduce, callback, reverse) { var self = this; @@ -6528,7 +6532,6 @@ NoSQLReader.prototype.compare = function(docs) { continue; b.$mappersexec && b.$mappersexec(output, item); - var val; switch (b.$options.scalar) { diff --git a/utils.js b/utils.js index cd566f4c6..a8b35f079 100755 --- a/utils.js +++ b/utils.js @@ -664,14 +664,16 @@ function ProxyAgent(options) { self.requests = []; } -ProxyAgent.prototype.createConnection = function(pending) { +const PAP = ProxyAgent.prototype; + +PAP.createConnection = function(pending) { var self = this; self.createSocket(pending, function(socket) { pending.request.onSocket(socket); }); }; -ProxyAgent.prototype.createSocket = function(options, callback) { +PAP.createSocket = function(options, callback) { var self = this; var proxy = self.options.proxy; @@ -719,13 +721,13 @@ function proxyagent_response(res) { res.upgrade = true; } -ProxyAgent.prototype.addRequest = function(req, options) { +PAP.addRequest = function(req, options) { this.createConnection({ host: options.host, port: options.port, request: req }); }; function createSecureSocket(options, callback) { var self = this; - ProxyAgent.prototype.createSocket.call(self, options, function(socket) { + PAP.createSocket.call(self, options, function(socket) { PROXYTLS.servername = self.options.uri.hostname; PROXYTLS.headers = self.options.uri.headers; PROXYTLS.socket = socket; @@ -2655,7 +2657,7 @@ exports.ls2 = function(path, callback, filter) { ls(path, callback, true, filter); }; -Date.prototype.add = function(type, value) { +DP.add = function(type, value) { var self = this; @@ -2719,13 +2721,15 @@ Date.prototype.add = function(type, value) { return dt; }; +const DP = Date.prototype; + /** * Date difference * @param {Date/Number/String} date Optional. * @param {String} type Date type: minutes, seconds, hours, days, months, years * @return {Number} */ -Date.prototype.diff = function(date, type) { +DP.diff = function(date, type) { if (arguments.length === 1) { type = date; @@ -2779,7 +2783,7 @@ Date.prototype.diff = function(date, type) { return NaN; }; -Date.prototype.extend = function(date) { +DP.extend = function(date) { var dt = new Date(this); var match = date.match(regexpDATE); @@ -2856,7 +2860,7 @@ Date.prototype.extend = function(date) { * @param {Date} date * @return {Number} Results: -1 = current date is earlier than @date, 0 = current date is same as @date, 1 = current date is later than @date */ -Date.prototype.compare = function(date) { +DP.compare = function(date) { var self = this; var r = self.getTime() - date.getTime(); @@ -2892,7 +2896,7 @@ Date.compare = function(d1, d2) { * @param {String} format * @return {String} */ -Date.prototype.format = function(format, resource) { +DP.format = function(format, resource) { if (!format) return this.getUTCFullYear() + '-' + (this.getUTCMonth() + 1).toString().padLeft(2, '0') + '-' + this.getUTCDate().toString().padLeft(2, '0') + 'T' + this.getUTCHours().toString().padLeft(2, '0') + ':' + this.getUTCMinutes().toString().padLeft(2, '0') + ':' + this.getUTCSeconds().toString().padLeft(2, '0') + '.' + this.getUTCMilliseconds().toString().padLeft(3, '0') + 'Z'; @@ -2978,22 +2982,24 @@ exports.$pmam = function(value) { return value >= 12 ? value - 12 : value; }; -Date.prototype.toUTC = function(ticks) { +DP.toUTC = function(ticks) { var dt = this.getTime() + this.getTimezoneOffset() * 60000; return ticks ? dt : new Date(dt); }; // +v2.2.0 parses JSON dates as dates and this is the fallback for backward compatibility -Date.prototype.parseDate = function() { +DP.parseDate = function() { return this; }; -String.prototype.isJSONDate = function() { +const SP = String.prototype; + +SP.isJSONDate = function() { var l = this.length - 1; return l > 22 && l < 30 && this[l] === 'Z' && this[10] === 'T' && this[4] === '-' && this[13] === ':' && this[16] === ':'; }; -String.prototype.ROOT = function(noremap) { +SP.ROOT = function(noremap) { var str = this; @@ -3018,26 +3024,26 @@ function $urlmaker(text) { return F.config['default-root'] ? F.config['default-root'] : (c || ''); } -if (!String.prototype.trim) { - String.prototype.trim = function() { +if (!SP.trim) { + SP.trim = function() { return this.replace(regexpTRIM, ''); }; } -if (!String.prototype.replaceAt) { - String.prototype.replaceAt = function(index, character) { +if (!SP.replaceAt) { + SP.replaceAt = function(index, character) { return this.substr(0, index) + character + this.substr(index + character.length); }; } /** * Checks if the string starts with the text - * @see {@link http://docs.totaljs.com/String.prototype/#String.prototype.startsWith|Documentation} + * @see {@link http://docs.totaljs.com/SP/#SP.startsWith|Documentation} * @param {String} text Text to find. * @param {Boolean/Number} ignoreCase Ingore case sensitive or position in the string. * @return {Boolean} */ -String.prototype.startsWith = function(text, ignoreCase) { +SP.startsWith = function(text, ignoreCase) { var self = this; var length = text.length; var tmp; @@ -3057,12 +3063,12 @@ String.prototype.startsWith = function(text, ignoreCase) { /** * Checks if the string ends with the text - * @see {@link http://docs.totaljs.com/String.prototype/#String.prototype.endsWith|Documentation} + * @see {@link http://docs.totaljs.com/SP/#SP.endsWith|Documentation} * @param {String} text Text to find. * @param {Boolean/Number} ignoreCase Ingore case sensitive or position in the string. * @return {Boolean} */ -String.prototype.endsWith = function(text, ignoreCase) { +SP.endsWith = function(text, ignoreCase) { var self = this; var length = text.length; var tmp; @@ -3080,7 +3086,7 @@ String.prototype.endsWith = function(text, ignoreCase) { return tmp.length === length && tmp === text; }; -String.prototype.replacer = function(find, text) { +SP.replacer = function(find, text) { var self = this; var beg = self.indexOf(find); return beg === -1 ? self : (self.substring(0, beg) + text + self.substring(beg + find.length)); @@ -3092,7 +3098,7 @@ String.prototype.replacer = function(find, text) { * @param {String} salt Optional, salt. * @return {String} */ -String.prototype.hash = function(type, salt) { +SP.hash = function(type, salt) { var str = salt ? this + salt : this; switch (type) { case 'md5': @@ -3112,7 +3118,7 @@ String.prototype.hash = function(type, salt) { } }; -String.prototype.crc32 = function(unsigned) { +SP.crc32 = function(unsigned) { var crc = -1; for (var i = 0, length = this.length; i < length; i++) crc = (crc >>> 8) ^ CRC32TABLE[(crc ^ this.charCodeAt(i)) & 0xFF]; @@ -3132,7 +3138,7 @@ function string_hash(s, convert) { return hash; } -String.prototype.count = function(text) { +SP.count = function(text) { var index = 0; var count = 0; do { @@ -3143,19 +3149,19 @@ String.prototype.count = function(text) { return count; }; -String.prototype.parseXML = function() { +SP.parseXML = function() { return F.onParseXML(this); }; -String.prototype.parseJSON = function(date) { +SP.parseJSON = function(date) { return exports.parseJSON(this, date); }; -String.prototype.parseQuery = function() { +SP.parseQuery = function() { return exports.parseQuery(this); }; -String.prototype.parseTerminal = function(fields, fn, skip, take) { +SP.parseTerminal = function(fields, fn, skip, take) { var lines = this.split('\n'); @@ -3273,7 +3279,7 @@ function parseTerminal2(lines, fn, skip, take) { } } -String.prototype.parseDate = function() { +SP.parseDate = function() { var self = this.trim(); var lc = self.charCodeAt(self.length - 1); @@ -3366,7 +3372,7 @@ String.prototype.parseDate = function() { return new Date(parsed[0], parsed[1] - 1, parsed[2], parsed[3], parsed[4] - NOW.getTimezoneOffset(), parsed[5]); }; -String.prototype.parseDateExpiration = function() { +SP.parseDateExpiration = function() { var self = this; var arr = self.split(' '); @@ -3385,7 +3391,7 @@ String.prototype.parseDateExpiration = function() { return dt; }; -String.prototype.contains = function(value, mustAll) { +SP.contains = function(value, mustAll) { var str = this; if (typeof(value) === 'string') @@ -3408,7 +3414,7 @@ String.prototype.contains = function(value, mustAll) { * @param {String} value * @return {Number} */ -String.prototype.localeCompare2 = function(value) { +SP.localeCompare2 = function(value) { return COMPARER(this, value); }; @@ -3418,7 +3424,7 @@ String.prototype.localeCompare2 = function(value) { * @onerr {Function} error handling * @return {Object} */ -String.prototype.parseConfig = function(def, onerr) { +SP.parseConfig = function(def, onerr) { if (typeof(def) === 'function') { onerr = def; @@ -3506,7 +3512,7 @@ String.prototype.parseConfig = function(def, onerr) { return obj; }; -String.prototype.format = function() { +SP.format = function() { var arg = arguments; return this.replace(regexpSTRINGFORMAT, function(text) { var value = arg[+text.substring(1, text.length - 1)]; @@ -3514,7 +3520,7 @@ String.prototype.format = function() { }); }; -String.prototype.encode = function() { +SP.encode = function() { var output = ''; for (var i = 0, length = this.length; i < length; i++) { var c = this[i]; @@ -3542,7 +3548,7 @@ String.prototype.encode = function() { return output; }; -String.prototype.decode = function() { +SP.decode = function() { return this.replace(regexpDECODE, function(s) { if (s.charAt(1) !== '#') return ALPHA_INDEX[s] || s; @@ -3551,15 +3557,15 @@ String.prototype.decode = function() { }); }; -String.prototype.urlEncode = function() { +SP.urlEncode = function() { return encodeURIComponent(this); }; -String.prototype.urlDecode = function() { +SP.urlDecode = function() { return decodeURIComponent(this); }; -String.prototype.arg = function(obj) { +SP.arg = function(obj) { return this.replace(regexpARG, function(text) { // Is double? var l = text[1] === '{' ? 2 : 1; @@ -3568,7 +3574,7 @@ String.prototype.arg = function(obj) { }); }; -String.prototype.params = function(obj) { +SP.params = function(obj) { OBSOLETE('String.params()', 'The method is deprecated instead of it use F.viewCompile() or String.format().'); @@ -3639,14 +3645,14 @@ String.prototype.params = function(obj) { }); }; -String.prototype.max = function(length, chars) { +SP.max = function(length, chars) { var str = this; if (typeof(chars) !== 'string') chars = '...'; return str.length > length ? str.substring(0, length - chars.length) + chars : str; }; -String.prototype.isJSON = function() { +SP.isJSON = function() { var self = this; if (self.length <= 1) return false; @@ -3673,48 +3679,48 @@ String.prototype.isJSON = function() { return (a === '"' && b === '"') || (a === '[' && b === ']') || (a === '{' && b === '}') || (a.charCodeAt(0) > 47 && b.charCodeAt(0) < 57); }; -String.prototype.isURL = function() { +SP.isURL = function() { return this.length <= 7 ? false : F.validators.url.test(this); }; -String.prototype.isZIP = function() { +SP.isZIP = function() { return F.validators.zip.test(this); }; -String.prototype.isEmail = function() { +SP.isEmail = function() { return this.length <= 4 ? false : F.validators.email.test(this); }; -String.prototype.isPhone = function() { +SP.isPhone = function() { return this.length < 6 ? false : F.validators.phone.test(this); }; -String.prototype.isUID = function() { +SP.isUID = function() { return this.length < 18 ? false : F.validators.uid.test(this); }; -String.prototype.parseInt = function(def) { +SP.parseInt = function(def) { var str = this.trim(); var num = +str; return isNaN(num) ? (def || 0) : num; }; -String.prototype.parseInt2 = function(def) { +SP.parseInt2 = function(def) { var num = this.match(regexpINTEGER); return num ? +num[0] : def || 0; }; -String.prototype.parseFloat2 = function(def) { +SP.parseFloat2 = function(def) { var num = this.match(regexpFLOAT); return num ? +num[0].toString().replace(/,/g, '.') : def || 0; }; -String.prototype.parseBool = String.prototype.parseBoolean = function() { +SP.parseBool = SP.parseBoolean = function() { var self = this.toLowerCase(); return self === 'true' || self === '1' || self === 'on'; }; -String.prototype.parseFloat = function(def) { +SP.parseFloat = function(def) { var str = this.trim(); if (str.indexOf(',') !== -1) str = str.replace(',', '.'); @@ -3722,7 +3728,7 @@ String.prototype.parseFloat = function(def) { return isNaN(num) ? (def || 0) : num; }; -String.prototype.capitalize = function(first) { +SP.capitalize = function(first) { if (first) return this[0].toUpperCase() + this.substring(1); @@ -3743,7 +3749,7 @@ String.prototype.capitalize = function(first) { }; -String.prototype.toUnicode = function() { +SP.toUnicode = function() { var result = ''; var self = this; var length = self.length; @@ -3756,35 +3762,35 @@ String.prototype.toUnicode = function() { return result; }; -String.prototype.fromUnicode = function() { +SP.fromUnicode = function() { return unescape(this.replace(regexpUNICODE, (match, v) => String.fromCharCode(parseInt(v, 16)))); }; -String.prototype.sha1 = function(salt) { +SP.sha1 = function(salt) { var hash = Crypto.createHash('sha1'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.sha256 = function(salt) { +SP.sha256 = function(salt) { var hash = Crypto.createHash('sha256'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.sha512 = function(salt) { +SP.sha512 = function(salt) { var hash = Crypto.createHash('sha512'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.md5 = function(salt) { +SP.md5 = function(salt) { var hash = Crypto.createHash('md5'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.toSearch = function() { +SP.toSearch = function() { var str = this.replace(regexpSEARCH, '').trim().toLowerCase().removeDiacritics(); var buf = []; var prev = ''; @@ -3801,7 +3807,7 @@ String.prototype.toSearch = function() { return buf.join(''); }; -String.prototype.toKeywords = String.prototype.keywords = function(forSearch, alternative, max_count, max_length, min_length) { +SP.toKeywords = SP.keywords = function(forSearch, alternative, max_count, max_length, min_length) { return exports.keywords(this, forSearch, alternative, max_count, max_length, min_length); }; @@ -3812,7 +3818,7 @@ function checksum(val) { return sum; } -String.prototype.encrypt = function(key, isUnique, secret) { +SP.encrypt = function(key, isUnique, secret) { var str = '0' + this; var data_count = str.length; var key_count = key.length; @@ -3839,7 +3845,7 @@ String.prototype.encrypt = function(key, isUnique, secret) { return (sum + checksum((secret || F.config.secret) + key)) + '-' + str; }; -String.prototype.decrypt = function(key, secret) { +SP.decrypt = function(key, secret) { var index = this.indexOf('-'); if (index === -1) @@ -3912,7 +3918,7 @@ exports.decryptUID = function(val, key) { return exports.encryptUID(raw, key) === val ? raw : null; }; -String.prototype.base64ToFile = function(filename, callback) { +SP.base64ToFile = function(filename, callback) { var self = this; var index = self.indexOf(','); if (index === -1) @@ -3923,7 +3929,7 @@ String.prototype.base64ToFile = function(filename, callback) { return this; }; -String.prototype.base64ToBuffer = function() { +SP.base64ToBuffer = function() { var self = this; var index = self.indexOf(','); @@ -3935,17 +3941,17 @@ String.prototype.base64ToBuffer = function() { return exports.createBuffer(self.substring(index), 'base64'); }; -String.prototype.base64ContentType = function() { +SP.base64ContentType = function() { var self = this; var index = self.indexOf(';'); return index === -1 ? '' : self.substring(5, index); }; -String.prototype.removeDiacritics = function() { +SP.removeDiacritics = function() { return exports.removeDiacritics(this); }; -String.prototype.indent = function(max, c) { +SP.indent = function(max, c) { var plus = ''; if (c === undefined) c = ' '; @@ -3954,7 +3960,7 @@ String.prototype.indent = function(max, c) { return plus + this; }; -String.prototype.isNumber = function(isDecimal) { +SP.isNumber = function(isDecimal) { var self = this; var length = self.length; @@ -3981,8 +3987,8 @@ String.prototype.isNumber = function(isDecimal) { return true; }; -if (!String.prototype.padLeft) { - String.prototype.padLeft = function(max, c) { +if (!SP.padLeft) { + SP.padLeft = function(max, c) { var self = this; var len = max - self.length; if (len < 0) @@ -3996,8 +4002,8 @@ if (!String.prototype.padLeft) { } -if (!String.prototype.padRight) { - String.prototype.padRight = function(max, c) { +if (!SP.padRight) { + SP.padRight = function(max, c) { var self = this; var len = max - self.length; if (len < 0) @@ -4010,7 +4016,7 @@ if (!String.prototype.padRight) { }; } -String.prototype.insert = function(index, value) { +SP.insert = function(index, value) { var str = this; var a = str.substring(0, index); var b = value.toString() + str.substring(index); @@ -4022,7 +4028,7 @@ String.prototype.insert = function(index, value) { * @param {Number} max A maximum length, default: 60 and optional. * @return {String} */ -String.prototype.slug = String.prototype.toSlug = String.prototype.toLinker = String.prototype.linker = function(max) { +SP.slug = SP.toSlug = SP.toLinker = SP.linker = function(max) { max = max || 60; var self = this.trim().toLowerCase().removeDiacritics(); @@ -4049,11 +4055,11 @@ String.prototype.slug = String.prototype.toSlug = String.prototype.toLinker = St return builder[l] === '-' ? builder.substring(0, l) : builder; }; -String.prototype.pluralize = function(zero, one, few, other) { +SP.pluralize = function(zero, one, few, other) { return this.parseInt().pluralize(zero, one, few, other); }; -String.prototype.isBoolean = function() { +SP.isBoolean = function() { var self = this.toLowerCase(); return (self === 'true' || self === 'false') ? true : false; }; @@ -4062,11 +4068,11 @@ String.prototype.isBoolean = function() { * Check if the string contains only letters and numbers. * @return {Boolean} */ -String.prototype.isAlphaNumeric = function() { +SP.isAlphaNumeric = function() { return regexpALPHA.test(this); }; -String.prototype.soundex = function() { +SP.soundex = function() { var arr = this.toLowerCase().split(''); var first = arr.shift(); @@ -4090,23 +4096,25 @@ String.prototype.soundex = function() { * Remove all Html Tags from a string * @return {string} */ -String.prototype.removeTags = function() { +SP.removeTags = function() { return this.replace(regexpTags, ''); }; -Number.prototype.floor = function(decimals) { +const NP = Number.prototype; + +NP.floor = function(decimals) { return Math.floor(this * Math.pow(10, decimals)) / Math.pow(10, decimals); }; -Number.prototype.padLeft = function(max, c) { +NP.padLeft = function(max, c) { return this.toString().padLeft(max, c || '0'); }; -Number.prototype.padRight = function(max, c) { +NP.padRight = function(max, c) { return this.toString().padRight(max, c || '0'); }; -Number.prototype.round = function(precision) { +NP.round = function(precision) { var m = Math.pow(10, precision) || 1; return Math.round(this * m) / m; }; @@ -4117,7 +4125,7 @@ Number.prototype.round = function(precision) { * @param {Function} callback * @return {Number} */ -Number.prototype.async = function(fn, callback) { +NP.async = function(fn, callback) { var number = this; if (number) fn(number--, () => setImmediate(() => number.async(fn, callback))); @@ -4133,7 +4141,7 @@ Number.prototype.async = function(fn, callback) { * @param {String} separatorDecimal Decimal separator, default '.' if number separator is ',' or ' '. * @return {String} */ -Number.prototype.format = function(decimals, separator, separatorDecimal) { +NP.format = function(decimals, separator, separatorDecimal) { var self = this; @@ -4184,7 +4192,7 @@ Number.prototype.format = function(decimals, separator, separatorDecimal) { return minus + output + (dec.length ? separatorDecimal + dec : ''); }; -Number.prototype.add = function(value, decimals) { +NP.add = function(value, decimals) { if (value == null) return this; @@ -4254,7 +4262,7 @@ Number.prototype.add = function(value, decimals) { return num; }; -Number.prototype.format2 = function(format) { +NP.format2 = function(format) { var index = 0; var num = this.toString(); var beg = 0; @@ -4363,7 +4371,7 @@ Number.prototype.format2 = function(format) { return this.format(output); }; -Number.prototype.pluralize = function(zero, one, few, other) { +NP.pluralize = function(zero, one, few, other) { var num = this; var value = ''; @@ -4386,14 +4394,14 @@ Number.prototype.pluralize = function(zero, one, few, other) { return num.format(format) + value.replace(format, ''); }; -Number.prototype.hex = function(length) { +NP.hex = function(length) { var str = this.toString(16).toUpperCase(); while(str.length < length) str = '0' + str; return str; }; -Number.prototype.VAT = function(percentage, decimals, includedVAT) { +NP.VAT = function(percentage, decimals, includedVAT) { var num = this; var type = typeof(decimals); @@ -4415,25 +4423,25 @@ Number.prototype.VAT = function(percentage, decimals, includedVAT) { return includedVAT ? (num / ((percentage / 100) + 1)).floor(decimals) : (num * ((percentage / 100) + 1)).floor(decimals); }; -Number.prototype.discount = function(percentage, decimals) { +NP.discount = function(percentage, decimals) { var num = this; if (decimals === undefined) decimals = 2; return (num - (num / 100) * percentage).floor(decimals); }; -Number.prototype.parseDate = function(plus) { +NP.parseDate = function(plus) { return new Date(this + (plus || 0)); }; -if (!Number.prototype.toRad) { - Number.prototype.toRad = function () { +if (!NP.toRad) { + NP.toRad = function () { return this * Math.PI / 180; }; } -Number.prototype.filesize = function(decimals, type) { +NP.filesize = function(decimals, type) { if (typeof(decimals) === 'string') { var tmp = type; @@ -4501,12 +4509,14 @@ function filesizehelper(number, count) { return number; } +var AP = Array.prototype; + /** * Take items from array * @param {Number} count * @return {Array} */ -Array.prototype.take = function(count) { +AP.take = function(count) { var arr = []; var self = this; var length = self.length; @@ -4524,7 +4534,7 @@ Array.prototype.take = function(count) { * @param {Boolean} rewrite Default: false. * @return {Array} Returns self */ -Array.prototype.extend = function(obj, rewrite) { +AP.extend = function(obj, rewrite) { var isFn = typeof(obj) === 'function'; for (var i = 0, length = this.length; i < length; i++) { if (isFn) @@ -4540,7 +4550,7 @@ Array.prototype.extend = function(obj, rewrite) { * @param {Object} def Default value. * @return {Object} */ -Array.prototype.first = function(def) { +AP.first = function(def) { var item = this[0]; return item === undefined ? def : item; }; @@ -4550,7 +4560,7 @@ Array.prototype.first = function(def) { * @param {String} name Optional, property name. * @return {Object} */ -Array.prototype.toObject = function(name) { +AP.toObject = function(name) { var self = this; var obj = {}; @@ -4572,7 +4582,7 @@ Array.prototype.toObject = function(name) { * @param {Array} b Second array. * @param {Function(itemA, itemB, indexA, indexB)} executor */ -Array.prototype.compare = function(id, b, executor) { +AP.compare = function(id, b, executor) { var a = this; var ak = {}; @@ -4634,7 +4644,7 @@ Array.prototype.compare = function(id, b, executor) { * @param {Boolean} remove Optional, remove item from this array if the item doesn't exist int arr (default: false). * @return {Array} */ -Array.prototype.pair = function(property, arr, fn, remove) { +AP.pair = function(property, arr, fn, remove) { if (property instanceof Array) { var tmp = property; @@ -4678,12 +4688,12 @@ Array.prototype.pair = function(property, arr, fn, remove) { * @param {Object} def Default value. * @return {Object} */ -Array.prototype.last = function(def) { +AP.last = function(def) { var item = this[this.length - 1]; return item === undefined ? def : item; }; -Array.prototype.quicksort = Array.prototype.orderBy = function(name, asc) { +AP.quicksort = AP.orderBy = function(name, asc) { var length = this.length; if (!length || length === 1) @@ -4750,7 +4760,7 @@ Array.prototype.quicksort = Array.prototype.orderBy = function(name, asc) { return self; }; -Array.prototype.trim = function() { +AP.trim = function() { var self = this; var output = []; for (var i = 0, length = self.length; i < length; i++) { @@ -4766,7 +4776,7 @@ Array.prototype.trim = function() { * @param {Number} count * @return {Array} */ -Array.prototype.skip = function(count) { +AP.skip = function(count) { var arr = []; var self = this; var length = self.length; @@ -4781,7 +4791,7 @@ Array.prototype.skip = function(count) { * @param {Object} value Optional. * @return {Array} */ -Array.prototype.where = function(cb, value) { +AP.where = function(cb, value) { var self = this; var selected = []; @@ -4812,7 +4822,7 @@ Array.prototype.where = function(cb, value) { * @param {Object} value Optional. * @return {Array} */ -Array.prototype.findItem = Array.prototype.find = function(cb, value) { +AP.findItem = AP.find = function(cb, value) { var self = this; var index = self.findIndex(cb, value); if (index === -1) @@ -4820,7 +4830,7 @@ Array.prototype.findItem = Array.prototype.find = function(cb, value) { return self[index]; }; -Array.prototype.findIndex = function(cb, value) { +AP.findIndex = function(cb, value) { var self = this; var isFN = typeof(cb) === 'function'; @@ -4853,7 +4863,7 @@ Array.prototype.findIndex = function(cb, value) { * @param {Object} value Optional. * @return {Array} */ -Array.prototype.remove = function(cb, value) { +AP.remove = function(cb, value) { var self = this; var arr = []; @@ -4877,7 +4887,7 @@ Array.prototype.remove = function(cb, value) { return arr; }; -Array.prototype.wait = Array.prototype.waitFor = function(onItem, callback, thread, tmp) { +AP.wait = AP.waitFor = function(onItem, callback, thread, tmp) { var self = this; var init = false; @@ -4931,7 +4941,7 @@ function next_wait(self, onItem, callback, thread, tmp) { * @param {Function} callback Optional * @return {Array} */ -Array.prototype.async = function(thread, callback, pending) { +AP.async = function(thread, callback, pending) { var self = this; @@ -4970,13 +4980,13 @@ Array.prototype.async = function(thread, callback, pending) { return self; }; -Array.prototype.randomize = function() { +AP.randomize = function() { OBSOLETE('Array.randomize()', 'Use Array.random().'); return this.random(); }; // Fisher-Yates shuffle -Array.prototype.random = function() { +AP.random = function() { for (var i = this.length - 1; i > 0; i--) { var j = Math.floor(Math.random() * (i + 1)); var temp = this[i]; @@ -4986,7 +4996,7 @@ Array.prototype.random = function() { return this; }; -Array.prototype.limit = function(max, fn, callback, index) { +AP.limit = function(max, fn, callback, index) { if (index === undefined) index = 0; @@ -5031,7 +5041,7 @@ Array.prototype.limit = function(max, fn, callback, index) { * Get unique elements from Array * @return {[type]} [description] */ -Array.prototype.unique = function(property) { +AP.unique = function(property) { var self = this; var result = []; @@ -5210,21 +5220,23 @@ Async.prototype = { } }; -Async.prototype.__proto__ = Object.create(Events.EventEmitter.prototype, { +const ACP = Async.prototype; + +ACP.__proto__ = Object.create(Events.EventEmitter.prototype, { constructor: { value: Async, enumberable: false } }); -Async.prototype.reload = function() { +ACP.reload = function() { var self = this; self.tasksAll = Object.keys(self.tasksPending); self.emit('percentage', self.percentage); return self; }; -Async.prototype.cancel = function(name) { +ACP.cancel = function(name) { var self = this; @@ -5250,7 +5262,7 @@ Async.prototype.cancel = function(name) { return true; }; -Async.prototype.await = function(name, fn, cb) { +ACP.await = function(name, fn, cb) { var self = this; @@ -5273,7 +5285,7 @@ Async.prototype.await = function(name, fn, cb) { return true; }; -Async.prototype.wait = function(name, waitingFor, fn, cb) { +ACP.wait = function(name, waitingFor, fn, cb) { var self = this; @@ -5296,34 +5308,34 @@ Async.prototype.wait = function(name, waitingFor, fn, cb) { return true; }; -Async.prototype.complete = function(fn) { +ACP.complete = function(fn) { return this.run(fn); }; -Async.prototype.run = function(fn) { +ACP.run = function(fn) { this._isRunning = true; fn && this.onComplete.push(fn); this.refresh(); return this; }; -Async.prototype.isRunning = function(name) { +ACP.isRunning = function(name) { if (!name) return this._isRunning; var task = this.tasksPending[name]; return task ? task.isRunning === 1 : false; }; -Async.prototype.isWaiting = function(name) { +ACP.isWaiting = function(name) { var task = this.tasksPending[name]; return task ? task.isRunning === 0 : false; }; -Async.prototype.isPending = function(name) { +ACP.isPending = function(name) { return this.tasksPending[name] ? true : false; }; -Async.prototype.timeout = function(name, timeout) { +ACP.timeout = function(name, timeout) { if (timeout) this.tasksTimeout[name] = timeout; else @@ -5331,7 +5343,7 @@ Async.prototype.timeout = function(name, timeout) { return this; }; -Async.prototype.refresh = function(name) { +ACP.refresh = function(name) { var self = this; @@ -5398,14 +5410,16 @@ function FileList() { this.advanced = false; } -FileList.prototype.reset = function() { +const FLP = FileList.prototype; + +FLP.reset = function() { this.file.length = 0; this.directory.length = 0; this.pendingDirectory.length = 0; return this; }; -FileList.prototype.walk = function(directory) { +FLP.walk = function(directory) { var self = this; @@ -5427,7 +5441,7 @@ FileList.prototype.walk = function(directory) { }); }; -FileList.prototype.stat = function(path) { +FLP.stat = function(path) { var self = this; Fs.stat(path, function(err, stats) { @@ -5448,11 +5462,11 @@ FileList.prototype.stat = function(path) { }); }; -FileList.prototype.clean = function(path) { +FLP.clean = function(path) { return path[path.length - 1] === '/' ? path : path + '/'; }; -FileList.prototype.next = function() { +FLP.next = function() { var self = this; if (self.pending.length) { @@ -5856,7 +5870,9 @@ function EventEmitter2() { this.$events = {}; } -EventEmitter2.prototype.emit = function(name, a, b, c, d, e, f, g) { +const EE2P = EventEmitter2.prototype; + +EE2P.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -5876,7 +5892,7 @@ EventEmitter2.prototype.emit = function(name, a, b, c, d, e, f, g) { return this; }; -EventEmitter2.prototype.on = function(name, fn) { +EE2P.on = function(name, fn) { if (this.$events[name]) this.$events[name].push(fn); else @@ -5884,12 +5900,12 @@ EventEmitter2.prototype.on = function(name, fn) { return this; }; -EventEmitter2.prototype.once = function(name, fn) { +EE2P.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -EventEmitter2.prototype.removeListener = function(name, fn) { +EE2P.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -5901,7 +5917,7 @@ EventEmitter2.prototype.removeListener = function(name, fn) { return this; }; -EventEmitter2.prototype.removeAllListeners = function(name) { +EE2P.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -5928,7 +5944,9 @@ function Chunker(name, max) { this.filename = F.path.temp(this.filename); } -Chunker.prototype.append = Chunker.prototype.write = function(obj) { +const CHP = Chunker.prototype; + +CHP.append = CHP.write = function(obj) { var self = this; self.stack.push(obj); @@ -5956,7 +5974,7 @@ Chunker.prototype.append = Chunker.prototype.write = function(obj) { return self; }; -Chunker.prototype.end = function() { +CHP.end = function() { var self = this; var tmp = self.stack.length; if (tmp) { @@ -5979,7 +5997,7 @@ Chunker.prototype.end = function() { return self; }; -Chunker.prototype.each = function(onItem, onEnd, indexer) { +CHP.each = function(onItem, onEnd, indexer) { var self = this; @@ -5999,7 +6017,7 @@ Chunker.prototype.each = function(onItem, onEnd, indexer) { return self; }; -Chunker.prototype.read = function(index, callback) { +CHP.read = function(index, callback) { var self = this; if (self.flushing) { @@ -6034,7 +6052,7 @@ Chunker.prototype.read = function(index, callback) { return self; }; -Chunker.prototype.clear = function() { +CHP.clear = function() { var files = []; for (var i = 0; i < this.index; i++) files.push(this.filename + i + '.chunker'); @@ -6042,7 +6060,7 @@ Chunker.prototype.clear = function() { return this; }; -Chunker.prototype.destroy = function() { +CHP.destroy = function() { this.clear(); this.indexer = 0; this.flushing = 0; @@ -6079,13 +6097,14 @@ function Callback(count, callback) { this.pending = count; this.$callback = callback; } +const CP = Callback.prototype; -Callback.prototype.done = function(callback) { +CP.done = function(callback) { this.$callback = callback; return this; }; -Callback.prototype.next = function() { +CP.next = function() { var self = this; self.pending--; if (!self.pending && self.$callback) { @@ -6101,5 +6120,37 @@ exports.Callback = function(count, callback) { return new Callback(count, callback); }; +function Reader() {} +const RP = Reader.prototype; + +RP.push = function(data) { + if (data == null || !data.length) + this.reader.done(); + else + this.reader.compare(data); + return this; +}; + +RP.find = function() { + var self = this; + var builder = new framework_nosql.DatabaseBuilder(); + self.reader = new framework_nosql.NoSQLReader(builder); + return builder; +}; + +RP.count = function() { + var builder = this.find(); + builder.$options.readertype = 1; + return builder; +}; + +RP.scalar = function(type, field) { + return this.find().scalar(type, field); +}; + +exports.reader = function() { + return new Reader(); +}; + global.WAIT = exports.wait; !global.F && require('./index'); \ No newline at end of file From d7da68c32f10d3e97216439988ed99585f31b609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 00:02:50 +0200 Subject: [PATCH 0592/1669] New improvements again. --- changes.txt | 3 ++- utils.js | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 443b22595..e309a609e 100755 --- a/changes.txt +++ b/changes.txt @@ -1,7 +1,8 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` -- added: NoSQL storage `.find()` + `.count()` with multiple thread support +- added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support +- added: `U.reader()` - improved: NoSQL reader diff --git a/utils.js b/utils.js index a8b35f079..3c3822dd0 100755 --- a/utils.js +++ b/utils.js @@ -99,6 +99,10 @@ for (var i=0; i 22 && l < 30 && this[l] === 'Z' && this[10] === 'T' && this[4] === '-' && this[13] === ':' && this[16] === ':'; @@ -4100,8 +4100,6 @@ SP.removeTags = function() { return this.replace(regexpTags, ''); }; -const NP = Number.prototype; - NP.floor = function(decimals) { return Math.floor(this * Math.pow(10, decimals)) / Math.pow(10, decimals); }; From b2066a2ff4d9933fdf8b29b68beae9bb82b45c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 09:04:19 +0200 Subject: [PATCH 0593/1669] Updated bundles. --- bundles.js | 9 +++++++-- changes.txt | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bundles.js b/bundles.js index f390944cf..e04a46443 100755 --- a/bundles.js +++ b/bundles.js @@ -6,6 +6,7 @@ const CONSOLE = process.argv.indexOf('restart') === -1; const META = {}; const INTERNAL = { '/sitemap': 1, '/versions': 1, '/workflows': 1, '/dependencies': 1, '/config': 1, '/config-release': 1, '/config-debug': 1 }; const isWindows = require('os').platform().substring(0, 3).toLowerCase() === 'win'; +const REGAPPEND = /\/--[a-z0-9]+/i; META.version = 1; META.created = new Date(); @@ -112,11 +113,15 @@ exports.make = function(callback) { for (var i = 0, length = files.length; i < length; i++) { var file = files[i].substring(Length); var type = 0; - if (file.startsWith(F.config['directory-databases'])) type = 1; else if (file.startsWith(F.config['directory-public'])) type = 2; + else if (REGAPPEND.test(file)) { + file = file.replace(/\/--/g, '/'); + type = 3; + } + Files.push({ name: file, filename: files[i], type: type }); } @@ -216,7 +221,7 @@ function copyFiles(files, callback) { var filename = Path.join(path, file.name); var exists = false; var ext = U.getExtension(file.name); - var append = false; + var append = file.type === 3; try { exists = Fs.statSync(filename) != null; diff --git a/changes.txt b/changes.txt index e309a609e..b8f85dfc5 100755 --- a/changes.txt +++ b/changes.txt @@ -3,6 +3,7 @@ - added: CSS variables support default values `border-radius: $radius || 10px` - added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support - added: `U.reader()` +- added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - improved: NoSQL reader From 48aa3625f60f14024519ecb1c561ca4bb631e5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 20:24:21 +0200 Subject: [PATCH 0594/1669] Fixed filtering. --- nosql.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nosql.js b/nosql.js index 51f477d4b..76b3ad46c 100755 --- a/nosql.js +++ b/nosql.js @@ -21,7 +21,7 @@ /** * @module NoSQL - * @version 3.0.0 + * @version 3.0.1 */ 'use strict'; @@ -1356,12 +1356,12 @@ DP.$inmemory = function(callback) { var arr = data.toString('utf8').split('\n'); for (var i = 0, length = arr.length; i < length; i++) { var item = arr[i]; - if (!item) - continue; - try { - item = JSON.parse(item.trim(), jsonparser); - item && self.inmemory[view].push(item); - } catch (e) {} + if (item) { + try { + item = JSON.parse(item.trim(), jsonparser); + item && self.inmemory[view].push(item); + } catch (e) {} + } } callback(); @@ -1646,6 +1646,7 @@ DP.$update_inmemory = function() { change && self.$save(); setImmediate(function() { self.next(0); + filters.done(); change && self.$events.change && self.emit('change', 'update'); }); }); @@ -1862,8 +1863,9 @@ function nosqlresort(arr, builder, doc) { DP.$reader2_inmemory = function(items, callback) { var self = this; - var filters = new NoSQLReader(items); return self.$inmemory(function() { + var filters = new NoSQLReader(items); + filters.clone = true; filters.compare(self.inmemory['#']); filters.done(); callback(); @@ -2255,7 +2257,8 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository if (err || !self.$join) { self.$options.log && self.log(); self.$done && setImmediate(self.$done); - return self.$callback(err, response, count, repository); + self.$callback && self.$callback(err, response, count, repository); + return; } self.$response = response; @@ -2280,8 +2283,8 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository } self.$options.log && self.log(); - self.$callback(err, response, count, repository); self.$done && setImmediate(self.$done); + self.$callback && self.$callback(err, response, count, repository); }); return self; @@ -6502,7 +6505,7 @@ NoSQLReader.prototype.compare = function(docs) { var self = this; for (var i = 0; i < docs.length; i++) { - var doc = docs[i]; + var doc = self.clone ? U.clone(docs[i]) : docs[i]; if (self.builders.length === self.canceled) return false; From 5e434c898b7208d653b2a2f510a302accc8d290e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 25 Aug 2018 11:26:57 +0200 Subject: [PATCH 0595/1669] Improved WebSocket. --- index.js | 90 ++++++++++++++++++++++++++++------------------ websocketclient.js | 3 ++ 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/index.js b/index.js index 2c666b859..f5516e3ac 100755 --- a/index.js +++ b/index.js @@ -7451,26 +7451,20 @@ F.$websocketcontinue = function(req, path) { var auth = F.onAuthorize; if (auth) { auth.call(F, req, req.websocket, req.flags, function(isLogged, user) { - if (user) req.user = user; - var route = F.lookup_websocket(req, req.websocket.uri.pathname, isLogged ? 1 : 2); if (route) { F.$websocketcontinue_process(route, req, path); - } else { - req.websocket.close(); - req.connection.destroy(); - } + } else + req.websocket.$close(4001, '401: unauthorized'); }); } else { var route = F.lookup_websocket(req, req.websocket.uri.pathname, 0); if (route) { F.$websocketcontinue_process(route, req, path); - } else { - req.websocket.close(); - req.connection.destroy(); - } + } else + req.websocket.$close(4004, '404: not found'); } }; @@ -7479,8 +7473,7 @@ F.$websocketcontinue_process = function(route, req, path) { var socket = req.websocket; if (!socket.prepare(route.flags, route.protocols, route.allow, route.length)) { - socket.close(); - req.connection.destroy(); + socket.$close(4001, '401: unauthorized'); return; } @@ -13583,6 +13576,28 @@ WebSocketClient.prototype.cookie = function(name) { return this.req.cookie(name); }; +WebSocketClient.prototype.$close = function(code, message) { + + var self = this; + + if ((self.req.headers['user-agent'] || '').indexOf('Total.js') !== -1) { + self.close(); + return; + } + + var header = SOCKET_RESPONSE.format(self.$websocket_key(self.req)); + self.socket.write(U.createBuffer(header, 'binary')); + self.ready = true; + self.close(message, code); + + setTimeout(function(self) { + self.req = null; + self.socket = null; + }, 1000, self); + + return self; +}; + WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { flags = flags || EMPTYARRAY; @@ -13623,6 +13638,7 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { var header = protocols.length ? (compress ? SOCKET_RESPONSE_PROTOCOL_COMPRESS : SOCKET_RESPONSE_PROTOCOL).format(self.$websocket_key(self.req), protocols.join(', ')) : (compress ? SOCKET_RESPONSE_COMPRESS : SOCKET_RESPONSE).format(self.$websocket_key(self.req)); self.socket.write(U.createBuffer(header, 'binary')); + self.ready = true; if (compress) { self.inflatepending = []; @@ -13630,10 +13646,10 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { self.inflate = Zlib.createInflateRaw(WEBSOCKET_COMPRESS_OPTIONS); self.inflate.$websocket = self; self.inflate.on('error', function() { - if (self.$uerror) - return; - self.$uerror = true; - self.close('Unexpected error'); + if (!self.$uerror) { + self.$uerror = true; + self.close('Invalid data', 1003); + } }); self.inflate.on('data', websocket_inflate); @@ -13642,10 +13658,10 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { self.deflate = Zlib.createDeflateRaw(WEBSOCKET_COMPRESS_OPTIONS); self.deflate.$websocket = self; self.deflate.on('error', function() { - if (self.$uerror) - return; - self.$uerror = true; - self.close('Unexpected error'); + if (!self.$uerror) { + self.$uerror = true; + self.close('Invalid data', 1003); + } }); self.deflate.on('data', websocket_deflate); } @@ -13833,9 +13849,8 @@ WebSocketClient.prototype.$parse = function() { // total message length (data + header) var mlength = index + length; - // ??? if (mlength > this.length) { - this.close('Maximum request length exceeded.'); + this.close('Frame is too large', 1009); return; } @@ -13963,7 +13978,7 @@ WebSocketClient.prototype.parseInflate = function() { self.inflatelock = false; if (data.length > self.length) { - self.close('Maximum request length exceeded.'); + self.close('Frame is too large', 1009); return; } @@ -14062,14 +14077,14 @@ WebSocketClient.prototype.sendDeflate = function() { self.deflatelock = true; self.deflate.write(buf); self.deflate.flush(function() { - if (!self.deflatechunks) - return; - var data = buffer_concat(self.deflatechunks, self.deflatechunkslength); - data = data.slice(0, data.length - 4); - self.deflatelock = false; - self.deflatechunks = null; - self.socket.write(U.getWebSocketFrame(0, data, self.type === 1 ? 0x02 : 0x01, true)); - self.sendDeflate(); + if (self.deflatechunks) { + var data = buffer_concat(self.deflatechunks, self.deflatechunkslength); + data = data.slice(0, data.length - 4); + self.deflatelock = false; + self.deflatechunks = null; + self.socket.write(U.getWebSocketFrame(0, data, self.type === 1 ? 0x02 : 0x01, true)); + self.sendDeflate(); + } }); } }; @@ -14093,11 +14108,16 @@ WebSocketClient.prototype.ping = function() { * @return {WebSocketClient} */ WebSocketClient.prototype.close = function(message, code) { - if (!this.isClosed) { - this.isClosed = true; - this.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + var self = this; + if (!self.isClosed) { + self.isClosed = true; + if (self.ready) + self.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + else + self.socket.end(); + self.req.connection.destroy(); } - return this; + return self; }; /** diff --git a/websocketclient.js b/websocketclient.js index 5cbb89827..5ecce29b6 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -69,6 +69,7 @@ WebSocketClient.prototype.connect = function(url, protocol, origin) { options.path = url.path; options.query = url.query; options.headers = {}; + options.headers['User-Agent'] = 'Total.js/v' + F.version_header; options.headers['Sec-WebSocket-Version'] = '13'; options.headers['Sec-WebSocket-Key'] = key; options.headers['Sec-Websocket-Extensions'] = (self.options.compress ? 'permessage-deflate, ' : '') + 'client_max_window_bits'; @@ -157,6 +158,8 @@ function websocket_close() { ws.closed = true; ws.$onclose(); ws.options.reconnect && setTimeout(function(ws) { + ws.isClosed = false; + ws._isClosed = false; ws.reconnect++; ws.connect(ws.url, ws.protocol, ws.origin); }, ws.options.reconnect, ws); From 30dea7b6ac4bf955ad62eb0e3c235091693c90ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 25 Aug 2018 11:34:15 +0200 Subject: [PATCH 0596/1669] Updated `debug.js` by changing `debug.pid`. --- debug.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/debug.js b/debug.js index d7363e4f8..609821efa 100644 --- a/debug.js +++ b/debug.js @@ -21,7 +21,7 @@ /** * @module FrameworkDebug - * @version 3.0.0 + * @version 3.0.1 */ const Path = require('path'); @@ -91,6 +91,7 @@ function runwatching() { const isRELOAD = !!options.livereload; const SPEED = isRELOAD ? 1000 : 1500; const ARGV = CLONE(process.argv); + const PIDNAME = FILENAME.replace(/\.js$/, '.pid'); function copyFile(oldname, newname, callback) { var writer = Fs.createWriteStream(newname); @@ -149,6 +150,7 @@ function runwatching() { var WS = null; var speed = isRELOAD ? 1000 : 4000; + blacklist['/' + PIDNAME] = 1; blacklist['/debug.pid'] = 1; blacklist['/debug.js'] = 1; blacklist['/bundle.json'] = 1; @@ -408,7 +410,7 @@ function runwatching() { console.log(prefix.substring(8) + 'DEBUG PID: ' + process.pid + ' (v' + VERSION + ')'); - pid = Path.join(directory, 'debug.pid'); + pid = Path.join(directory, PIDNAME); Fs.writeFileSync(pid, process.pid); setInterval(function() { @@ -434,7 +436,7 @@ function runwatching() { refresh_directory(); } - var filename = Path.join(directory, 'debug.pid'); + var filename = Path.join(directory, PIDNAME); if (Fs.existsSync(filename)) { Fs.unlinkSync(filename); setTimeout(app, 3500); From b2953d8d38ec7c7f89d41809c9f22ce88a6b1d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 25 Aug 2018 11:34:19 +0200 Subject: [PATCH 0597/1669] Added new changes. --- changes.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index b8f85dfc5..69b618c0e 100755 --- a/changes.txt +++ b/changes.txt @@ -5,12 +5,15 @@ - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` -- improved: NoSQL reader +- updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories +- fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) + +- improved: NoSQL reader ======= 3.0.0 From 02912d2ae787f0408a438bfc70f922b0f4dc954f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 13:40:31 +0200 Subject: [PATCH 0598/1669] Fixed `.remove()` method in `TABLE`. --- nosql.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 76b3ad46c..ada884b06 100755 --- a/nosql.js +++ b/nosql.js @@ -5953,13 +5953,13 @@ TP.$remove = function() { var data = { keys: self.$keys }; var remove = function(docs, d, dindex, f) { - var rec = fs.docsbuffer[dindex + 1]; + var rec = fs.docsbuffer[dindex]; f.builder.$options.backup && f.builder.$backupdoc(rec.doc); return 1; }; var removeflush = function(docs, d, dindex) { - var rec = fs.docsbuffer[dindex + 1]; + var rec = fs.docsbuffer[dindex]; !change && (change = true); self.$events.remove && self.emit('remove', d); fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); @@ -5970,6 +5970,9 @@ TP.$remove = function() { var lines = fs.docs.split(fs.divider); var arr = []; + if (!indexer) + arr.push(EMPTYOBJECT); + for (var a = indexer ? 0 : 1; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = indexer++; From 112b815c3126a2bc1e0db36c91826b69c9e9f2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 20:26:06 +0200 Subject: [PATCH 0599/1669] Removed GraphDB. --- graphdb.js | 2582 ---------------------------------------------------- 1 file changed, 2582 deletions(-) delete mode 100644 graphdb.js diff --git a/graphdb.js b/graphdb.js deleted file mode 100644 index 1917c48f3..000000000 --- a/graphdb.js +++ /dev/null @@ -1,2582 +0,0 @@ -// Copyright 2012-2018 (c) Peter Širka -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -/** - * @module FrameworkGraphDB - * @version 1.0.0 - */ - -const Fs = require('fs'); -const Zlib = require('zlib'); - -const ZLIBOPTIONS = { level: Zlib.constants.Z_FULL_FLUSH, memLevel: Zlib.constants.Z_BEST_COMPRESSION, strategy: Zlib.constants.Z_DEFAULT_STRATEGY }; -const VERSION = 1; -const DOCUMENTSIZE = 1000; -const PAGESIZE = 20; -const PAGELIMIT = 50; -const DATAOFFSET = 17; -const EMPTYBUFFER = U.createBufferSize(1); -const HEADERSIZE = 7000; -const DELAY = 100; -const REGTUNESCAPE = /%7C|%0D|%0A/g; -const REGTESCAPETEST = /\||\n|\r/; -const REGTESCAPE = /\||\n|\r/g; -const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; -const DatabaseBuilder = framework_nosql.DatabaseBuilder; - -// STATES -const STATE_UNCOMPRESSED = 1; -const STATE_COMPRESSED = 2; -const STATE_REMOVED = 255; - -// META -const META_PAGE_ADD = 100; -const META_CLASSESRELATIONS = 101; -const META_PAGE_ADD3 = 102; -const META_RELATIONPAGEINDEX = 103; - -// OPERATIONS -const NEXT_READY = 1; -const NEXT_INSERT = 2; -const NEXT_RELATION = 3; -const NEXT_UPDATE = 4; -const NEXT_FIND = 5; -const NEXT_REMOVE = 6; -const NEXT_RESIZE = 7; -const NEXT_CONTINUE = 100; - -// TYPES -const TYPE_CLASS = 1; -const TYPE_RELATION = 2; -const TYPE_RELATION_DOCUMENT = 3; - -var IMPORTATOPERATIONS = 0; - -function GraphDB(name) { - - F.path.verify('databases'); - - var self = this; - self.name = name; - self.filename = F.path.databases(name + '.gdb'); - self.filenameBackup = self.filename.replace(/\.gdb$/, '.gdp-backup'); - self.ready = false; - - self.$classes = {}; - self.$relations = {}; - self.$events = {}; - - self.header = {}; - - self.pending = {}; - self.pending.insert = []; - self.pending.find = []; - self.pending.update = []; - self.pending.remove = []; - self.pending.relation = []; - self.pending.meta = []; - - self.states = {}; - self.states.resize = false; - self.states.insert = false; - self.states.read = false; - self.states.remove = false; - self.states.update = false; - - F.path.verify('databases'); - // t.open(); - - self.cb_error = function(err) { - err && console.log(err); - }; - - self.cb_next = function(value) { - self.next(value); - }; - - F.grapdbinstance = true; - self.open(); -} - -var GP = GraphDB.prototype; - -// ==== DB:HEADER (7000b) -// name (30b) = from: 0 -// version (1b) = from: 30 -// pages (4b) = from: 31 -// pagesize (2b) = from: 35 -// pagelimit (2b) = from: 37 -// documents (4b) = from: 39 -// documentsize (2b) = from: 43 -// classindex (1b) = from: 45 -// relationindex (1b) = from: 46 -// relationnodeindex = from: 47 -// classes + relations = from: 51 - -// ==== DB:PAGE (20b) -// type (1b) = from: 0 -// index (1b) = from: 1 -// documents (2b) = from: 2 -// freeslots (1b) = from: 4 -// parentindex (4b) = from: 5 - -// ==== DB:DOCUMENT (SIZE) -// type (1b) = from: 0 -// index (1b) = from: 1 -// state (1b) = from: 2 -// pageindex (4b) = from: 3 -// relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) -// parentindex (4b) = from: 11 -// size/count (2b) = from: 15 -// data = from: 17 - -// Creates new page -function addPage(self, type, index, parentindex, callback) { - - // @type - // 1: classes - // 2: relations - // 3: relations private - - // @index - // index of value - - // Add a new page - self.header.pages++; - - var indexer = self.header.pages; - var buffer = []; - var page = U.createBufferSize(self.header.pagesize); - - // console.log('CREATING PAGE:', TYPES[type], indexer, type, index); - - page.writeUInt8(type, 0); // type (1:class, 2:relation, 3:private) - page.writeUInt8(index, 1); // index - page.writeUInt16LE(0, 2); // documents - page.writeUInt8(0, 4); // freeslots - page.writeUInt32LE(parentindex, 5); // parentindex - - buffer.push(page); - - for (var i = 0; i < self.header.pagelimit; i++) { - var doc = U.createBufferSize(self.header.documentsize); - doc.writeUInt8(type, 0); - doc.writeUInt8(index, 1); - doc.writeUInt8(STATE_REMOVED, 2); - doc.writeUInt32LE(self.header.pages, 3); - doc.writeUInt32LE(0, 7); // continuerindex - doc.writeUInt32LE(0, 11); // parentindex - doc.writeUInt16LE(0, 15); // size/count - buffer.push(doc); - } - - buffer = Buffer.concat(buffer); - - var offset = offsetPage(self, indexer); - - Fs.write(self.fd, buffer, 0, buffer.length, offset, function(err) { - err && self.error(err, 'createPage.write'); - !err && updMeta(self, type === TYPE_RELATION_DOCUMENT ? META_PAGE_ADD3 : META_PAGE_ADD); - callback && callback(err, indexer); - }); - - return indexer; -} - -function addNodeFree(self, meta, callback) { - - if (!meta.type.findfreeslots) { - addNode(self, meta, callback); - return; - } - - findDocumentFree(self, meta.type.pageindex, function(err, documentindex, pageindex) { - - if (!documentindex) { - meta.type.findfreeslots = false; - addNode(self, meta, callback); - return; - } - - var buffer = U.createBufferSize(self.header.documentsize); - buffer.writeUInt8(meta.typeid, 0); // type - buffer.writeUInt8(meta.type.index, 1); // index - buffer.writeUInt32LE(pageindex, 3); // pageindex - buffer.writeUInt8(meta.state || STATE_UNCOMPRESSED, 2); // state - buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex - buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex - buffer.writeUInt16LE(meta.size, 15); - meta.data && meta.data.copy(buffer, DATAOFFSET); - - Fs.write(self.fd, buffer, 0, buffer.length, offsetDocument(self, documentindex), function() { - meta.type.locked = false; - callback(null, documentindex, pageindex); - }); - }); -} - -function addNode(self, meta, callback) { - - // meta.typeid (1 CLASS, 2 RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.buffer - - var buf = U.createBufferSize(self.header.pagesize); - var offset = offsetPage(self, meta.type.pageindex); - - meta.type.locked = true; - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - - if (err) - throw err; - - if (buf[0] !== meta.typeid) - throw new Error('Not a class page'); - - if (!meta.type.private && buf[1] !== meta.type.index) - throw new Error('Not same class type'); - - // type : buf[0] - // index : buf[1] - // documents : buf.readUInt16LE(2) - // freeslots : buf[4] - // parentindex : readUInt32LE(5) - - var buffer = U.createBufferSize(self.header.documentsize); - buffer.writeUInt8(buf[0], 0); // type - buffer.writeUInt8(meta.type.index, 1); // index - buffer.writeUInt32LE(meta.type.pageindex, 3); // pageindex - buffer.writeUInt8(meta.state || STATE_UNCOMPRESSED, 2); // state - buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex - buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex - buffer.writeUInt16LE(meta.size, 15); - meta.data && meta.data.copy(buffer, DATAOFFSET); - - var documents = buf.readUInt16LE(2); - var documentsbuf = U.createBufferSize(2); - - documents++; - documentsbuf.writeUInt16LE(documents); - - Fs.write(self.fd, documentsbuf, 0, documentsbuf.length, offset + 2, function(err) { - - err && console.log('addNode.write.meta', err); - Fs.write(self.fd, buffer, 0, buffer.length, offset + self.header.pagesize + ((documents - 1) * self.header.documentsize), function(err) { - - err && console.log('addNode.write.data', err); - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // continuerindex (4b) = from: 7 - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - // We must create a new page - if (documents + 1 > self.header.pagelimit) { - addPage(self, meta.typeid, meta.type.index, meta.type.pageindex, function(err, index) { - - var documentindex = getDocumentIndex(self, meta.type.pageindex, documents); - meta.type.documentindex = documentindex; - meta.type.pageindex = index; - meta.type.locked = false; - - // Problem with classes - // meta.type.index = 0; - - if (meta.type.private) - self.header.relationpageindex = index; - - updMeta(self, meta.type.private ? META_RELATIONPAGEINDEX : META_CLASSESRELATIONS); - callback(null, documentindex, index); - }); - } else { - var documentindex = getDocumentIndex(self, meta.type.pageindex, documents); - meta.type.locked = false; - meta.type.documentindex = documentindex; - callback(null, documentindex, meta.type.pageindex); - } - }); - }); - }); -} - -function addDocument(self, cls, value, callback) { - - // meta.typeid (1 CLASS, 2 RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.data - - var meta = {}; - meta.type = cls; - meta.typeid = TYPE_CLASS; - meta.state = 1; - meta.parentindex = 0; - meta.relationindex = 0; - meta.data = U.createBuffer(stringifyData(cls.schema, value)); - meta.size = meta.data.length; - - var limit = self.header.documentsize - DATAOFFSET; - - if (meta.data.length > limit) { - Zlib.deflate(meta.data, ZLIBOPTIONS, function(err, buf) { - if (err || buf.length > limit) - callback(new Error('GraphDB: Data too long'), 0); - else { - meta.state = STATE_COMPRESSED; - meta.data = buf; - meta.size = buf.length; - addNodeFree(self, meta, callback); - } - }); - } else - addNodeFree(self, meta, callback); -} - -function addRelation(self, relation, indexA, indexB, callback) { - - // Workflow: - // Has "A" relation nodes? - // Has "B" relation nodes? - // Create "A" relation with "B" - // Create "B" relation with "A" - // Register relation to global relations - - var tasks = []; - var relA = null; - var relB = null; - - var tmprelation = { index: relation.index, pageindex: 0, documentindex: 0, locked: false, private: true }; - - tasks.push(function(next) { - self.read(indexA, function(err, doc, relid) { - if (doc) { - relA = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (A) "{0}" not exists.'.format(indexA))); - } - }); - }); - - tasks.push(function(next) { - self.read(indexB, function(err, doc, relid) { - if (doc) { - relB = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (B) "{0}" not exists.'.format(indexB))); - } - }); - }); - - tasks.push(function(next) { - - if (relA == 0) { - next(); - return; - } - - checkRelation(self, relation, relA, indexB, function(err, is) { - if (is) { - tasks = null; - next = null; - callback(new Error('GraphDB: Same relation already exists between nodes (A) "{0}" and (B) "{1}".'.format(indexA, indexB))); - } else - next(); - }); - }); - - // Obtaining indexA a relation document - tasks.push(function(next) { - - if (F.isKilled) - return; - - IMPORTATOPERATIONS++; - - if (relA) - next(); - else { - addRelationDocument(self, relation, indexA, function(err, index) { - relA = index; - next(); - }, true); - } - }); - - // Obtaining indexB a relation document - tasks.push(function(next) { - - if (F.isKilled) - return; - - if (relB) - next(); - else { - addRelationDocument(self, relation, indexB, function(err, index) { - relB = index; - next(); - }, true); - } - }); - - // Push "indexB" relation to "indexA" - tasks.push(function(next) { - tmprelation.documentindex = relA; - tmprelation.pageindex = self.header.relationpageindex; - pushRelationDocument(self, relA, tmprelation, indexB, true, function(err, index) { - // Updated relation, document was full - if (relA !== index) { - relA = index; - updDocumentRelation(self, indexA, relA, next); - } else - next(); - }, true); - }); - - tasks.push(function(next) { - tmprelation.documentindex = relB; - tmprelation.pageindex = self.header.relationpageindex; - pushRelationDocument(self, relB, tmprelation, indexA, false, function(err, index) { - // Updated relation, document was full - if (relB !== index) { - relB = index; - updDocumentRelation(self, indexB, relB, next); - } else - next(); - }, true); - }); - - tasks.push(function(next) { - // console.log('PUSH COMMON', relation.documentindex, indexA); - pushRelationDocument(self, relation.documentindex, relation, indexA, true, next); - }); - - tasks.async(function() { - IMPORTATOPERATIONS--; - // console.log('REL ====', relA, relB); - callback(null, true); - }); -} - -function remRelation(self, relation, indexA, indexB, callback) { - - var tasks = []; - var relA = null; - var relB = null; - - tasks.push(function(next) { - self.read(indexA, function(err, doc, relid) { - if (doc) { - relA = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (A) "{0}" not exists.'.format(indexA))); - } - }); - }); - - tasks.push(function(next) { - self.read(indexB, function(err, doc, relid) { - if (doc) { - relB = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (B) "{0}" not exists.'.format(indexB))); - } - }); - }); - - tasks.async(function() { - - if (F.isKilled) - return; - - IMPORTATOPERATIONS++; - remRelationLink(self, relA, indexB, function(err, countA) { - remRelationLink(self, relB, indexA, function(err, countB) { - remRelationLink(self, relation.documentindex, indexA, function(err, countC) { - IMPORTATOPERATIONS--; - callback(null, (countA + countB + countC) > 1); - }); - }); - }); - }); -} - -function remRelationLink(self, index, documentindex, callback, nochild, counter) { - - var buf = U.createBufferSize(self.header.documentsize); - var offset = offsetDocument(self, index); - - !counter && (counter = 0); - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - if ((buf[0] !== TYPE_RELATION && buf[0] !== TYPE_RELATION_DOCUMENT) || (buf[2] === STATE_REMOVED)) { - callback(null, counter); - return; - } - - var relid = buf.readUInt32LE(7); - var count = buf.readUInt16LE(15); - var arr = []; - var is = false; - - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - var obj = {}; - obj.INDEX = buf[off]; - obj.INIT = buf[off + 1]; - obj.ID = buf.readUInt32LE(off + 2); - if (obj.ID === documentindex && obj.INIT === 1) - is = true; - else - arr.push(obj); - } - - if (is) { - count = arr.length; - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - var obj = arr[i]; - buf.writeUInt8(obj.INDEX, off); - buf.writeUInt8(obj.INIT, off + 1); - buf.writeUInt32LE(obj.ID, off + 2); - } - buf.writeUInt16LE(count, 15); - buf.fill(EMPTYBUFFER, DATAOFFSET + ((count + 1) * 6)); - Fs.write(self.fd, buf, 0, buf.length, offset, function() { - counter++; - if (relid && !nochild) - setImmediate(remRelationLink, self, relid, documentindex, callback, null, counter); - else - callback(null, counter); - }); - } else if (relid && !nochild) - setImmediate(remRelationLink, self, relid, documentindex, callback, null, counter); - else - callback(null, counter); - }); -} - -// Traverses all RELATIONS documents and remove specific "documentindex" -function remRelationAll(self, index, documentindex, callback, counter) { - - var buf = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - var offset = offsetDocument(self, index); - - !counter && (counter = 0); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err || !size) { - callback(null, counter); - return; - } - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - var removed = []; - - while (true) { - - if (!buf.length) - break; - - index++; - - var data = buf.slice(0, self.header.documentsize); - - if ((data[0] !== TYPE_RELATION && data[0] !== TYPE_RELATION_DOCUMENT) || (data[2] === STATE_REMOVED)) { - buf = buf.slice(self.header.documentsize); - continue; - } - - var count = data.readUInt16LE(15); - var arr = []; - var is = false; - - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - var obj = {}; - obj.INDEX = data[off]; - obj.INIT = data[off + 1]; - obj.ID = data.readUInt32LE(off + 2); - if (obj.ID === documentindex) - is = true; - else - arr.push(obj); - } - - if (is) { - - var newcount = arr.length; - - for (var i = 0; i < newcount; i++) { - var off = DATAOFFSET + (i * 6); - var obj = arr[i]; - data.writeUInt8(obj.INDEX, off); - data.writeUInt8(obj.INIT, off + 1); - data.writeUInt32LE(obj.ID, off + 2); - } - - data.writeUInt16LE(newcount, 15); - data.fill(EMPTYBUFFER, DATAOFFSET + ((newcount + 1) * 6)); - - removed.push({ index: index - 1, buf: data }); - } - - buf = buf.slice(self.header.documentsize); - } - - if (!removed.length) { - setImmediate(remRelationAll, self, index, documentindex, callback, counter); - return; - } - - counter += removed.length; - removed.wait(function(item, next) { - Fs.write(self.fd, item.buf, 0, item.buf.length, offsetDocument(self, item.index), next); - }, function() { - setImmediate(remRelationAll, self, index, documentindex, callback, counter); - }); - - }); -} - -function addRelationDocument(self, relation, index, callback, between) { - - // meta.typeid (1 CLASS, 2 RELATION, 3 PRIVATE RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.data - - var meta = {}; - meta.typeid = between ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; - meta.type = between ? { index: 0, pageindex: self.header.relationpageindex, documentindex: index, locked: false, private: true } : relation; - meta.state = 1; - meta.parentindex = 0; - meta.relationindex = 0; - meta.size = 0; - - // Creates a new node - addNode(self, meta, function(err, relationindex) { - - // Updates exiting document by updating relation index - updDocumentRelation(self, index, relationindex, function(err) { - // Returns a new relation index - callback(err, relationindex); - }); - }); -} - -function findDocumentFree(self, pageindex, callback, ready) { - - var offset = offsetPage(self, pageindex); - var buf = U.createBufferSize(self.header.pagesize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - // ==== DB:PAGE (20b) - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - var relid = buf.readUInt32LE(5); - if (!relid) { - if (!ready) { - callback(null, 0); - return; - } - } - - // First page is the last page saved in meta therefore is needed to perform recursive with "ready" - if (!ready) { - findDocumentFree(self, relid, callback, true); - return; - } - - var documents = buf.readUInt16LE(2); - if (documents >= self.header.pagelimit) { - // Finds in parent if exists - if (relid) - findDocumentFree(self, relid, callback, true); - else - callback(null, 0); - return; - } - - // Finds a free document slot - var index = getDocumentIndex(self, pageindex) - 1; - var buffer = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - - Fs.read(self.fd, buffer, 0, buffer.length, offset + self.header.pagesize, function() { - while (true) { - index++; - var data = buffer.slice(0, self.header.documentsize); - if (!data.length) - break; - - if (data[2] === STATE_REMOVED) { - - if (F.isKilled) - return; - - updPageMeta(self, pageindex, function(err, buf) { - buf.writeUInt16LE(documents + 1, 2); - setImmediate(callback, null, index, pageindex); - }); - buffer = buffer.slice(self.header.documentsize); - return; - } - } - - if (relid) - findDocumentFree(self, relid, callback, true); - else - callback(null, 0); - - }); - }); -} - -// Finds a free space for new relation in "pushRelationDocument" -function findRelationDocument(self, relid, callback) { - - if (!relid) { - callback(null, 0); - return; - } - - var offset = offsetDocument(self, relid); - var buf = U.createBufferSize(self.header.documentsize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err || !size) { - callback(err, 0); - return; - } - - var count = buf.readUInt16LE(15); - if (count + 1 > self.header.relationlimit) { - // Checks if the relation index has next relation - - if (relid === buf.readUInt32LE(7)) - return; - - relid = buf.readUInt32LE(7); - if (relid) - setImmediate(findRelationDocument, self, relid, callback); - else - callback(null, 0); - } else { - // Free space for this relation - callback(null, relid); - } - }); -} - -// Pushs "documentindex" to "index" document (document with all relations) -function pushRelationDocument(self, index, relation, documentindex, initializator, callback, between, recovered) { - - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(self.header.documentsize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - var count = buf.readUInt16LE(15); - if (count + 1 > self.header.relationlimit) { - findRelationDocument(self, buf.readUInt32LE(7), function(err, newindex) { - - // Is some relation document exist? - if (newindex && !recovered) { - pushRelationDocument(self, newindex, relation, documentindex, initializator, callback, between, true); - return; - } - - // meta.typeid (1 CLASS, 2 RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.buffer - - var meta = {}; - meta.typeid = relation.private ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; - meta.type = relation; - meta.state = STATE_UNCOMPRESSED; - meta.parentindex = 0; - meta.relationindex = index; - meta.size = 0; - - addNode(self, meta, function(err, docindex, pageindex) { - relation.pageindex = pageindex; - relation.documentindex = docindex; - updDocumentRelation(self, relation.documentindex, index, function() { - updDocumentParent(self, index, relation.documentindex, function() { - pushRelationDocument(self, relation.documentindex, relation, documentindex, initializator, callback, between); - }); - }); - }); - }); - - } else { - - var buffer = U.createBufferSize(6); - buffer.writeUInt8(relation.index, 0); - buffer.writeUInt8(initializator ? 1 : 0, 1); - buffer.writeUInt32LE(documentindex, 2); - buffer.copy(buf, DATAOFFSET + (count * 6)); - buf.writeUInt16LE(count + 1, 15); - - if (buf[2] === STATE_REMOVED) { - // We must update counts of documents in the page meta - var pageindex = Math.ceil(index / self.header.pagelimit); - updPageMeta(self, pageindex, function(err, buf) { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - buf.writeUInt16LE(buf.readUInt16LE(2) + 1, 2); - setImmediate(function() { - Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { - err && self.error(err, 'pushRelationDocument.read.write'); - callback(null, index); - }); - }); - }); - - buf.writeUInt8(STATE_UNCOMPRESSED, 2); - - } else { - // DONE - Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { - err && self.error(err, 'pushRelationDocument.read.write'); - callback(null, index); - }); - } - } - - }); -} - -function updDocumentRelation(self, index, relationindex, callback) { - - if (index === relationindex) - throw new Error('FET'); - - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(4); - buf.writeUInt32LE(relationindex); - Fs.write(self.fd, buf, 0, buf.length, offset + 7, callback); -} - -function updDocumentParent(self, index, parentindex, callback) { - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(4); - buf.writeUInt32LE(parentindex); - Fs.write(self.fd, buf, 0, buf.length, offset + 11, callback); -} - -function updPageMeta(self, index, fn) { - var offset = offsetPage(self, index); - var buf = U.createBufferSize(self.header.pagesize); - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - fn(null, buf); - Fs.write(self.fd, buf, 0, buf.length, offset, self.cb_error); - }); -} - -function remDocument(self) { - if (!self.ready || self.states.remove || !self.pending.remove.length || F.isKilled) - return; - self.states.remove = true; - var doc = self.pending.remove.shift(); - IMPORTATOPERATIONS++; - remRelationAll(self, doc.id, doc.id, function() { - remDocumentAll(self, doc.id, function(err, count) { - IMPORTATOPERATIONS--; - self.states.remove = false; - doc.callback && doc.callback(err, count); - setImmediate(self.cb_next, NEXT_REMOVE); - }); - }); -} - -function remDocumentAll(self, index, callback, count) { - - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(17); - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - if (!count) - count = 0; - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - var relid = buf.readUInt32LE(7); - - if (buf[2] === STATE_REMOVED) { - if (relid) - remDocumentAll(self, relid, callback, count); - else - callback(null, count); - return; - } - - buf.writeUInt8(STATE_REMOVED, 2); - buf.writeUInt16LE(0, 15); - - if (buf[0] === TYPE_CLASS) - self.$classes[buf[1]].findfreeslots = true; - - var pageindex = buf.readUInt32LE(3); - - Fs.write(self.fd, buf, 0, buf.length, offset, function() { - - // Updates "documents" in the current page - updPageMeta(self, pageindex, function(err, buf) { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - var documents = buf.readUInt16LE(2); - buf.writeUInt16LE(documents > 0 ? documents - 1 : documents, 2); - count++; - - setImmediate(function() { - if (relid) - remDocumentAll(self, relid, callback, count); - else - callback(null, count); - }); - }); - }); - }); -} - -function offsetPage(self, index) { - return HEADERSIZE + ((index - 1) * (self.header.pagesize + (self.header.pagelimit * self.header.documentsize))); -} - -function offsetDocument(self, index) { - var page = Math.ceil(index / self.header.pagelimit); - var offset = page * self.header.pagesize; - return HEADERSIZE + offset + ((index - 1) * self.header.documentsize); -} - -function getIndexPage(self, offset) { - return ((offset - HEADERSIZE) / (self.header.pagesize + (self.header.pagelimit * self.header.documentsize))); -} - -function getDocumentIndex(self, pageindex, count) { - return ((pageindex - 1) * self.header.pagelimit) + (count || 1); -} - -function checkRelation(self, relation, indexA, indexB, callback) { - - self.read(indexA, function(err, docs, relid) { - - if (docs) { - for (var i = 0; i < docs.length; i++) { - var doc = docs[i]; - if (doc.ID === indexB && (relation.both || doc.INIT)) { - callback(null, true); - return; - } - } - } - - if (relid) - setImmediate(checkRelation, self, relation, relid, indexB, callback); - else - callback(null, false); - }); -} - -function updMeta(self, type) { - var buf; - switch (type) { - - case META_PAGE_ADD: - buf = U.createBufferSize(4); - buf.writeUInt32LE(self.header.pages); - Fs.write(self.fd, buf, 0, buf.length, 31, self.cb_error); - break; - - case META_PAGE_ADD3: - buf = U.createBufferSize(4); - buf.writeUInt32LE(self.header.pages, 0); - Fs.write(self.fd, buf, 0, buf.length, 31, function() { - buf.writeUInt32LE(self.header.relationpageindex, 0); - Fs.write(self.fd, buf, 0, buf.length, 47, self.cb_error); - }); - break; - - case META_RELATIONPAGEINDEX: - buf = U.createBufferSize(4); - buf.writeUInt32LE(self.header.relationpageindex, 0); - Fs.write(self.fd, buf, 0, buf.length, 47, self.cb_error); - break; - - case META_CLASSESRELATIONS: - - var obj = {}; - obj.c = []; // classes - obj.r = []; // relations - - for (var i = 0; i < self.header.classindex; i++) { - var item = self.$classes[i + 1]; - obj.c.push({ n: item.name, i: item.index, p: item.pageindex, r: item.schema.raw, d: item.documentindex }); - } - - for (var i = 0; i < self.header.relationindex; i++) { - var item = self.$relations[i + 1]; - obj.r.push({ n: item.name, i: item.index, p: item.pageindex, b: item.both ? 1 :0, d: item.documentindex }); - } - - buf = U.createBufferSize(HEADERSIZE - 45); - buf.writeUInt8(self.header.classindex, 0); - buf.writeUInt8(self.header.relationindex, 1); - buf.writeUInt32LE(self.header.relationpageindex, 2); - buf.write(JSON.stringify(obj), 6); - Fs.write(self.fd, buf, 0, buf.length, 45, self.cb_error); - break; - } -} - -function insDocument(self) { - - if (!self.ready || self.states.insert || !self.pending.insert.length || F.isKilled) - return; - - var doc = self.pending.insert.shift(); - if (doc) { - - var cls = self.$classes[doc.name]; - if (cls == null) { - doc.callback(new Error('GraphDB: Class "{0}" not found.'.format(doc.name))); - return; - } - - if (cls.locked || !cls.ready) { - self.pending.insert.push(doc); - setTimeout(self.cb_next, DELAY, NEXT_INSERT); - return; - } - - self.states.insert = true; - - addDocument(self, cls, doc.value, function(err, id) { - // setTimeout(insDocument, 100, self); - self.states.insert = false; - setImmediate(insDocument, self); - doc.callback(err, id); - }); - } -} - -function updDocument(self) { - - if (!self.ready || self.states.update || !self.pending.update.length || F.isKilled) - return; - - var upd = self.pending.update.shift(); - if (upd) { - self.states.update = true; - - var offset = offsetDocument(self, upd.id); - var buf = U.createBufferSize(self.header.documentsize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err) { - self.states.update = false; - upd.callback(err); - setImmediate(self.cb_next, NEXT_UPDATE); - return; - } - - if (!size) { - upd.callback(null, 0); - self.states.update = false; - setImmediate(self.cb_next, NEXT_UPDATE); - return; - } - - var save = function(err) { - self.states.update = false; - !err && Fs.write(self.fd, buf, 0, buf.length, offset, self.cb_error); - upd.callback(err, err ? 0 : 1); - setImmediate(self.cb_next, NEXT_UPDATE); - }; - - var data = buf.slice(DATAOFFSET, buf.readUInt16LE(15) + DATAOFFSET); - var limit = self.header.documentsize - DATAOFFSET; - var schema = self.$classes[buf[1]].schema; - var doc; - - if (buf[2] === STATE_COMPRESSED) { - Zlib.inflate(data, ZLIBOPTIONS, function(err, buffer) { - doc = parseData(schema, buffer.toString('utf8').split('|')); - buffer = U.createBuffer(stringifyData(schema, upd.fn(doc, upd.value))); - if (buffer.length > limit) { - Zlib.deflate(buffer, ZLIBOPTIONS, function(err, buffer) { - if (buffer.length <= limit) { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_COMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } else - save(new Error('GraphDB: Data too long')); - }); - } else { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_UNCOMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } - }); - } else { - doc = parseData(schema, data.toString('utf8').split('|')); - var o = stringifyData(schema, upd.fn(doc, upd.value)); - var buffer = U.createBuffer(o); - if (buffer.length > limit) { - Zlib.deflate(buffer, ZLIBOPTIONS, function(err, buffer) { - if (buffer.length <= limit) { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_COMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } else - save(new Error('GraphDB: Data too long')); - }); - } else { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_UNCOMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } - } - }); - } -} - -function insRelation(self) { - - if (!self.ready || self.states.relation) - return; - - var doc = self.pending.relation.shift(); - if (doc) { - - var rel = self.$relations[doc.name]; - if (rel == null) { - doc.callback(new Error('GraphDB: Relation "{0}" not found.'.format(doc.name))); - return; - } - - if (rel.locked || !rel.ready) { - self.pending.relation.push(doc); - setTimeout(insRelation, DELAY, self); - return; - } - - self.states.relation = true; - - if (doc.connect) { - addRelation(self, rel, doc.indexA, doc.indexB, function(err, id) { - self.states.relation = false; - doc.callback(err, id); - setImmediate(insRelation, self); - }); - } else { - remRelation(self, rel, doc.indexA, doc.indexB, function(err, id) { - self.states.relation = false; - doc.callback(err, id); - setImmediate(insRelation, self); - }); - } - } -} - -GP.create = function(filename, documentsize, callback) { - var self = this; - Fs.unlink(filename, function() { - var buf = U.createBufferSize(HEADERSIZE); - buf.write('Total.js GraphDB embedded', 0); - buf.writeUInt8(VERSION, 30); // version - buf.writeUInt32LE(0, 31); // pages - buf.writeUInt16LE(PAGESIZE, 35); // pagesize - buf.writeUInt16LE(PAGELIMIT, 37); // pagelimit - buf.writeUInt32LE(0, 39); // documents - buf.writeUInt16LE(documentsize, 43); // documentsize - buf.writeUInt8(0, 45); // classindex - buf.writeUInt8(0, 46); // relationindex - buf.writeUInt8(0, 47); // relationpageindex - buf.write('{"c":[],"r":[]}', 51); // classes and relations - Fs.open(filename, 'w', function(err, fd) { - Fs.write(fd, buf, 0, buf.length, 0, function(err) { - err && self.error(err, 'create'); - Fs.close(fd, function() { - callback && callback(); - }); - }); - }); - }); - return self; -}; - -GP.open = function() { - var self = this; - Fs.stat(self.filename, function(err, stat) { - if (err) { - // file not found - self.create(self.filename, DOCUMENTSIZE, () => self.open()); - } else { - self.header.size = stat.size; - Fs.open(self.filename, 'r+', function(err, fd) { - self.fd = fd; - err && self.error(err, 'open'); - var buf = U.createBufferSize(HEADERSIZE); - Fs.read(self.fd, buf, 0, buf.length, 0, function() { - - self.header.pages = buf.readUInt32LE(31); - self.header.pagesize = buf.readUInt16LE(35); - self.header.pagelimit = buf.readUInt16LE(37); - self.header.documents = buf.readUInt32LE(39); - self.header.documentsize = buf.readUInt16LE(43); - - var size = F.config['graphdb.' + self.name] || DOCUMENTSIZE; - if (size > self.header.documentsize) { - setTimeout(function() { - self.next(NEXT_RESIZE); - }, DELAY); - } - - self.header.relationlimit = ((self.header.documentsize - DATAOFFSET) / 6) >> 0; - self.header.classindex = buf[45]; - self.header.relationindex = buf[46]; - self.header.relationpageindex = buf.readUInt32LE(47); - - var data = buf.slice(51, buf.indexOf(EMPTYBUFFER, 51)).toString('utf8'); - var meta = data.parseJSON(true); - - for (var i = 0; i < meta.c.length; i++) { - var item = meta.c[i]; - self.class(item.n, item.r, item); - } - - for (var i = 0; i < meta.r.length; i++) { - var item = meta.r[i]; - self.relation(item.n, item.b === 1, item); - } - - !self.header.relationpageindex && addPage(self, TYPE_RELATION_DOCUMENT, 0, 0, function(err, index) { - self.header.relationpageindex = index; - }); - - self.ready = true; - self.next(NEXT_READY); - }); - }); - } - }); - return self; -}; - -GP.next = function(type) { - - var self = this; - var tmp; - - switch (type) { - case NEXT_READY: - for (var i = 0; i < self.pending.meta.length; i++) { - tmp = self.pending.meta[i]; - if (tmp.type === TYPE_CLASS) - self.class(tmp.name, tmp.data); - else - self.relation(tmp.name, tmp.data); - } - self.emit('ready'); - break; - - case NEXT_RESIZE: - - clearTimeout(self.$resizedelay); - self.$resizedelay = setTimeout(function() { - if (!self.states.resize) { - self.ready = false; - self.states.resize = true; - var size = (F.config['graphdb.' + self.name] || DOCUMENTSIZE); - var meta = { documentsize: size > self.header.documentsize ? size : self.header.documentsize }; - var keys = Object.keys(self.$classes); - - for (var i = 0; i < keys.length; i++) { - var key = keys[i]; - var cls = self.$classes[key]; - if (cls.$resize) { - !meta.classes && (meta.classes = {}); - meta.classes[cls.index] = cls.$resize; - cls.$resize = null; - } - } - self.resize(meta, function() { - self.states.resize = false; - self.ready = true; - setImmediate(self.cb_next, NEXT_CONTINUE); - }); - } - }, DELAY); - - break; - - case NEXT_INSERT: - insDocument(self); - break; - case NEXT_RELATION: - insRelation(self); - break; - case NEXT_UPDATE: - updDocument(self); - break; - case NEXT_REMOVE: - remDocument(self); - break; - case NEXT_FIND: - if (self.pending.find.length) { - tmp = self.pending.find.shift(); - $find(self, tmp.name, tmp.builder, tmp.reverse); - } - break; - } -}; - -GP.class = function(name, meta, data) { - - var self = this; - - if (!self.ready && !data) { - self.pending.meta.push({ name: name, data: meta, type: 1 }); - return self; - } - - var item = self.$classes[name]; - var save = false; - - if (item == null) { - - item = {}; - item.locked = false; - - if (data) { - item.ready = true; - item.name = name; - item.index = data.i; - item.pageindex = data.p; - item.documentindex = data.d; - item.findfreeslots = true; - } else { - self.header.classindex++; - item.name = name; - item.index = self.header.classindex; - item.ready = false; - item.pageindex = addPage(self, TYPE_CLASS, item.index, 0, function() { - item.ready = true; - }); - item.documentindex = getDocumentIndex(self, item.pageindex); - save = true; - } - - item.schema = parseSchema(meta); - self.$classes[item.name] = self.$classes[item.index] = item; - - } else { - var newschema = parseSchema(meta); - var raw = item.schema.raw; - if (raw !== newschema.raw) { - item.$resize = newschema; - self.next(NEXT_RESIZE); - } - } - - save && updMeta(self, META_CLASSESRELATIONS); - return self; -}; - -GP.relation = function(name, both, data) { - - var self = this; - - if (!self.ready && !data) { - self.pending.meta.push({ name: name, data: both, type: 2 }); - return self; - } - - var self = this; - var item = self.$relations[name]; - var save = false; - - if (item == null) { - - item = {}; - item.ready = true; - item.locked = false; - - if (data) { - item.name = name; - item.index = data.i; - item.pageindex = data.p; - item.documentindex = data.d; - item.both = both; - } else { - self.header.relationindex++; - item.name = name; - item.index = self.header.relationindex; - item.ready = false; - item.both = both; - item.pageindex = addPage(self, TYPE_RELATION, item.index, 0, function() { - item.ready = true; - }); - item.documentindex = getDocumentIndex(self, item.pageindex); - save = true; - } - - self.$relations[item.name] = self.$relations[item.index] = item; - - } else { - // compare - } - - save && updMeta(self, META_CLASSESRELATIONS); - return self; -}; - -GP.emit = function(name, a, b, c, d, e, f, g) { - var evt = this.$events[name]; - if (evt) { - var clean = false; - for (var i = 0, length = evt.length; i < length; i++) { - if (evt[i].$once) - clean = true; - evt[i].call(this, a, b, c, d, e, f, g); - } - if (clean) { - evt = evt.remove(n => n.$once); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - } - return this; -}; - -GP.on = function(name, fn) { - if (this.$ready && (name === 'ready' || name === 'load')) { - fn(); - return this; - } - if (!fn.$once) - this.$free = false; - if (this.$events[name]) - this.$events[name].push(fn); - else - this.$events[name] = [fn]; - return this; -}; - -GP.once = function(name, fn) { - fn.$once = true; - return this.on(name, fn); -}; - -GP.removeListener = function(name, fn) { - var evt = this.$events[name]; - if (evt) { - evt = evt.remove(n => n === fn); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - return this; -}; - -GP.removeAllListeners = function(name) { - if (name === true) - this.$events = EMPTYOBJECT; - else if (name) - this.$events[name] = undefined; - else - this.$events[name] = {}; - return this; -}; - -GP.resize = function(meta, callback) { - - // meta.documentsize - // meta.classes - - var self = this; - var filename = self.filename + '-tmp'; - - self.create(filename, meta.documentsize, function(err) { - - if (err) - throw err; - - Fs.open(filename, 'r+', function(err, fd) { - - var offset = HEADERSIZE; - var newoffset = HEADERSIZE; - var size = self.header.pagesize + (self.header.pagelimit * self.header.documentsize); - var newsize = self.header.pagesize + (self.header.pagelimit * meta.documentsize); - var pageindex = 0; - var totaldocuments = 0; - - var finish = function() { - - var buf = U.createBufferSize(HEADERSIZE); - Fs.read(fd, buf, 0, buf.length, 0, function() { - - // ==== DB:HEADER (7000b) - // name (30b) = from: 0 - // version (1b) = from: 30 - // pages (4b) = from: 31 - // pagesize (2b) = from: 35 - // pagelimit (2b) = from: 37 - // documents (4b) = from: 39 - // documentsize (2b) = from: 43 - // classindex (1b) = from: 45 - // relationindex (1b) = from: 46 - // relationnodeindex = from: 47 - // classes + relations = from: 51 - - // buf. - - buf.writeUInt32LE(pageindex > 0 ? (pageindex - 1) : 0, 31); - buf.writeUInt32LE(totaldocuments, 39); - buf.writeUInt16LE(meta.documentsize, 43); - - var obj = {}; - obj.c = []; // classes - obj.r = []; // relations - - for (var i = 0; i < self.header.classindex; i++) { - var item = self.$classes[i + 1]; - var schema = meta.classes[i + 1]; - obj.c.push({ n: item.name, i: item.index, p: item.pageindex, r: schema ? schema.raw : item.schema.raw, d: item.documentindex }); - } - - for (var i = 0; i < self.header.relationindex; i++) { - var item = self.$relations[i + 1]; - obj.r.push({ n: item.name, i: item.index, p: item.pageindex, b: item.both ? 1 :0, d: item.documentindex }); - } - - buf.writeUInt8(self.header.classindex, 45); - buf.writeUInt8(self.header.relationindex, 46); - buf.writeUInt32LE(self.header.relationpageindex, 47); - buf.write(JSON.stringify(obj), 51); - - Fs.write(fd, buf, 0, buf.length, 0, function() { - // console.log(pageindex, meta.documentsize, totaldocuments); - Fs.close(fd, function() { - Fs.close(self.fd, function() { - Fs.copyFile(self.filename, self.filename.replace(/\.gdb$/, NOW.format('_yyyyMMddHHmm') + '.gdp'), function() { - Fs.rename(self.filename + '-tmp', self.filename, function() { - callback(null); - }); - }); - }); - }); - }); - }); - }; - - var readvalue = function(docbuf, callback) { - var data = docbuf.slice(DATAOFFSET, docbuf.readUInt16LE(15) + DATAOFFSET); - if (docbuf[2] === STATE_COMPRESSED) - Zlib.inflate(data, ZLIBOPTIONS, (err, data) => callback(data ? data.toString('utf8') : '')); - else - callback(data.toString('utf8')); - }; - - var writevalue = function(value, callback) { - var maxsize = meta.documentsize - DATAOFFSET; - var data = U.createBuffer(value); - if (data.length > maxsize) { - Zlib.deflate(data, ZLIBOPTIONS, (err, data) => callback((!data || data.length > maxsize) ? EMPTYBUFFER : data)); - } else - callback(data); - }; - - var process = function() { - - pageindex++; - - // ==== DB:PAGE (20b) - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - // ==== DB:DOCUMENT (SIZE) - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - var buf = U.createBufferSize(size); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (!size) { - finish(); - return; - } - - var newbuf = U.createBufferSize(newsize); - - // Copies page info - newbuf.fill(buf, 0, self.header.pagesize); - buf = buf.slice(self.header.pagesize); - - var index = self.header.pagesize; - var documents = 0; - - (self.header.pagelimit).async(function(i, next) { - - // Unexpected problem - if (!buf.length) { - next(); - return; - } - - var docbuf = buf.slice(0, self.header.documentsize); - var typeid = docbuf[0]; - var indexid = docbuf[1]; - - if (docbuf[2] !== STATE_REMOVED) { - totaldocuments++; - documents++; - } - - if (docbuf[2] !== STATE_REMOVED && meta.classes && typeid === TYPE_CLASS && meta.classes[indexid]) { - readvalue(docbuf, function(value) { - - // parseData - // stringifyData - value = stringifyData(meta.classes[indexid], parseData(self.$classes[indexid].schema, value.split('|'))); - - writevalue(value, function(value) { - - if (value === EMPTYBUFFER) { - // BIG PROBLEM - docbuf.writeUInt16LE(0, 15); - docbuf.writeUInt8(STATE_REMOVED, 2); - documents--; - } else { - docbuf.writeUInt16LE(value.length, 15); - docbuf.fill(value, DATAOFFSET, DATAOFFSET + value.length); - } - - newbuf.fill(docbuf, index, index + self.header.documentsize); - index += meta.documentsize; - buf = buf.slice(self.header.documentsize); - next(); - }); - - }); - } else { - newbuf.fill(docbuf, index, index + self.header.documentsize); - index += meta.documentsize; - buf = buf.slice(self.header.documentsize); - next(); - } - - }, function() { - - // Update count of documents - if (newbuf.readUInt16LE(2) !== documents) - newbuf.writeUInt16LE(documents, 2); - - Fs.write(fd, newbuf, 0, newbuf.length, newoffset, function() { - offset += size; - newoffset += newsize; - setImmediate(process); - }); - }); - - }); - }; - - process(); - }); - }); - return self; -}; - - -function $update(doc, value) { - return value; -} - -function $modify(doc, value) { - var keys = Object.keys(value); - for (var i = 0; i < keys.length; i++) { - var key = keys[i]; - - switch (key[0]) { - case '+': - case '-': - case '*': - case '/': - var tmp = key.substring(1); - if (typeof(doc[tmp]) === 'number') { - if (key[0] === '+') - doc[tmp] += value[key]; - else if (key[0] === '-') - doc[tmp] -= value[key]; - else if (key[0] === '*') - doc[tmp] *= value[key]; - else if (key[0] === '/') - doc[tmp] = doc[tmp] / value[key]; - } - break; - default: - if (doc[key] != undefined) - doc[key] = value[key]; - break; - } - } - return doc; -} - -GP.remove = function(id, callback) { - var self = this; - var rem = { id: id, callback: callback || NOOP }; - self.pending.remove.push(rem); - self.next(NEXT_REMOVE); - return self; -}; - -GP.update = function(id, value, callback) { - var self = this; - var upd = { id: id, value: value, fn: typeof(value) === 'function' ? value : $update, callback: callback || NOOP }; - self.pending.update.push(upd); - self.next(NEXT_UPDATE); - return self; -}; - -GP.modify = function(id, value, callback) { - var self = this; - var upd = { id: id, value: value, fn: $modify, callback: callback || NOOP }; - self.pending.update.push(upd); - self.next(NEXT_UPDATE); - return self; -}; - -GP.insert = function(name, value, callback) { - var self = this; - self.pending.insert.push({ name: name, value: value, callback: callback || NOOP }); - self.next(NEXT_INSERT); - return self; -}; - -GP.cursor = function(type, name, callback) { - - var self = this; - var index; - var tmp; - - switch (type) { - case TYPE_CLASS: - tmp = self.$classes[name]; - index = tmp.pageindex; - break; - case TYPE_RELATION: - tmp = self.$relations[name]; - index = tmp.pageindex; - break; - } - - var offset = offsetPage(self, index); - var buf = U.createBufferSize(PAGESIZE); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - - if (err) { - callback(err); - return; - } - - if (buf[0] !== TYPE_CLASS) { - callback(new Error('Invalid page type')); - return; - } - - if (buf[1] !== tmp.index) { - callback(new Error('Invalid type index')); - return; - } - - var data = {}; - data.count = buf.readUInt16LE(2); - data.parent = buf.readUInt32LE(5); - data.offset = offset; - data.type = buf[0]; - data.index = buf[1]; - data.freeslots = buf[4]; - - data.next = function(callback) { - - if (data.parent == 0) { - callback(new Error('This is the last page'), data); - return; - } - - offset = offsetPage(self, data.parent); - Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - data.count = buf.readUInt16LE(2); - data.parent = buf.readUInt32LE(5); - data.offset = offset; - data.type = buf[0]; - data.index = buf[1]; - data.freeslots = buf[4]; - data.INDEX = getIndexPage(self, offset) + 1; - callback(err, data); - }); - }; - - data.documents = function(callback) { - - if (!data.count) { - callback(null, EMPTYARRAY); - return; - } - - var index = getIndexPage(self, data.offset) * self.header.pagelimit; - var buffer = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - var offset = data.offset + self.header.pagesize; - var decompress = []; - - index += self.header.pagelimit + 1; - - Fs.read(self.fd, buffer, 0, buffer.length, offset, function(err) { - - if (err) { - callback(err, EMPTYARRAY); - return; - } - - var arr = []; - while (true) { - - if (!buffer.length) - break; - - index--; - var data = buffer.slice(buffer.length - self.header.documentsize); - // index++; - // var data = buffer.slice(0, self.header.documentsize); - if (!data.length) - break; - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // continuerindex (4b) = from: 7 - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - if (data[2] !== STATE_REMOVED) { - var raw = data.slice(DATAOFFSET, data.readUInt16LE(15) + DATAOFFSET); - if (type === TYPE_CLASS) { - // Document is compressed - if (data[2] === STATE_COMPRESSED) { - var obj = {}; - obj.CLASS = tmp.name; - obj.ID = index; - obj.BUFFER = raw; - decompress.push({ CLASS: tmp, ID: index, BUFFER: raw, index: arr.push(null) }); - } else { - var obj = parseData(tmp.schema, raw.toString('utf8').split('|')); - obj.CLASS = tmp.name; - obj.ID = index; - arr.push(obj); - } - } - } - - buffer = buffer.slice(0, buffer.length - self.header.documentsize); - // buffer = buffer.slice(self.header.documentsize); - } - - if (decompress.length) { - decompress.wait(function(item, next) { - Zlib.inflate(item.BUFFER, ZLIBOPTIONS, function(err, data) { - var obj = parseData(item.CLASS.schema, data.toString('utf8').split('|')); - obj.CLASS = item.CLASS.name; - obj.ID = item.ID; - arr[item.index] = obj; - setImmediate(next); - }); - }, () => callback(null, arr)); - } else - callback(null, arr); - }); - }; - - callback(null, data); - }); -}; - -GP.read = function(index, callback) { - var self = this; - var buf = U.createBufferSize(self.header.documentsize); - Fs.read(self.fd, buf, 0, buf.length, offsetDocument(self, index), function(err) { - - if (err) { - callback(err); - return; - } - - if (buf[2] === STATE_REMOVED) { - callback(null, buf[0] === TYPE_CLASS ? null : EMPTYARRAY); - return; - } - - var tmp; - - switch(buf[0]) { - case TYPE_CLASS: - tmp = self.$classes[buf[1]]; - if (tmp) { - var data = buf.slice(DATAOFFSET, buf.readUInt16LE(15) + DATAOFFSET); - if (buf[2] === STATE_COMPRESSED) { - Zlib.inflate(data, ZLIBOPTIONS, function(err, data) { - data = parseData(tmp.schema, data.toString('utf8').split('|')); - data.ID = index; - data.CLASS = tmp.name; - callback(null, data, buf.readUInt32LE(7), buf.readUInt32LE(11)); - }); - } else { - data = parseData(tmp.schema, data.toString('utf8').split('|')); - data.ID = index; - data.CLASS = tmp.name; - callback(null, data, buf.readUInt32LE(7), buf.readUInt32LE(11)); - } - } else - callback(new Error('GraphDB: invalid document'), null); - break; - case TYPE_RELATION: - tmp = self.$relations[buf[1]]; - if (tmp) { - - var count = buf.readUInt16LE(15); - var arr = []; - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - arr.push({ RELATION: tmp.name, ID: buf.readUInt32LE(off + 2), INIT: buf[1], INDEX: i }); - } - - callback(null, arr, buf.readUInt32LE(7), buf.readUInt32LE(11), 'RELATION'); - - } else - callback(new Error('GraphDB: invalid document'), null); - break; - - case TYPE_RELATION_DOCUMENT: - - var count = buf.readUInt16LE(15); - var arr = []; - - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - tmp = self.$relations[buf[off]]; - arr.push({ RELATION: tmp.name, ID: buf.readUInt32LE(off + 2), INIT: buf[off + 1], INDEX: i }); - } - - callback(null, arr, buf.readUInt32LE(7), buf.readUInt32LE(11), 'PRIVATE'); - break; - - default: - callback(null, null); - break; - } - }); -}; - -GP.connect = function(name, indexA, indexB, callback) { - var self = this; - self.pending.relation.push({ name: name, indexA: indexA, indexB: indexB, callback: callback, connect: true }); - self.next(NEXT_RELATION); - return self; -}; - -GP.disconnect = function(name, indexA, indexB, callback) { - var self = this; - self.pending.relation.push({ name: name, indexA: indexA, indexB: indexB, callback: callback }); - self.next(NEXT_RELATION); - return self; -}; - -GP.find = function(cls) { - var self = this; - var builder = new DatabaseBuilder(self); - self.pending.find.push({ name: cls, builder: builder }); - setImmediate(self.cb_next, NEXT_FIND); - return builder; -}; - -GP.find2 = function(cls) { - var self = this; - var builder = new DatabaseBuilder(self); - self.pending.find.push({ name: cls, builder: builder, reverse: true }); - setImmediate(self.cb_next, NEXT_FIND); - return builder; -}; - -GP.scalar = function(cls, type, field) { - var self = this; - var builder = new DatabaseBuilder(self); - builder.scalar(type, field); - self.pending.find.push({ name: cls, builder: builder }); - setImmediate(self.cb_next, NEXT_FIND); - return builder; -}; - -GP.count = function(cls) { - return this.scalar(cls, 'count', 'ID'); -}; - -function GraphDBFilter(db) { - var t = this; - t.db = db; - t.levels = null; -} - -GraphDBFilter.prototype.level = function(num) { - var self = this; - if (self.levels == null) - self.levels = {}; - return self.levels[num] = new DatabaseBuilder(self.db); -}; - -GraphDBFilter.prototype.prepare = function() { - - var self = this; - - if (!self.levels) - return self; - - var arr = Object.keys(self.levels); - - for (var i = 0; i < arr.length; i++) { - var key = arr[i]; - var builder = self.levels[key]; - var filter = {}; - filter.builder = builder; - filter.scalarcount = 0; - filter.filter = builder.makefilter(); - filter.compare = builder.compile(); - filter.index = 0; - filter.count = 0; - filter.counter = 0; - filter.first = builder.$options.first && !builder.$options.sort; - self.levels[key] = filter; - } - - return self; -}; - -GP.graph = function(id, options, callback, filter) { - - if (typeof(options) === 'function') { - callback = options; - options = EMPTYOBJECT; - } else if (!options) - options = EMPTYOBJECT; - - var self = this; - - if (!filter) - filter = new GraphDBFilter(self); - - - self.read(id, function(err, doc, linkid) { - - if (err || !doc) { - callback(err, null, 0); - return; - } - - // options.depth (Int) - // options.relation (String or String Array) - // options.class (String or String Array) - - var relations = null; - var classes = null; - - if (options.relation) { - - var rel; - relations = {}; - - if (options.relation instanceof Array) { - for (var i = 0; i < options.relation.length; i++) { - rel = self.$relations[options.relation[i]]; - if (rel) - relations[rel.name] = rel.both ? 1 : 0; - } - } else { - rel = self.$relations[options.relation]; - if (rel) - relations[rel.name] = rel.both ? 1 : 0; - } - } - - if (options.class) { - - var clstmp; - classes = {}; - - if (options.class instanceof Array) { - for (var i = 0; i < options.class.length; i++) { - clstmp = self.$classes[options.class[i]]; - if (clstmp) - classes[clstmp.name] = 1; - } - } else { - clstmp = self.$classes[options.class]; - if (clstmp) - classes[clstmp.name] = clstmp.index + 1; - } - } - - filter.prepare(); - - var pending = []; - var tmp = {}; - var count = 1; - var sort = false; - - tmp[id] = 1; - - doc.INDEX = 0; - doc.LEVEL = 0; - doc.NODES = []; - - var reader = function(parent, id, depth) { - - if ((options.depth && depth >= options.depth) || (tmp[id])) { - process(); - return; - } - - tmp[id] = 1; - - self.read(id, function(err, links, linkid) { - - if (linkid && !tmp[linkid]) { - pending.push({ id: linkid, parent: parent, depth: depth }); - sort = true; - } - - // because of seeking on HDD - links.quicksort('ID'); - - var fil; - - links.wait(function(item, next) { - - var key = item.ID + '-' + item.RELATION; - - if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && !item.INIT && !relations) || (relations && relations[item.RELATION] === item.TYPE)) - return next(); - - tmp[key] = 1; - - self.read(item.ID, function(err, doc, linkid) { - - if (doc && (!classes || classes[doc.CLASS])) { - - count++; - - doc.INDEX = item.INDEX; - doc.LEVEL = depth + 1; - doc.NODES = []; - - var rel = self.$relations[item.RELATION]; - - if (rel) { - // doc.RELATION = rel.relation; - doc.RELATION = rel.name; - } - - fil = filter.levels ? filter.levels[depth + 1] : null; - - if (fil) { - !fil.response && (fil.response = parent.NODES); - if (!framework_nosql.compare(fil, doc)) - linkid = null; - } else - parent.NODES.push(doc); - - if (linkid && !tmp[linkid]) { - pending.push({ id: linkid, parent: doc, depth: depth + 1 }); - sort = true; - } - } - - next(); - }); - - }, process); - }); - }; - - var process = function() { - - if (pending.length) { - - // because of seeking on HDD - if (sort && pending.length > 1) { - pending.quicksort('id'); - sort = false; - } - - var item = pending.shift(); - reader(item.parent, item.id, item.depth); - - } else { - - if (filter.levels) { - var keys = Object.keys(filter.levels); - for (var i = 0; i < keys.length; i++) { - var f = filter.levels[keys[i]]; - framework_nosql.callback(f); - } - } - - callback(null, doc, count); - } - }; - - linkid && pending.push({ id: linkid, parent: doc, depth: 0 }); - process(); - - }, options.type); - - return filter; -}; - -function $find(self, cls, builder, reverse) { - - var filter = {}; - - filter.builder = builder; - filter.scalarcount = 0; - filter.filter = builder.makefilter(); - filter.compare = builder.compile(); - filter.index = 0; - filter.count = 0; - filter.counter = 0; - filter.first = builder.$options.first && !builder.$options.sort; - - var tmp = self.$classes[cls]; - if (!tmp) { - framework_nosql.callback(filter, 'GraphDB: Class "{0}" is not registered.'.format(cls)); - setImmediate(self.cb_next, NEXT_FIND); - return; - } - - var read = function(err, data) { - - if (err || (!data.count && !data.parent)) { - framework_nosql.callback(filter); - return; - } - - data.documents(function(err, docs) { - for (var i = 0; i < docs.length; i++) { - var doc = docs[i]; - filter.index++; - if ((doc && framework_nosql.compare(filter, doc) === false) || (reverse && filter.done)) { - framework_nosql.callback(filter); - data.next = null; - data.documents = null; - data = null; - setImmediate(self.cb_next, NEXT_FIND); - return; - } - } - data.next(read); - }); - }; - - self.cursor(1, tmp.name, read); -} - -function parseSchema(schema) { - - var obj = {}; - var arr = schema.split('|').trim(); - - obj.meta = {}; - obj.keys = []; - obj.raw = schema; - - for (var i = 0; i < arr.length; i++) { - var arg = arr[i].split(':'); - var type = 0; - switch ((arg[1] || '').toLowerCase().trim()) { - case 'number': - type = 2; - break; - case 'boolean': - case 'bool': - type = 3; - break; - case 'date': - type = 4; - break; - case 'object': - type = 5; - break; - case 'string': - default: - type = 1; - break; - } - var name = arg[0].trim(); - obj.meta[name] = { type: type, pos: i }; - obj.keys.push(name); - } - - return obj; -} - -function stringifyData(schema, doc) { - - var output = ''; - var esc = false; - var size = 0; - - for (var i = 0; i < schema.keys.length; i++) { - var key = schema.keys[i]; - var meta = schema.meta[key]; - var val = doc[key]; - - switch (meta.type) { - case 1: // String - val = val ? val : ''; - size += 4; - break; - case 2: // Number - val = (val || 0); - size += 2; - break; - case 3: // Boolean - val = (val == true ? '1' : '0'); - break; - case 4: // Date - // val = val ? val.toISOString() : ''; - val = val ? val.getTime() : ''; - !val && (size += 13); - break; - case 5: // Object - val = val ? JSON.stringify(val) : ''; - size += 4; - break; - } - - if (!esc && (meta.type === 1 || meta.type === 5)) { - val += ''; - if (REGTESCAPETEST.test(val)) { - esc = true; - val = val.replace(REGTESCAPE, regtescape); - } - } - - output += '|' + val; - } - - return (esc ? '*' : '+') + output; -} - -function parseData(schema, lines, cache) { - - var obj = {}; - var esc = lines === '*'; - var val; - - for (var i = 0; i < schema.keys.length; i++) { - var key = schema.keys[i]; - - if (cache && cache !== EMPTYOBJECT && cache[key] != null) { - obj[key] = cache[key]; - continue; - } - - var meta = schema.meta[key]; - if (meta == null) - continue; - - var pos = meta.pos + 1; - - switch (meta.type) { - case 1: // String - obj[key] = lines[pos]; - if (esc && obj[key]) - obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); - break; - case 2: // Number - val = +lines[pos]; - obj[key] = val < 0 || val > 0 ? val : 0; - break; - case 3: // Boolean - val = lines[pos]; - obj[key] = BOOLEAN[val] == 1; - break; - case 4: // Date - val = lines[pos]; - obj[key] = val ? new Date(val[10] === 'T' ? val : +val) : null; - break; - case 5: // Object - val = lines[pos]; - if (esc && val) - val = val.replace(REGTUNESCAPE, regtescapereverse); - obj[key] = val ? val.parseJSON(true) : null; - break; - } - } - - return obj; -} - -function regtescapereverse(c) { - switch (c) { - case '%0A': - return '\n'; - case '%0D': - return '\r'; - case '%7C': - return '|'; - } - return c; -} - -function regtescape(c) { - switch (c) { - case '\n': - return '%0A'; - case '\r': - return '%0D'; - case '|': - return '%7C'; - } - return c; -} - -exports.load = function(name, size) { - return new GraphDB(name, size); -}; - -exports.getImportantOperations = function() { - return IMPORTATOPERATIONS; -}; \ No newline at end of file From b52bc2111d3d6cbaf1d8df4ddb7b80ea9b2e8416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 20:26:40 +0200 Subject: [PATCH 0600/1669] Updated version. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 3c3822dd0..43956b4a7 100755 --- a/utils.js +++ b/utils.js @@ -21,7 +21,7 @@ /** * @module FrameworkUtils - * @version 3.0.0 + * @version 3.0.1 */ 'use strict'; From 82e47b92b18e594607978ec0d4c287762584fa6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 20:27:41 +0200 Subject: [PATCH 0601/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e819243bd..261e638ce 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-2", + "version": "3.0.1-3", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 20e8d28fec968d8be09597a6126a43d33604d9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 13:13:28 +0200 Subject: [PATCH 0602/1669] Fixed insert after modify/update. --- nosql.js | 99 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/nosql.js b/nosql.js index ada884b06..45efce0ac 100755 --- a/nosql.js +++ b/nosql.js @@ -1534,9 +1534,7 @@ DP.$update = function() { return; !change && (change = true); - var was = true; - !change && (change = true); if (rec.doc.length === upd.length) { var b = Buffer.byteLength(upd); @@ -1558,10 +1556,26 @@ DP.$update = function() { }; fs.$callback = function() { - filters.done(); + fs = null; self.$writting = false; self.next(0); + + for (var i = 0; i < filters.builders.length; i++) { + var item = filters.builders[i]; + var fil = filter[i]; + if (fil.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + var tmp = self.insert(fil.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + item.builder.$callback = null; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + } + } + change && self.$events.change && self.emit('change', 'update'); }; @@ -1644,9 +1658,23 @@ DP.$update_inmemory = function() { filters.compare2(self.inmemory['#'], update, updateflush); change && self.$save(); + + for (var i = 0; i < filter.length; i++) { + var item = filter[i]; + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); + var tmp = self.insert(item.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + item.builder.$callback = null; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + } + } + setImmediate(function() { self.next(0); - filters.done(); change && self.$events.change && self.emit('change', 'update'); }); }); @@ -5890,7 +5918,6 @@ TP.$update = function() { fs.write(tmp, rec.position); fs.write2(upd + NEWLINE); } - }; fs.ondocuments = function() { @@ -5913,10 +5940,26 @@ TP.$update = function() { fs.$callback = function() { - filters.done(); + fs = null; self.$writting = false; self.next(0); + + for (var i = 0; i < filters.builders.length; i++) { + var item = filters.builders[i]; + var fil = filter[i]; + if (fil.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + var tmp = self.insert(fil.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + item.builder.$callback = null; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + } + } + change && self.$events.change && self.emit('change', 'update'); }; @@ -6611,46 +6654,46 @@ NoSQLReader.prototype.done = function() { var item = self.builders[i]; var builder = item.builder; var output; + var opt = builder.$options; - if (builder.$options.scalar || !builder.$options.sort) { - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) + if (opt.scalar || !opt.sort) { + if (opt.scalar) + output = opt.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (opt.first) output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) + else if (opt.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); continue; } if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) + if (opt.sort.name) { + if (!builder.$inlinesort || opt.take !== item.response.length) + item.response.quicksort(opt.sort.name, opt.sort.asc); + } else if (opt.sort === null) item.response.random(); else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); + item.response.sort(opt.sort); + + if (opt.skip && opt.take) + item.response = item.response.splice(opt.skip, opt.take); + else if (opt.skip) + item.response = item.response.splice(opt.skip); + else if (!builder.$inlinesort && opt.take) + item.response = item.response.splice(0, opt.take); } - if (builder.$options.first) + if (opt.first) output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) + else if (opt.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); - builder.done(); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); } }; From 7a79a7fc2313cc44ea2b4f4127e0f6e16cf1d370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 13:33:03 +0200 Subject: [PATCH 0603/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 261e638ce..efb0e00e6 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-3", + "version": "3.0.1-4", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 5776a42b3dac11b6b90ee76aa6e042e0047eda33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 21:24:40 +0200 Subject: [PATCH 0604/1669] Fixed `versions`. --- changes.txt | 1 + index.js | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 69b618c0e..ddf93ca71 100755 --- a/changes.txt +++ b/changes.txt @@ -12,6 +12,7 @@ - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) +- fixed: `versions` and `auto` feature with enabled `F.wait()` - improved: NoSQL reader diff --git a/index.js b/index.js index f5516e3ac..7ddf8c653 100755 --- a/index.js +++ b/index.js @@ -8461,9 +8461,19 @@ F.$configure_versions = function(arr, clean) { return F; }; -function makehash(url, callback) { - url = 'http://' + (F.ip === 'auto' ? '0.0.0.0' : F.ip) + ':' + F.port + url; - U.download(url, ['get'], function(err, stream, status) { +function makehash(url, callback, count) { + var target = 'http://' + (F.ip === 'auto' ? '0.0.0.0' : F.ip) + ':' + F.port + url; + U.download(target, ['get'], function(err, stream, status) { + + // Maybe F.wait() + if (status === 503) { + // Unhandled problem + if (count > 60) + callback(''); + else + setTimeout((url, callback, count) => makehash(url, callback, (count || 1) + 1), 1000, url, callback, count); + return; + } if (status !== 200) { callback(''); From 3cb9554cd863b48daa8ab4bc613c41e0df25b768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 21:26:09 +0200 Subject: [PATCH 0605/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index efb0e00e6..1e48fa7e2 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-4", + "version": "3.0.1-5", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 1d1256ec828e965f4890417795879ed779726f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 09:52:29 +0200 Subject: [PATCH 0606/1669] Added a custom Schema data type. --- builders.js | 12 ++++++++++++ utils.js | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index ecedd6515..4b4556c72 100755 --- a/builders.js +++ b/builders.js @@ -643,6 +643,11 @@ SchemaBuilderEntity.prototype.$parse = function(name, value, required, custom) { return result; } + if (value instanceof SchemaBuilderEntity) + result.type = 7; + else + result.type = 10; + return result; } @@ -1880,6 +1885,13 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { item[property] = null; break; + + case 10: + // custom object type + item[property] = type.raw(val == null ? '' : val.toString()); + if (item[property] === undefined) + item[property] = null; + break; } continue; } diff --git a/utils.js b/utils.js index 43956b4a7..c067fe55e 100755 --- a/utils.js +++ b/utils.js @@ -2227,7 +2227,7 @@ function validate_builder_default(name, value, entity) { var type = typeof(value); - // Enum + KeyValue (8+9) + // Enum + KeyValue + Custom (8+9+10) if (entity.type > 7) return value !== undefined; From 1195ec5c7dd5c13de8ac6b0cf67ac4f5d2a0e3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 09:52:56 +0200 Subject: [PATCH 0607/1669] New beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1e48fa7e2..145079f97 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-5", + "version": "3.0.1-6", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From fa2e7d971f9e674c65668d06f7a918127a5b5697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 10:36:33 +0200 Subject: [PATCH 0608/1669] Added `.bundlesignore` --- bundles.js | 43 +++++++++++++++++++++++++++++++++++++++++++ changes.txt | 1 + 2 files changed, 44 insertions(+) diff --git a/bundles.js b/bundles.js index e04a46443..a8fc63ed5 100755 --- a/bundles.js +++ b/bundles.js @@ -14,6 +14,7 @@ META.total = 'v' + F.version_header; META.node = F.version_node; META.files = []; META.directories = []; +META.ignore = () => true; exports.make = function(callback) { @@ -25,6 +26,10 @@ exports.make = function(callback) { console.time('Done'); } + try { + META.ignore = makeignore(Fs.readFileSync(Path.join(path, '.bundlesignore')).toString('utf8').split('\n')); + } catch (e) {} + blacklist[F.config['directory-temp']] = 1; blacklist[F.config['directory-bundles']] = 1; blacklist[F.config['directory-src']] = 1; @@ -150,6 +155,41 @@ exports.make = function(callback) { }; +function makeignore(arr) { + + var ext; + var code = ['var path=P.substring(0,P.lastIndexOf(\'/\') + 1);', 'var ext=U.getExtension(P);', 'var name=U.getName(P).replace(\'.\' + ext, \'\');']; + + for (var i = 0; i < arr.length; i++) { + var item = arr[i]; + var index = item.lastIndexOf('*.'); + + if (index !== -1) { + // only extensions on this path + ext = item.substring(index + 2); + item = item.substring(0, index); + code.push('tmp=\'{0}\';'.format(item)); + code.push('if((!tmp||path===tmp)&&ext===\'{0}\')return;'.format(ext)); + continue; + } + + ext = U.getExtension(item); + if (ext) { + // only filename + index = item.lastIndexOf('/'); + code.push('tmp=\'{0}\';'.format(item.substring(0, index + 1))); + code.push('if(path===tmp&&U.getName(\'{0}\').replace(\'.{1}\', \'\')===name&&ext===\'{1}\')return;'.format(item.substring(index + 1), ext)); + continue; + } + + // all nested path + code.push('if(path.startsWith(\'{0}\'))return;'.format(item.replace('*', ''))); + } + + code.push('return true'); + return new Function('P', code.join('')); +} + function normalize(path) { return isWindows ? path.replace(/\\/g, '/') : path; } @@ -218,6 +258,9 @@ function copyFiles(files, callback) { var path = F.path.root(F.config['directory-src']); files.wait(function(file, next) { + if (!META.ignore(file.name)) + return next(); + var filename = Path.join(path, file.name); var exists = false; var ext = U.getExtension(file.name); diff --git a/changes.txt b/changes.txt index ddf93ca71..6a09b2c79 100755 --- a/changes.txt +++ b/changes.txt @@ -4,6 +4,7 @@ - added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` +- added: `.bundlesignore` support with similiar functionality like `.gitignore` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` From 9960c86fde5b18c634927a15b234cfafe47bd6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 10:45:11 +0200 Subject: [PATCH 0609/1669] Added `tests` options to `F.http()`. --- index.js | 7 +++++++ test.js | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 7ddf8c653..b8b9408c1 100755 --- a/index.js +++ b/index.js @@ -6486,6 +6486,13 @@ F.initialize = function(http, debug, options) { global.DEBUG = debug; global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; + + if (options.tests) { + F.testlist = options.tests; + for (var i = 0; i < F.testlist.length; i++) + F.testlist[i] = F.testlist[i].replace(/\.js$/, ''); + } + F.$bundle(function() { F.$configure_configs(); diff --git a/test.js b/test.js index 58e61b814..d4abd2114 100644 --- a/test.js +++ b/test.js @@ -168,8 +168,16 @@ global.TESTUSER = function(user, flags) { }; exports.load = function() { - U.ls(F.path.tests(), function(files) { + var dir = F.path.tests(); + U.ls(dir, function(files) { files.waitFor(function(filename, next) { + + if (F.testlist) { + var tn = filename.replace(dir, '').replace(/\.js$/, ''); + if (F.testlist.indexOf(tn) === -1) + return next(); + } + T.current = { filename: filename, items: [] }; var m = require(filename); T.current.module = m; From 0862f448481751e560019a41472cfa9ba9b8f993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 19:33:43 +0200 Subject: [PATCH 0610/1669] Improved cloning values in Schemas because of ObjectID from MongoDB. --- builders.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 4b4556c72..38e6f49c5 100755 --- a/builders.js +++ b/builders.js @@ -2553,7 +2553,12 @@ function clone(obj) { continue; } - o[m] = clone(obj[m]); + // Because here can be a problem with MongoDB.ObjectID + // I assume plain/simple model + if (val.constructor === Object) + o[m] = clone(obj[m]); + else + o[m] = val; } return o; From 301792440433756d631d0acea0f1b361f13ebd95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 20:14:45 +0200 Subject: [PATCH 0611/1669] Fixed nullable values. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 38e6f49c5..44c6c79b5 100755 --- a/builders.js +++ b/builders.js @@ -2555,7 +2555,7 @@ function clone(obj) { // Because here can be a problem with MongoDB.ObjectID // I assume plain/simple model - if (val.constructor === Object) + if (val && val.constructor === Object) o[m] = clone(obj[m]); else o[m] = val; From ee3f9fabc95b64216841bd34e48d5ecab2a6186c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 1 Sep 2018 20:58:05 +0200 Subject: [PATCH 0612/1669] Fixed merging files #663. --- debug.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/debug.js b/debug.js index 609821efa..455d62dbd 100644 --- a/debug.js +++ b/debug.js @@ -262,14 +262,19 @@ function runwatching() { if (files[filename]) { var tmp = isViewPublic(filename); if (tmp) { + var skip = true; if (isBUNDLE) { - copyFile(filename, Path.join(SRC, tmp)); - console.log(log); + if (filename.lastIndexOf('--') === -1) + copyFile(filename, Path.join(SRC, tmp)); + else + skip = false; + } + if (skip) { + files[filename] = ticks; + reload = true; + next(); + return; } - files[filename] = ticks; - reload = true; - next(); - return; } } From df7316074c706688f563c4d57efbbe25640a3210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 1 Sep 2018 20:58:20 +0200 Subject: [PATCH 0613/1669] Update beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 145079f97..bce70aed9 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-6", + "version": "3.0.1-7", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9754fc63b95753d1e2dd6eb0f21e50d796350945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 2 Sep 2018 11:11:16 +0200 Subject: [PATCH 0614/1669] Fixed repository. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 45efce0ac..d3a15ce0d 100755 --- a/nosql.js +++ b/nosql.js @@ -1669,7 +1669,7 @@ DP.$update_inmemory = function() { item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); } } From 832ae5d8807adb651d126be6506f396c9d9bdf19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 2 Sep 2018 22:57:37 +0200 Subject: [PATCH 0615/1669] Extend `nosqlstorage.scalar()`. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index d3a15ce0d..d22e8c261 100755 --- a/nosql.js +++ b/nosql.js @@ -5140,8 +5140,8 @@ SP.count = function(beg, end, threads) { return builder; }; -SP.scalar = function(type, field) { - return this.find().scalar(type, field); +SP.scalar = function(beg, end, type, field, threads) { + return this.find(beg, end, threads).scalar(type, field); }; SP.scan = function(beg, end, mapreduce, callback, reverse) { From 719fd4a212eea695230b92c92716d1511c14eea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 2 Sep 2018 22:58:08 +0200 Subject: [PATCH 0616/1669] Updated readme. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 6a09b2c79..b04c6dff3 100755 --- a/changes.txt +++ b/changes.txt @@ -1,7 +1,7 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` -- added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support +- added: NoSQL storage `.find(beg, end, [threads])` + `.count(beg, end, [threads])` + '.scalar(beg, end, type, field, [threads])' with multiple thread support - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` From f57fb4370ca4eb5869f96d0b6b4bd0bf98f1904c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 3 Sep 2018 09:02:15 +0200 Subject: [PATCH 0617/1669] Updated version. --- changes.txt | 3 ++- debug.js | 2 +- image.js | 2 +- index.js | 35 ++++++++++++++++++++++++++++------- nosql.js | 2 +- package.json | 2 +- utils.js | 2 +- websocketclient.js | 2 +- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/changes.txt b/changes.txt index b04c6dff3..3309b9321 100755 --- a/changes.txt +++ b/changes.txt @@ -1,10 +1,11 @@ -======= 3.0.1 +======= 3.1.0 - added: CSS variables support default values `border-radius: $radius || 10px` - added: NoSQL storage `.find(beg, end, [threads])` + `.count(beg, end, [threads])` + '.scalar(beg, end, type, field, [threads])' with multiple thread support - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` +- added: support for `SameSite` cookie attribute - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` diff --git a/debug.js b/debug.js index 455d62dbd..1924a6728 100644 --- a/debug.js +++ b/debug.js @@ -21,7 +21,7 @@ /** * @module FrameworkDebug - * @version 3.0.1 + * @version 3.1.0 */ const Path = require('path'); diff --git a/image.js b/image.js index 0894df10b..04bcd26bc 100755 --- a/image.js +++ b/image.js @@ -21,7 +21,7 @@ /** * @module FrameworkImage - * @version 2.9.2 + * @version 3.0.0 */ 'use strict'; diff --git a/index.js b/index.js index b8b9408c1..0ec53c0f2 100755 --- a/index.js +++ b/index.js @@ -21,7 +21,7 @@ /** * @module Framework - * @version 3.0.1 + * @version 3.1.0 */ 'use strict'; @@ -626,8 +626,8 @@ var PERF = {}; function Framework() { this.$id = null; // F.id ==> property - this.version = 3001; - this.version_header = '3.0.1'; + this.version = 3100; + this.version_header = '3.1.0'; this.version_node = process.version.toString(); this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); @@ -14848,8 +14848,8 @@ function extend_response(PROTO) { if (self.headersSent || self.success) return; - var cookieHeaderStart = name + '='; - var builder = [cookieHeaderStart + value]; + var cookiename = name + '='; + var builder = [cookiename + value]; var type = typeof(expires); if (expires && !U.isDate(expires) && type === 'object') { @@ -14872,11 +14872,32 @@ function extend_response(PROTO) { if (options.httpOnly || options.httponly || options.HttpOnly) builder.push('HttpOnly'); + var same = options.security || options.samesite || options.sameSite; + if (same) { + switch (same) { + case 1: + same = 'lax'; + break; + case 2: + same = 'strict'; + break; + } + builder.push('SameSite=' + same); + } + var arr = self.getHeader('set-cookie') || []; // Cookie, already, can be in array, resulting in duplicate 'set-cookie' header - var idx = arr.findIndex(cookieStr => cookieStr.startsWith(cookieHeaderStart)); - idx !== -1 && arr.splice(idx, 1); + if (arr.length) { + var l = cookiename.length; + for (var i = 0; i < arr.length; i++) { + if (arr[i].substring(0, l) === cookiename) { + arr.splice(i, 1); + break; + } + } + } + arr.push(builder.join('; ')); self.setHeader('Set-Cookie', arr); return self; diff --git a/nosql.js b/nosql.js index d22e8c261..2ff84898c 100755 --- a/nosql.js +++ b/nosql.js @@ -21,7 +21,7 @@ /** * @module NoSQL - * @version 3.0.1 + * @version 3.1.0 */ 'use strict'; diff --git a/package.json b/package.json index bce70aed9..471541549 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-7", + "version": "3.0.1-8", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/utils.js b/utils.js index c067fe55e..640793530 100755 --- a/utils.js +++ b/utils.js @@ -21,7 +21,7 @@ /** * @module FrameworkUtils - * @version 3.0.1 + * @version 3.1.0 */ 'use strict'; diff --git a/websocketclient.js b/websocketclient.js index 5ecce29b6..b3aa7a69b 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -21,7 +21,7 @@ /** * @module WebSocketClient - * @version 3.0.0 + * @version 3.1.0 */ if (!global.framework_utils) From 1279b2dae3c829b6509fafb53e5bdc4fba378367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 3 Sep 2018 14:47:32 +0200 Subject: [PATCH 0618/1669] Improved `U.reader()`. --- utils.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 640793530..bf6102b13 100755 --- a/utils.js +++ b/utils.js @@ -6121,11 +6121,21 @@ exports.Callback = function(count, callback) { function Reader() {} const RP = Reader.prototype; +RP.clear = function() { + var self = this; + var builder = self.reader.builders[0]; + builder.scalarcount = 0; + builder.count = 0; + builder.counter = 0; + builder.response = null; + return self; +}; + RP.push = function(data) { - if (data == null || !data.length) + if (data == null) this.reader.done(); else - this.reader.compare(data); + this.reader.compare(data instanceof Array ? data : [data]); return this; }; From f751cdf1943d65fba13dd10a9c635047d0dc6b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 3 Sep 2018 19:11:27 +0200 Subject: [PATCH 0619/1669] Added support for multiple filter in `U.reader()`. --- utils.js | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/utils.js b/utils.js index bf6102b13..1530d9c5f 100755 --- a/utils.js +++ b/utils.js @@ -6122,12 +6122,18 @@ function Reader() {} const RP = Reader.prototype; RP.clear = function() { + var self = this; - var builder = self.reader.builders[0]; - builder.scalarcount = 0; - builder.count = 0; - builder.counter = 0; - builder.response = null; + var builders = self.reader.builders; + + for (var i = 0; i < builders.length; i++) { + var builder = builders[i]; + builder.scalarcount = 0; + builder.count = 0; + builder.counter = 0; + builder.response = null; + } + return self; }; @@ -6142,7 +6148,12 @@ RP.push = function(data) { RP.find = function() { var self = this; var builder = new framework_nosql.DatabaseBuilder(); - self.reader = new framework_nosql.NoSQLReader(builder); + + if (self.reader) + self.reader.add(builder); + else + self.reader = new framework_nosql.NoSQLReader(builder); + return builder; }; From 263e3326f23b01f08571b274a4a77bcf93559eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Sep 2018 21:06:37 +0200 Subject: [PATCH 0620/1669] Fixed schema declaration with white characters. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 44c6c79b5..a6f8d09fb 100755 --- a/builders.js +++ b/builders.js @@ -31,7 +31,7 @@ const DEFAULT_SCHEMA = 'default'; const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true, $$workflow: true }; const REGEXP_CLEAN_EMAIL = /\s/g; const REGEXP_CLEAN_PHONE = /\s|\.|-|\(|\)/g; -const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; +const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?\([a-zA-Z$\s]+\)|^function anonymous\(\$|^\([a-zA-Z$\s]+\)/; const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); From 189e3d58b492dd9bb80d3131cccf9c3a8f5ac114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Sep 2018 21:17:43 +0200 Subject: [PATCH 0621/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 471541549..c4b027ce5 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-8", + "version": "3.0.1-9", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4ce5f8e2fcc50f5c1e1c9ea1b9529b05f2448508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 6 Sep 2018 10:19:30 +0200 Subject: [PATCH 0622/1669] Fixed `LOAD('release')`. --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 0ec53c0f2..f42863185 100755 --- a/index.js +++ b/index.js @@ -6351,6 +6351,7 @@ global.LOAD = F.load = function(debug, types, pwd) { switch (debug.toLowerCase().replace(/\.|\s/g, '-')) { case 'release': case 'production': + debug = false; break; case 'debug': From 8c49dd990f5215915d05772a90d023bb5ae018d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 6 Sep 2018 10:19:42 +0200 Subject: [PATCH 0623/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c4b027ce5..b910f58ae 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-9", + "version": "3.0.1-10", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 22761e630c4f794b8a19943838bcb7250cd82a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 6 Sep 2018 10:20:23 +0200 Subject: [PATCH 0624/1669] Updated changelog. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 3309b9321..b23d0ad10 100755 --- a/changes.txt +++ b/changes.txt @@ -15,6 +15,7 @@ - fixed: `LOCALIZE()` for nested directories - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) - fixed: `versions` and `auto` feature with enabled `F.wait()` +- fixed: `LOAD('release')` a release mode - improved: NoSQL reader From 6053437cdc4838b004f60f13c572a0d74cbd6052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 10:23:34 +0200 Subject: [PATCH 0625/1669] Fixed `$.model.$clean()` in Schemas. --- builders.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/builders.js b/builders.js index a6f8d09fb..4d37e1409 100755 --- a/builders.js +++ b/builders.js @@ -2152,9 +2152,9 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk var $type = 'hook'; - if (skip === true) { - var builder = new ErrorBuilder(); + if (skip === true || model instanceof SchemaInstance) { + var builder = new ErrorBuilder(); self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2286,7 +2286,7 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca if (model && !controller && model.$$controller) controller = model.$$controller; - if (skip === true) { + if (skip === true || model instanceof SchemaInstance) { var builder = new ErrorBuilder(); self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2526,7 +2526,10 @@ function clone(obj) { o[i] = obj[i]; continue; } - o[i] = clone(obj[i]); + if (obj[i] instanceof SchemaInstance) + o[i] = obj[i].$clean(); + else + o[i] = clone(obj[i]); } return o; @@ -2541,11 +2544,16 @@ function clone(obj) { var val = obj[m]; - if (val instanceof SchemaInstance) { + if (val instanceof Array) { o[m] = clone(val); continue; } + if (val instanceof SchemaInstance) { + o[m] = val.$clean(); + continue; + } + var type = typeof(val); if (type !== 'object' || val instanceof Date) { if (type !== 'function') From 1c27a43c31cee2768eaa3ea92521b577cb4b6da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 10:46:28 +0200 Subject: [PATCH 0626/1669] Added a new bug fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index b23d0ad10..0f37582f6 100755 --- a/changes.txt +++ b/changes.txt @@ -16,6 +16,7 @@ - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) - fixed: `versions` and `auto` feature with enabled `F.wait()` - fixed: `LOAD('release')` a release mode +- fixed: `SchemaInstance.$clean()` for nested schemas - improved: NoSQL reader From 9893afcb766cafc540cbf8a544d772db83e9691f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 12:04:13 +0200 Subject: [PATCH 0627/1669] Adde NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method. --- changes.txt | 1 + nosql.js | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 0f37582f6..f6190f4d4 100755 --- a/changes.txt +++ b/changes.txt @@ -6,6 +6,7 @@ - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` - added: support for `SameSite` cookie attribute +- added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` diff --git a/nosql.js b/nosql.js index 2ff84898c..b9c84f1aa 100755 --- a/nosql.js +++ b/nosql.js @@ -892,6 +892,7 @@ DP.modify = function(doc, insert) { for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { + case '!': case '+': case '-': case '*': @@ -1500,6 +1501,9 @@ DP.$update = function() { doc[key] = val(doc[key], doc); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { + case '!': + doc[key] = !doc[key]; + break; case '+': doc[key] = (doc[key] || 0) + val; break; @@ -1624,6 +1628,9 @@ DP.$update_inmemory = function() { doc[key] = val(doc[key], doc); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { + case '!': + doc[key] = !doc[key]; + break; case '+': doc[key] = (doc[key] || 0) + val; break; @@ -5395,6 +5402,7 @@ TP.modify = function(doc, insert) { for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { + case '!': case '+': case '-': case '*': @@ -5874,6 +5882,9 @@ TP.$update = function() { doc[key] = val(doc[key], doc); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { + case '!': + doc[key] = doc[key] == null ? true : !doc[key]; + break; case '+': doc[key] = (doc[key] || 0) + val; break; @@ -5938,7 +5949,6 @@ TP.$update = function() { filters.compare2(arr, update, updateflush); }; - fs.$callback = function() { fs = null; @@ -6505,10 +6515,14 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { NoSQLReader.prototype.compare2 = function(docs, custom, done) { var self = this; + for (var i = 0; i < docs.length; i++) { var doc = docs[i]; + if (doc === EMPTYOBJECT) + continue; + if (self.builders.length === self.canceled) return false; From a6090ea357bcfa9ead139a6eeedd7bc95fa5fc06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 23:28:00 +0200 Subject: [PATCH 0628/1669] Fixed critical bug with updating in NoSQL embedded. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index b9c84f1aa..d74df3711 100755 --- a/nosql.js +++ b/nosql.js @@ -1522,7 +1522,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -1649,7 +1649,7 @@ DP.$update_inmemory = function() { } } } else - docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(old); @@ -5903,7 +5903,7 @@ TP.$update = function() { } } } else - docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); From 020cf47b048c46fe6c1ebcdd4565e7bd698d232e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 23:43:10 +0200 Subject: [PATCH 0629/1669] Fixed a problem with `repository` when updating. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index d74df3711..177293b08 100755 --- a/nosql.js +++ b/nosql.js @@ -1522,7 +1522,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -1649,7 +1649,7 @@ DP.$update_inmemory = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(old); @@ -5903,7 +5903,7 @@ TP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); From 3bc2833f04d0702fc124dc93ff93f26f3f55c00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 10 Sep 2018 08:08:10 +0200 Subject: [PATCH 0630/1669] Improved quicksort. --- utils.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/utils.js b/utils.js index 1530d9c5f..bbe2be5c8 100755 --- a/utils.js +++ b/utils.js @@ -4700,10 +4700,20 @@ AP.quicksort = AP.orderBy = function(name, asc) { if (typeof(name) === 'boolean') { asc = name; name = undefined; - } - - if (asc === undefined) + } else if (asc === undefined) asc = true; + else { + switch (asc) { + case 'asc': + case 'ASC': + asc = true; + break; + case 'desc': + case 'DESC': + asc = false; + break; + } + } var self = this; var type = 0; From c0e6378c8aabd3f7e33d214836b1feb27353b5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 10 Sep 2018 10:41:07 +0200 Subject: [PATCH 0631/1669] Improved OPERATIONS. --- builders.js | 125 +++++++++++++++++++++++++++++++++++++++++++++------- changes.txt | 2 + 2 files changed, 111 insertions(+), 16 deletions(-) diff --git a/builders.js b/builders.js index 4d37e1409..040fac05a 100755 --- a/builders.js +++ b/builders.js @@ -3046,6 +3046,9 @@ function ErrorBuilder(onResource) { // Hidden: when the .push() contains a classic Error instance // this.unexpected; + // A default path for .push() + // this.path; + !onResource && this._resource(); } @@ -3391,6 +3394,9 @@ ErrorBuilder.prototype.push = function(name, error, path, index, prefix) { path = undefined; } + if (this.path && !path) + path = this.path; + if (!error && typeof(name) === 'string') { var m = name.length; if (m > 15) @@ -4589,7 +4595,10 @@ function $decodeURIComponent(value) { } } -global.NEWOPERATION = function(name, fn) { +global.NEWOPERATION = function(name, fn, repeat, stop) { + + // @repeat {Number} How many times will be the operation repeated after error? + // @stop {Boolean} Stop when the error is thrown // Remove operation if (fn == null) { @@ -4600,6 +4609,9 @@ global.NEWOPERATION = function(name, fn) { operations[name] = fn; operations[name].$owner = F.$owner(); operations[name].$newversion = REGEXP_NEWOPERATION.test(fn.toString()); + operations[name].$repeat = repeat; + operations[name].$stop = stop; + return this; }; @@ -4618,25 +4630,32 @@ global.OPERATION = function(name, value, callback, param, controller) { if (fn) { if (fn.$newversion) { var self = new OperationOptions(error, value, param, controller); - if (callback && callback !== NOOP) { - self.callback = function(value) { - if (arguments.length > 1) { - if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { - self.error.push(value); - value = EMPTYOBJECT; - } else - value = arguments[1]; - } else if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + self.$repeat = fn.$repeat; + self.callback = function(value) { + + if (arguments.length > 1) { + if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { self.error.push(value); value = EMPTYOBJECT; - } + } else + value = arguments[1]; + } else if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + self.error.push(value); + value = EMPTYOBJECT; + } + + if (self.error.items.length && self.$repeat) { + self.error.clear(); + self.$repeat--; + fn(self); + } else + callback && callback(self.error.hasError() ? self.error : null, value, self.options); + + return self; + }; - callback(self.error.hasError() ? self.error : null, value, self.options); - return self; - }; - } else - self.callback = NOOP; fn(self); + } else fn(error, value, function(value) { if (callback) { @@ -4653,6 +4672,80 @@ global.OPERATION = function(name, value, callback, param, controller) { } }; +global.RUN = function(name, value, callback, param, controller) { + + if (typeof(value) === 'function') { + controller = param; + param = callback; + callback = value; + value = EMPTYOBJECT; + } + + if (typeof(name) === 'string') + name = name.split(',').trim(); + + var error = new ErrorBuilder(); + var opt = new OperationOptions(error, value, param, controller); + + opt.meta = {}; + opt.meta.items = name; + opt.response = {}; + + opt.callback = function(value) { + + if (arguments.length > 1) { + if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + opt.error.push(value); + value = EMPTYOBJECT; + } else + value = arguments[1]; + } else if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + opt.error.push(value); + value = EMPTYOBJECT; + } + + if (opt.error.items.length && opt.$repeat > 0) { + opt.error.clear(); + opt.$repeat--; + opt.repeated = true; + setImmediate(opt => opt.$current(opt), opt); + } else { + opt.error.items.length && error.push(opt.error); + if (opt.error.items.length && opt.$current.$stop) { + name = null; + opt.next = null; + callback(error, opt.response, opt); + } else { + opt.response[opt.meta.current] = value; + opt.meta.prev = opt.meta.current; + opt.$next(); + } + } + }; + + name.wait(function(key, next, index) { + + var fn = operations[key]; + if (!fn) { + // What now? + // F.error('Operation "{0}" not found'.format(key), 'RUN()'); + return next(); + } + + opt.repeated = false; + opt.error = new ErrorBuilder(); + opt.error.path = 'operation: ' + key; + opt.meta.index = index; + opt.meta.current = key; + opt.$repeat = fn.$repeat; + opt.$current = fn; + opt.$next = next; + opt.meta.next = name[index]; + fn(opt); + + }, () => callback(error.items.length ? error : null, opt.response, opt)); +}; + function OperationOptions(error, value, options, controller) { if (!controller && options instanceof global.Controller) { diff --git a/changes.txt b/changes.txt index f6190f4d4..39d46fae8 100755 --- a/changes.txt +++ b/changes.txt @@ -7,8 +7,10 @@ - added: `.bundlesignore` support with similiar functionality like `.gitignore` - added: support for `SameSite` cookie attribute - added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method +- added: `RUN()` for executing multiple Total.js operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` +- updated: `NEWOPERATION()` supports `repeat` and `stop` argument - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function From ce5907332454a106216f714f75d722250d1afec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 10 Sep 2018 13:21:05 +0200 Subject: [PATCH 0632/1669] Improved `RUN()`. --- builders.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builders.js b/builders.js index 040fac05a..54168ca23 100755 --- a/builders.js +++ b/builders.js @@ -4681,6 +4681,12 @@ global.RUN = function(name, value, callback, param, controller) { value = EMPTYOBJECT; } + if (param instanceof global.Controller || (param && param.isWebSocket)) { + controller = param; + param = EMPTYOBJECT; + } else if (param instanceof OperationOptions) + controller = param.controller; + if (typeof(name) === 'string') name = name.split(',').trim(); @@ -4690,6 +4696,7 @@ global.RUN = function(name, value, callback, param, controller) { opt.meta = {}; opt.meta.items = name; opt.response = {}; + opt.errors = error; opt.callback = function(value) { From 66ac4b0c6f9b64f35e3d3bedaecc6572bbd30ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 11 Sep 2018 09:52:05 +0200 Subject: [PATCH 0633/1669] Fixed extracting bundles. --- bundles.js | 26 +++--- changes.txt | 1 + test/test-tmp.js | 211 ++++------------------------------------------- 3 files changed, 31 insertions(+), 207 deletions(-) diff --git a/bundles.js b/bundles.js index a8fc63ed5..7eaa944cc 100755 --- a/bundles.js +++ b/bundles.js @@ -33,10 +33,9 @@ exports.make = function(callback) { blacklist[F.config['directory-temp']] = 1; blacklist[F.config['directory-bundles']] = 1; blacklist[F.config['directory-src']] = 1; + blacklist[F.config['directory-logs']] = 1; blacklist['/node_modules/'] = 1; - // blacklist['/debug.js'] = 1; blacklist['/debug.pid'] = 1; - //blacklist['/package.json'] = 1; blacklist['/package-lock.json'] = 1; var Files = []; @@ -118,7 +117,8 @@ exports.make = function(callback) { for (var i = 0, length = files.length; i < length; i++) { var file = files[i].substring(Length); var type = 0; - if (file.startsWith(F.config['directory-databases'])) + + if (file.startsWith(F.config['directory-databases']) || file.startsWith('/flow/') || file.startsWith('/dashboard/')) type = 1; else if (file.startsWith(F.config['directory-public'])) type = 2; @@ -158,7 +158,7 @@ exports.make = function(callback) { function makeignore(arr) { var ext; - var code = ['var path=P.substring(0,P.lastIndexOf(\'/\') + 1);', 'var ext=U.getExtension(P);', 'var name=U.getName(P).replace(\'.\' + ext, \'\');']; + var code = ['var path=P.substring(0,P.lastIndexOf(\'/\')+1);', 'var ext=U.getExtension(P);', 'var name=U.getName(P).replace(\'.\'+ ext,\'\');']; for (var i = 0; i < arr.length; i++) { var item = arr[i]; @@ -215,10 +215,13 @@ function cleanFiles(callback) { if (meta.files && meta.files.length) { for (var i = 0, length = meta.files.length; i < length; i++) { var filename = meta.files[i]; - try { - F.consoledebug('Remove', filename); - Fs.unlinkSync(Path.join(path, filename)); - } catch (e) {} + var dir = filename.substring(0, filename.indexOf('/', 1) + 1); + if (!blacklist[dir]) { + try { + F.consoledebug('Remove', filename); + Fs.unlinkSync(Path.join(path, filename)); + } catch (e) {} + } } } @@ -226,7 +229,8 @@ function cleanFiles(callback) { meta.directories.quicksort('length', false); for (var i = 0, length = meta.directories.length; i < length; i++) { try { - Fs.rmdirSync(Path.join(path, meta.directories[i])); + if (!blacklist[meta.directories[i]]) + Fs.rmdirSync(Path.join(path, meta.directories[i])); } catch (e) {} } } @@ -285,9 +289,9 @@ function copyFiles(files, callback) { append = true; if (CONSOLE && exists) { - F.config['allow-debug'] && F.consoledebug(append ? 'EXT: ' : 'REW:', p); + F.config['allow-debug'] && F.consoledebug(append ? 'EXT:' : 'REW:', p); } else - F.consoledebug(append ? 'EXT:' : 'COP:', p); + F.consoledebug(append ? 'EXT:' : 'COP:', p); if (append) { Fs.appendFile(filename, '\n' + Fs.readFileSync(file.filename).toString('utf8'), next); diff --git a/changes.txt b/changes.txt index 39d46fae8..b375a2380 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - fixed: `versions` and `auto` feature with enabled `F.wait()` - fixed: `LOAD('release')` a release mode - fixed: `SchemaInstance.$clean()` for nested schemas +- fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) - improved: NoSQL reader diff --git a/test/test-tmp.js b/test/test-tmp.js index b03fe94bc..b6aceaf82 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,201 +1,20 @@ require('../index'); +// NOSQLMEMORY('test'); +// NOSQL('test').find2().take(10).callback(console.log); +// NOSQL('test').count().callback(console.log); +// NOSQL('test').remove().between('index', 0, 10).callback(console.log).backup('fet'); +// NOSQL('test').remove().between('index', 1, 3).callback(console.log); +// NOSQL('test').on('modify', console.log); +// NOSQL('test').update({ name: GUID(5) }).between('index', 1, 3).callback(console.log); -function SQL(query) { +F.config['table.test'] = 'index:number | name:string'; - var self = this; - self.options = {}; - self.builder = new framework_nosql.DatabaseBuilder(null); - self.query = query; +TABLE('test').find().take(10).skip(10).callback(console.log); - var type = self.parseType(); +// TABLE('test').modify({ name: GUID(5) }).between('index', 3, 5).callback(console.log); +// TABLE('test').remove().between('index', 0, 2).callback(console.log); - switch (type) { - case 'select': - self.parseLimit(); - self.parseOrder(); - self.parseWhere(); - self.parseJoins(); - self.parseTable(); - self.parseNames(); - break; - case 'update': - self.parseWhere(); - self.parseTable(); - self.parseUpdate(); - break; - case 'insert': - self.parseWhere(); - self.parseInsert(); - break; - case 'delete': - self.parseWhere(); - self.parseTable(); - break; - } - - console.log(self.options); -} - -var SQLP = SQL.prototype; - -SQLP.parseLimit = function() { - var self = this; - var tmp = self.query.match(/take \d+ skip \d+$/i); - !tmp && (tmp = self.query.match(/skip \d+ take \d+$/i)); - !tmp && (tmp = self.query.match(/take \d+$/i)); - !tmp && (tmp = self.query.match(/skip \d+$/i)); - if (!tmp) - return self; - self.query = self.query.replace(tmp, '').trim(); - var arr = tmp[0].toString().toLowerCase().split(' '); - if (arr[0] === 'take') - self.options.take = +arr[1]; - else if (arr[2] === 'take') - self.options.take = +arr[3]; - if (arr[0] === 'skip') - self.options.skip = +arr[1]; - else if (arr[2] === 'skip') - self.options.skip = +arr[3]; - return self; -}; - -SQLP.parseOrder = function() { - var self = this; - var tmp = self.query.match(/order by .*?$/i); - - if (!tmp) - return self; - - self.query = self.query.replace(tmp, '').trim(); - var arr = tmp[0].toString().substring(9).split(','); - - self.options.sort = []; - - for (var i = 0; i < arr.length; i++) { - tmp = arr[i].trim().split(' '); - self.options.sort.push({ name: tmp[0], desc: (tmp[1] || '').toLowerCase() === 'desc' }); - } - - return self; -}; - -SQLP.parseWhere = function() { - var self = this; - var tmp = self.query.match(/where .*?$/i); - if (!tmp) - return self; - self.query = self.query.replace(tmp, ''); - - tmp = tmp[0].toString().substring(6).replace(/\sAND\s/gi, ' && ').replace(/\sOR\s/gi, ' || ').replace(/[a-z0-9]=/gi, function(text) { - return text + '='; - }); - - self.options.where = tmp; - return self; -}; - -SQLP.parseJoins = function() { - var self = this; - var tmp = self.query.match(/left join.*?$/i); - if (!tmp) { - tmp = self.query.match(/join.*?$/i); - if (!tmp) - return self; - } - self.query = self.query.replace(tmp, ''); - tmp = tmp[0].toString().trim(); - - self.options.joins = []; - - tmp = tmp.substring(5).split(/\s?JOIN\s/i); - for (var i = 0; i < tmp.length; i++) { - var join = tmp[i].split(/\son\s/i); - self.options.joins.push({ table: join[0], condition: join[1] }); - } - - return self; -}; - -SQLP.parseTable = function() { - var self = this; - var tmp = self.query.match(/from\s.*?$/i); - if (!tmp) - return self; - self.query = self.query.replace(tmp, ''); - tmp = tmp[0].toString().substring(5).trim(); - - var arr = tmp.split(' '); - // console.log(arr); - - return self; -}; - -SQLP.parseNames = function() { - var self = this; - var tmp = self.query.match(/select\s.*?$/i); - if (!tmp) - return self; - - self.query = self.query.replace(tmp, ''); - tmp = tmp[0].toString().substring(6).trim().split(','); - - self.options.fields = []; - - for (var i = 0; i < tmp.length; i++) { - var field = tmp[i].trim(); - var alias = field.match(/as\s.*?$/); - var name = ''; - var type = 0; - - if (alias) { - field = field.replace(alias, ''); - alias = alias.toString().substring(3); - } - - var index = field.indexOf('('); - if (index !== -1) { - switch (field.substring(0, index).toLowerCase()) { - case 'count': - type = 1; - break; - case 'min': - type = 2; - break; - case 'max': - type = 3; - break; - case 'avg': - type = 4; - break; - case 'sum': - type = 5; - break; - case 'distinct': - type = 6; - break; - } - name = field.substring(index + 1, field.lastIndexOf(')')); - } else - name = field; - self.options.fields.push({ alias: alias || name, name: name, type: type }); - } - - return self; -}; - -SQLP.parseUpdate = function() { - var self = this; - return self; -}; - -SQLP.parseInsert = function() { - var self = this; - return self; -}; - -SQLP.parseType = function() { - var self = this; - return self.query.substring(0, self.query.indexOf(' ')).toLowerCase(); -}; - -var sql = new SQL('SELECT COUNT(*) as count, id FROM table a JOIN users ON users.id=a.id JOIN orders ON orders.id=a.id WHERE a.name="Peter" AND a.age=30 ORDER BY a.name, a.age ASC TAKE 20 SKIP 10'); \ No newline at end of file +/* +for (var i = 0; i < 100; i++) + TABLE('test').insert({ index: i }); +*/ \ No newline at end of file From a6a2a6a8d3ca58a66e44f8366826a8cc5c7c2593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 11 Sep 2018 13:51:24 +0200 Subject: [PATCH 0634/1669] New improvements. --- changes.txt | 2 + index.js | 171 +++++++++++++++++++++++++++------------------------- internal.js | 6 +- 3 files changed, 94 insertions(+), 85 deletions(-) diff --git a/changes.txt b/changes.txt index b375a2380..5af3b299f 100755 --- a/changes.txt +++ b/changes.txt @@ -8,6 +8,7 @@ - added: support for `SameSite` cookie attribute - added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - added: `RUN()` for executing multiple Total.js operations +- added: a new global alias `CONF` (it's a reference to config) for `F.config` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument @@ -21,6 +22,7 @@ - fixed: `LOAD('release')` a release mode - fixed: `SchemaInstance.$clean()` for nested schemas - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) +- fixed: subdomain routing for `localhost` - improved: NoSQL reader diff --git a/index.js b/index.js index f42863185..e46acdb7c 100755 --- a/index.js +++ b/index.js @@ -62,6 +62,7 @@ const REG_WINDOWSPATH = /\\/g; const REG_SCRIPTCONTENT = /<|>|;/; const REG_HTTPHTTPS = /^(\/)?(http|https):\/\//i; const REG_NOCOMPRESS = /[.|-]+min(@[a-z0-9]*)?\.(css|js)$/i; +const REG_WWW = /^www\./i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; @@ -625,13 +626,15 @@ var PERF = {}; function Framework() { - this.$id = null; // F.id ==> property - this.version = 3100; - this.version_header = '3.1.0'; - this.version_node = process.version.toString(); - this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); + var self = this; + + self.$id = null; // F.id ==> property + self.version = 3100; + self.version_header = '3.1.0'; + self.version_node = process.version.toString(); + self.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); - this.config = { + global.CONF = self.config = { debug: true, trace: true, @@ -640,8 +643,8 @@ function Framework() { name: 'Total.js', version: '1.0.0', author: '', - secret: this.syshash, - 'secret-uid': this.syshash.substring(10), + secret: self.syshash, + 'secret-uid': self.syshash.substring(10), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -746,24 +749,24 @@ function Framework() { 'default-interval-uptodate': 5 }; - global.G = this.global = {}; - this.$bundling = true; - this.resources = {}; - this.connections = {}; - this.functions = {}; - this.themes = {}; - this.versions = null; - this.workflows = {}; - this.uptodates = null; - this.schedules = []; - - this.isDebug = true; - this.isTest = false; - this.isLoaded = false; - this.isWorker = true; - this.isCluster = process.env.PASSENGER_APP_ENV ? false : require('cluster').isWorker; - - this.routes = { + global.G = self.global = {}; + self.$bundling = true; + self.resources = {}; + self.connections = {}; + self.functions = {}; + self.themes = {}; + self.versions = null; + self.workflows = {}; + self.uptodates = null; + self.schedules = []; + + self.isDebug = true; + self.isTest = false; + self.isLoaded = false; + self.isWorker = true; + self.isCluster = process.env.PASSENGER_APP_ENV ? false : require('cluster').isWorker; + + self.routes = { sitemap: null, web: [], system: {}, @@ -783,27 +786,27 @@ function Framework() { resources: {} }; - this.owners = []; - this.modificators = null; - this.helpers = {}; - this.modules = {}; - this.models = {}; - this.sources = {}; - this.controllers = {}; - this.dependencies = {}; - this.isomorphic = {}; - this.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; - this.convertors = []; - this.convertors2 = null; - this.tests = []; - this.errors = []; - this.problems = []; - this.changes = []; - this.server = null; - this.port = 0; - this.ip = ''; - - this.validators = { + self.owners = []; + self.modificators = null; + self.helpers = {}; + self.modules = {}; + self.models = {}; + self.sources = {}; + self.controllers = {}; + self.dependencies = {}; + self.isomorphic = {}; + self.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; + self.convertors = []; + self.convertors2 = null; + self.tests = []; + self.errors = []; + self.problems = []; + self.changes = []; + self.server = null; + self.port = 0; + self.ip = ''; + + self.validators = { email: new RegExp('^[a-zA-Z0-9-_.+]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$'), url: /^(https?:\/\/(?:www\.|(?!www))[^\s.#!:?+=&@!$'~*,;/()[\]]+\.[^\s#!?+=&@!$'~*,;()[\]\\]{2,}\/?|www\.[^\s#!:.?+=&@!$'~*,;/()[\]]+\.[^\s#!?+=&@!$'~*,;()[\]\\]{2,}\/?)/i, phone: /^[+]?[(]?[0-9]{3}[)]?[-\s.]?[0-9]{3}[-\s.]?[0-9]{4,6}$/im, @@ -811,17 +814,17 @@ function Framework() { uid: /^\d{14,}[a-z]{3}[01]{1}$/ }; - this.workers = {}; - this.databases = {}; - this.databasescleaner = {}; - this.directory = HEADERS.workers.cwd = directory; - this.isLE = Os.endianness ? Os.endianness() === 'LE' : true; - this.isHTTPS = false; + self.workers = {}; + self.databases = {}; + self.databasescleaner = {}; + self.directory = HEADERS.workers.cwd = directory; + self.isLE = Os.endianness ? Os.endianness() === 'LE' : true; + self.isHTTPS = false; // It's hidden - // this.waits = {}; + // self.waits = {}; - this.temporary = { + self.temporary = { path: {}, notfound: {}, processing: {}, @@ -836,7 +839,7 @@ function Framework() { service: { redirect: 0, request: 0, file: 0 } }; - this.stats = { + self.stats = { performance: { request: 0, @@ -904,29 +907,29 @@ function Framework() { }; // intialize cache - this.cache = new FrameworkCache(); - this.path = new FrameworkPath(); - - this._request_check_redirect = false; - this._request_check_referer = false; - this._request_check_POST = false; - this._request_check_robot = false; - this._request_check_mobile = false; - this._length_middleware = 0; - this._length_request_middleware = 0; - this._length_files = 0; - this._length_wait = 0; - this._length_themes = 0; - this._length_cors = 0; - this._length_subdomain_web = 0; - this._length_subdomain_websocket = 0; - this._length_convertors = 0; - - this.isVirtualDirectory = false; - this.isTheme = false; - this.isWindows = Os.platform().substring(0, 3).toLowerCase() === 'win'; - - this.$events = {}; + self.cache = new FrameworkCache(); + self.path = new FrameworkPath(); + + self._request_check_redirect = false; + self._request_check_referer = false; + self._request_check_POST = false; + self._request_check_robot = false; + self._request_check_mobile = false; + self._length_middleware = 0; + self._length_request_middleware = 0; + self._length_files = 0; + self._length_wait = 0; + self._length_themes = 0; + self._length_cors = 0; + self._length_subdomain_web = 0; + self._length_subdomain_websocket = 0; + self._length_convertors = 0; + + self.isVirtualDirectory = false; + self.isTheme = false; + self.isWindows = Os.platform().substring(0, 3).toLowerCase() === 'win'; + + self.$events = {}; } // ====================================================== @@ -6465,7 +6468,7 @@ F.initialize = function(http, debug, options) { var ip = options.ip; var listenpath = options.listenpath; - options.config && U.extend(F.config, options.config, true); + options.config && U.extend_headers2(F.config, options.config); if (options.debug || options['allow-debug']) F.config['allow-debug'] = true; @@ -14189,9 +14192,11 @@ function extend_request(PROTO) { get: function() { if (this._subdomain) return this._subdomain; - var subdomain = this.uri.host.toLowerCase().replace(/^www\./i, '').split('.'); - if (subdomain.length > 2) - this._subdomain = subdomain.slice(0, subdomain.length - 2); // example: [subdomain].domain.com + var subdomain = this.uri.hostname.toLowerCase().replace(REG_WWW, '').split('.'); + if (subdomain.length > 2) // example: [subdomain].domain.com + this._subdomain = subdomain.slice(0, subdomain.length - 2); + else if (subdomain.length > 1 && subdomain[subdomain.length - 1] === 'localhost') // example: [subdomain].localhost + this._subdomain = subdomain.slice(0, subdomain.length - 1); else this._subdomain = null; return this._subdomain; diff --git a/internal.js b/internal.js index 4762f30f8..462cdbb0c 100755 --- a/internal.js +++ b/internal.js @@ -59,7 +59,7 @@ const REG_HEAD = /<\/head>/i; const REG_COMPONENTS = /@{(\s)?(component|components)(\s)?\(/i; const REG_COMPONENTS_GROUP = /('|")[a-z0-9]+('|")/i; const HTTPVERBS = { 'get': true, 'post': true, 'options': true, 'put': true, 'delete': true, 'patch': true, 'upload': true, 'head': true, 'trace': true, 'propfind': true }; -const RENDERNOW = ['self.$import(', 'self.route', 'self.$js(', 'self.$css(', 'self.$favicon(', 'self.$script(', '$STRING(self.resource(', '$STRING(RESOURCE(', 'self.translate(', 'language', 'self.sitemap_url(', 'self.sitemap_name(', '$STRING(CONFIG(', '$STRING(config.', '$STRING(config[', '$STRING(config(']; +const RENDERNOW = ['self.$import(', 'self.route', 'self.$js(', 'self.$css(', 'self.$favicon(', 'self.$script(', '$STRING(self.resource(', '$STRING(RESOURCE(', 'self.translate(', 'language', 'self.sitemap_url(', 'self.sitemap_name(', '$STRING(CONFIG(', '$STRING(config.', '$STRING(config[', '$STRING(CONF.', '$STRING(CONF[', '$STRING(config(']; const REG_NOTRANSLATE = /@\{notranslate\}/gi; const REG_NOCOMPRESS = /@\{nocompress\s\w+}/gi; const REG_TAGREMOVE = /[^>](\r)\n\s{1,}$/; @@ -80,7 +80,7 @@ const REG_CSS_11 = /\$.*?(;|\})/gi; const REG_CSS_12 = /(margin|padding):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; -const ALLOWEDMARKUP = { G: 1, M: 1, R: 1, repository: 1, model: 1, config: 1, global: 1, resource: 1, RESOURCE: 1, CONFIG: 1, author: 1, root: 1, functions: 1, NOW: 1, F: 1 }; +const ALLOWEDMARKUP = { G: 1, M: 1, R: 1, repository: 1, model: 1, CONF: 1, config: 1, global: 1, resource: 1, RESOURCE: 1, CONFIG: 1, author: 1, root: 1, functions: 1, NOW: 1, F: 1 }; var INDEXFILE = 0; @@ -2094,6 +2094,7 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'session': case 'user': case 'config': + case 'CONF': case 'controller': return view_is_assign(command) ? ('self.$set(' + command + ')') : ('$STRING(' + command + ').encode()'); @@ -2133,6 +2134,7 @@ function view_prepare(command, dynamicCommand, functions, controller) { case '!session': case '!user': case '!config': + case '!CONF': case '!functions': case '!model': case '!CONFIG': From 349f7ad87adc2528a4cea3099e85fb18a92eec32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 11 Sep 2018 14:32:44 +0200 Subject: [PATCH 0635/1669] Fixed `GET` priority in `ROUTE()`. --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index e46acdb7c..5f115f324 100755 --- a/index.js +++ b/index.js @@ -2146,6 +2146,10 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu tmp.push(flag); break; } + + if (flag === 'get') + priority -= 2; + } if (isROLE && !membertype) { From 5f7f942eb6922ea3b53dec7f5c3f224c504e587a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 13 Sep 2018 12:49:17 +0200 Subject: [PATCH 0636/1669] Improved `ROUTE()` by adding operations. --- builders.js | 4 ++-- changes.txt | 1 + index.js | 43 +++++++++++++++++++++++++++++++------------ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/builders.js b/builders.js index 54168ca23..c1ddb611a 100755 --- a/builders.js +++ b/builders.js @@ -4672,7 +4672,7 @@ global.OPERATION = function(name, value, callback, param, controller) { } }; -global.RUN = function(name, value, callback, param, controller) { +global.RUN = function(name, value, callback, param, controller, result) { if (typeof(value) === 'function') { controller = param; @@ -4750,7 +4750,7 @@ global.RUN = function(name, value, callback, param, controller) { opt.meta.next = name[index]; fn(opt); - }, () => callback(error.items.length ? error : null, opt.response, opt)); + }, () => callback(error.items.length ? error : null, result ? opt.response[result] : opt.response, opt)); }; function OperationOptions(error, value, options, controller) { diff --git a/changes.txt b/changes.txt index 5af3b299f..e57ec0ac9 100755 --- a/changes.txt +++ b/changes.txt @@ -12,6 +12,7 @@ - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument +- updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function diff --git a/index.js b/index.js index 5f115f324..bd0c44b3b 100755 --- a/index.js +++ b/index.js @@ -1851,7 +1851,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu } - url = url.replace(/(^|\s?)\*[a-z0-9].*?$/i, function(text) { + url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; @@ -2024,20 +2024,23 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu workflow = null; } - schema = schema.replace(/\\/g, '/').split('/'); + schema = schema.replace(/\\/g, '/').split('/').trim(); - if (schema.length === 1) { - schema[1] = schema[0]; - schema[0] = 'default'; - } + if (schema.length) { - index = schema[1].indexOf('#'); - if (index !== -1) { - schema[2] = schema[1].substring(index + 1).trim(); - schema[1] = schema[1].substring(0, index).trim(); - (schema[2] && schema[2][0] !== '*') && (schema[2] = '*' + schema[2]); - } + if (schema.length === 1) { + schema[1] = schema[0]; + schema[0] = 'default'; + } + index = schema[1].indexOf('#'); + if (index !== -1) { + schema[2] = schema[1].substring(index + 1).trim(); + schema[1] = schema[1].substring(0, index).trim(); + (schema[2] && schema[2][0] !== '*') && (schema[2] = '*' + schema[2]); + } + + } // else it's an operation continue; } @@ -16618,8 +16621,17 @@ function controller_json_workflow(id) { var self = this; self.id = id; var w = self.route.workflow; + if (w instanceof Object) { if (!w.type) { + + + // IS IT AN OPERATION? + if (!self.route.schema.length) { + OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); + return; + } + var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { @@ -16662,6 +16674,13 @@ function controller_json_workflow_multiple(id) { var w = self.route.workflow; if (w instanceof Object) { if (!w.type) { + + // IS IT AN OPERATION? + if (!self.route.schema.length) { + RUN(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), null, self, w.index ? w.id[w.index] : null); + return; + } + var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { From ae9c62fb580f6128633b27f5ee1560ffd472d4a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 13 Sep 2018 16:31:20 +0200 Subject: [PATCH 0637/1669] Added `nosql builder cache` (uncompleted). --- nosql.js | 305 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 190 insertions(+), 115 deletions(-) diff --git a/nosql.js b/nosql.js index 177293b08..64183e42a 100755 --- a/nosql.js +++ b/nosql.js @@ -2408,6 +2408,13 @@ DatabaseBuilder.prototype.join = function(field, name) { return join; }; +DatabaseBuilder.prototype.statement = function(id, fn) { + this.id = id; + this.$iscache = CACHE[self.db.name + '_' + id] == null; + fn.call(this, this); + return this; +}; + DatabaseBuilder.prototype.first = function() { this.$options.first = true; return this.take(1); @@ -2425,13 +2432,15 @@ DatabaseBuilder.prototype.filter = function(fn) { self.$functions = []; var index = self.$functions.push(fn) - 1; - var code = '$is=!!fn[{0}].call($F,doc,index,repository);'.format(index); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = '$is=!!fn[{0}].call($F,doc,index,repository);'.format(index); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; return self; }; @@ -2446,33 +2455,43 @@ DatabaseBuilder.prototype.scalar = function(type, name) { DatabaseBuilder.prototype.contains = function(name) { var self = this; - var code = '$is=doc.{0} instanceof Array?!!doc.{0}.length:!!doc.{0};'.format(name); - if (self.$scope) - code = 'if(!$is){' + code + '}'; - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + + if (!self.$iscache) { + var code = '$is=doc.{0} instanceof Array?!!doc.{0}.length:!!doc.{0};'.format(name); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } + self.$counter++; return self; }; DatabaseBuilder.prototype.empty = function(name) { var self = this; - var code = '$is=doc.{0} instanceof Array?!doc.{0}.length:!doc.{0};'.format(name); - if (self.$scope) - code = 'if(!$is){' + code + '}'; - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + + if (!self.$iscache) { + var code = '$is=doc.{0} instanceof Array?!doc.{0}.length:!doc.{0};'.format(name); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } + self.$counter++; return self; }; DatabaseBuilder.prototype.map = function(name, code) { var self = this; - var data = { name: name, code: code }; - if (self.$options.mappers) - self.$options.mappers.push(data); - else - self.$options.mappers = [data]; + if (!self.$iscache) { + var data = { name: name, code: code }; + if (self.$options.mappers) + self.$options.mappers.push(data); + else + self.$options.mappers = [data]; + } return self; }; @@ -2503,39 +2522,43 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { var date = framework_utils.isDate(value); self.$params[key] = date ? value.getTime() : value; - switch (operator) { - case '=': - operator = '=='; - break; - case '<>': - operator = '!='; - break; - } - - code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); + if (!self.$iscache) { + switch (operator) { + case '=': + operator = '=='; + break; + case '<>': + operator = '!='; + break; + } - if (self.$scope) - code = 'if(!$is){' + code + '}'; + code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code.format(name, key, operator)); + !self.$scope && self.$code.push('if(!$is)return;'); + self.$keys && self.$keys.push(name); + } - self.$keys && self.$keys.push(name); - self.$code.push(code.format(name, key, operator)); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; DatabaseBuilder.prototype.query = function(code) { var self = this; - code = '$is=(' + code + ');'; + if (!self.$iscache) { + code = '$is=(' + code + ');'; - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (self.$scope) + code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys = null; + if (self.$keys) + self.$keys = null; + + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; return self; }; @@ -2551,13 +2574,14 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('month', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + if (!self.$iscache) { + var code = compare_datetype('month', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } return self; }; @@ -2572,13 +2596,15 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('day', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('day', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } return self; }; @@ -2593,14 +2619,17 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('year', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('year', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); + self.$keys && self.$keys.push(name); + + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -2615,13 +2644,16 @@ DatabaseBuilder.prototype.hour = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('hour', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('hour', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -2636,13 +2668,16 @@ DatabaseBuilder.prototype.minute = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('minute', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('minute', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -2652,44 +2687,55 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam var code; var key = 'l' + (self.$counter++); - if (!where) - where = '*'; + if (!self.$iscache) { + if (!where) + where = '*'; - switch (where) { - case 'beg': - code = '$is=doc.{0}?doc.{0}.startsWith(arg.{1}):false;'; - break; - case 'end': - code = '$is=doc.{0}?doc.{0}.endsWith(arg.{1}):false;'; - break; - case '*': - code = '$is=false;if(doc.{0}){if(doc.{0} instanceof Array){for(var $i=0;$i> 0; - - var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i> 0; + var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Thu, 13 Sep 2018 19:10:47 +0200 Subject: [PATCH 0638/1669] Improved code. --- nosql.js | 185 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 107 insertions(+), 78 deletions(-) diff --git a/nosql.js b/nosql.js index 64183e42a..34b4421c0 100755 --- a/nosql.js +++ b/nosql.js @@ -2408,19 +2408,18 @@ DatabaseBuilder.prototype.join = function(field, name) { return join; }; -DatabaseBuilder.prototype.statement = function(id, fn) { - this.id = id; - this.$iscache = CACHE[self.db.name + '_' + id] == null; - fn.call(this, this); - return this; -}; - DatabaseBuilder.prototype.first = function() { this.$options.first = true; return this.take(1); }; -DatabaseBuilder.prototype.make = function(fn) { +DatabaseBuilder.prototype.make = function(id, fn) { + if (fn == null) { + fn = id; + } else { + this.$options.id = id; + this.$iscache = !!CACHE[this.db.name + '_' + id]; + } fn.call(this, this); return this; }; @@ -2743,6 +2742,7 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { var self = this; var key = 'l' + (self.$counter++); + var key2 = 'l' + (self.$counter++); if (value instanceof Array) { for (var i = 0; i < value.length; i++) @@ -2756,17 +2756,22 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { self.$params[key] = value; + var count = 1; + + if (weight) + count = ((value.length / 100) * weight) >> 0; + + self.$params[key2] = count || 1; + if (!self.$iscache) { self.$keys && self.$keys.push(name); - var count = 1; - if (weight) - count = ((value.length / 100) * weight) >> 0; - var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Thu, 13 Sep 2018 19:17:51 +0200 Subject: [PATCH 0639/1669] Improved code. --- nosql.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/nosql.js b/nosql.js index 34b4421c0..c773820cb 100755 --- a/nosql.js +++ b/nosql.js @@ -2777,21 +2777,22 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function() { + var self = this; var obj = {}; - obj.options = this.$options; - obj.code = this.$code; - obj.params = this.$params; - obj.insert = this.$insertcallback ? this.$insertcallback.toString() : null; + obj.options = self.$options; + obj.code = self.$code; + obj.params = self.$params; + obj.insert = self.$insertcallback ? self.$insertcallback.toString() : null; - if (this.$functions) { + if (self.$functions) { obj.functions = []; - for (var i = 0; i < this.$functions.length; i++) - obj.functions.push(this.$functions[i].toString()); + for (var i = 0; i < self.$functions.length; i++) + obj.functions.push(self.$functions[i].toString()); } - if (this.$repository) - obj.repository = this.$repository; + if (self.$repository) + obj.repository = self.$repository; return JSON.stringify(obj); }; From 38caa139b52aafd96b7c2e2c67c1358dc94a72eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 13 Sep 2018 19:53:58 +0200 Subject: [PATCH 0640/1669] Improved code again. --- nosql.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index c773820cb..7f3230fbf 100755 --- a/nosql.js +++ b/nosql.js @@ -2908,7 +2908,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$mappersexec = cache.mexec; self.$keys = cache.keys; self.$options.sort = cache.sort; - self.$functions = cache.functions; return cache.filter; } @@ -2916,14 +2915,13 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { var code = 'var repository=$F.repository,options=$F.options,arg=$F.arg,fn=$F.fn,$is=false,$tmp;var R=repository;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + (noTrimmer ? 'return doc' : 'if(options.fields){var $doc={};for(var $i=0;$i Date: Sat, 15 Sep 2018 09:44:56 +0200 Subject: [PATCH 0641/1669] Improved OPERATIONS. --- builders.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index c1ddb611a..150b4840a 100755 --- a/builders.js +++ b/builders.js @@ -4714,7 +4714,7 @@ global.RUN = function(name, value, callback, param, controller, result) { if (opt.error.items.length && opt.$repeat > 0) { opt.error.clear(); opt.$repeat--; - opt.repeated = true; + opt.repeated++; setImmediate(opt => opt.$current(opt), opt); } else { opt.error.items.length && error.push(opt.error); @@ -4739,7 +4739,7 @@ global.RUN = function(name, value, callback, param, controller, result) { return next(); } - opt.repeated = false; + opt.repeated = 0; opt.error = new ErrorBuilder(); opt.error.path = 'operation: ' + key; opt.meta.index = index; From 73af913bd8cce068433c94f730ab71f8cdeac6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 15 Sep 2018 23:31:37 +0200 Subject: [PATCH 0642/1669] Fixed NoSQL cleaner + improved code. --- changes.txt | 1 + nosql.js | 67 ++++++++++++++++++++--------------------------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/changes.txt b/changes.txt index e57ec0ac9..103cdb3d8 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,7 @@ - fixed: `SchemaInstance.$clean()` for nested schemas - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) - fixed: subdomain routing for `localhost` +- fixed: service for database cleaner - improved: NoSQL reader diff --git a/nosql.js b/nosql.js index 7f3230fbf..b626935de 100755 --- a/nosql.js +++ b/nosql.js @@ -1580,7 +1580,10 @@ DP.$update = function() { } } - change && self.$events.change && self.emit('change', 'update'); + if (change) { + self.$events.change && self.emit('change', 'update'); + !F.databasescleaner[self.name] && (F.databasescleaner[self.name] = 1); + } }; fs.openupdate(); @@ -1952,7 +1955,10 @@ DP.$remove = function() { fs = null; self.$writting = false; self.next(0); - change && self.$events.change && self.emit('change', 'remove'); + if (change) { + self.$events.change && self.emit('change', 'remove'); + !F.databasescleaner[self.name] && (F.databasescleaner[self.name] = 1); + } }; fs.openupdate(); @@ -2413,10 +2419,8 @@ DatabaseBuilder.prototype.first = function() { return this.take(1); }; -DatabaseBuilder.prototype.make = function(id, fn) { - if (fn == null) { - fn = id; - } else { +DatabaseBuilder.prototype.make = function(fn, id) { + if (id) { this.$options.id = id; this.$iscache = !!CACHE[this.db.name + '_' + id]; } @@ -2530,13 +2534,11 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { operator = '!='; break; } - code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); if (self.$scope) code = 'if(!$is){' + code + '}'; self.$code.push(code.format(name, key, operator)); !self.$scope && self.$code.push('if(!$is)return;'); - self.$keys && self.$keys.push(name); } return self; @@ -2547,13 +2549,8 @@ DatabaseBuilder.prototype.query = function(code) { if (!self.$iscache) { code = '$is=(' + code + ');'; - if (self.$scope) code = 'if(!$is){' + code + '}'; - - if (self.$keys) - self.$keys = null; - self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2562,6 +2559,11 @@ DatabaseBuilder.prototype.query = function(code) { return self; }; +DatabaseBuilder.prototype.param = function(key, value) { + this.$params[key] = value; + return this; +}; + DatabaseBuilder.prototype.month = function(name, operator, value) { var self = this; var key = 'dm' + (self.$counter++); @@ -2577,7 +2579,6 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { var code = compare_datetype('month', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2599,8 +2600,6 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { var code = compare_datetype('day', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2622,9 +2621,6 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { var code = compare_datetype('year', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); - self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2647,8 +2643,6 @@ DatabaseBuilder.prototype.hour = function(name, operator, value) { var code = compare_datetype('hour', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2671,8 +2665,6 @@ DatabaseBuilder.prototype.minute = function(name, operator, value) { var code = compare_datetype('minute', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2710,7 +2702,6 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key)); !self.$scope && self.$code.push('if(!$is)return;'); } else { @@ -2731,7 +2722,6 @@ DatabaseBuilder.prototype.regexp = function(name, value) { var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){$is=({1}).test(doc.{0})}'; if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); self.$code.push(code.format(name, value.toString())); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2764,7 +2754,6 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { self.$params[key2] = count || 1; if (!self.$iscache) { - self.$keys && self.$keys.push(name); var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a=arg.{2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Sat, 15 Sep 2018 23:38:27 +0200 Subject: [PATCH 0643/1669] Added a missing `repository`. --- nosql.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index b626935de..77b855743 100755 --- a/nosql.js +++ b/nosql.js @@ -1498,7 +1498,7 @@ DP.$update = function() { var val = fil.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - doc[key] = val(doc[key], doc); + doc[key] = val(doc[key], doc, f.filter.repository); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { case '!': @@ -5956,7 +5956,7 @@ TP.$update = function() { var val = fil.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - doc[key] = val(doc[key], doc); + doc[key] = val(doc[key], doc, f.filter.repository); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { case '!': @@ -5992,7 +5992,6 @@ TP.$update = function() { var rec = fs.docsbuffer[dindex]; var upd = self.stringify(doc, null, rec.length); - if (upd === rec.doc) return; From 8798ef2a2fd81efed29606194d7aeeaa3ada392d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 09:33:53 +0200 Subject: [PATCH 0644/1669] Improved NoSQL. --- changes.txt | 1 + nosql.js | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/changes.txt b/changes.txt index 103cdb3d8..8fe27b629 100755 --- a/changes.txt +++ b/changes.txt @@ -9,6 +9,7 @@ - added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` +- added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/nosql.js b/nosql.js index 77b855743..80e7e03ad 100755 --- a/nosql.js +++ b/nosql.js @@ -2180,7 +2180,7 @@ function DatabaseBuilder(db) { this.$scope = 0; this.$callback = NOOP; this.$code = []; - this.$params = {}; + this.$args = {}; this.$options = {}; this.$repository = {}; this.$counter = 0; @@ -2190,7 +2190,7 @@ function DatabaseBuilder(db) { DatabaseBuilder.prototype.promise = promise; DatabaseBuilder.prototype.makefilter = function() { - return { repository: this.$repository, options: this.$options, arg: this.$params, fn: this.$functions }; + return { repository: this.$repository, options: this.$options, arg: this.$args, fn: this.$functions }; }; DatabaseBuilder.prototype.id = function(id) { @@ -2258,7 +2258,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { builder.$code = join.builder.$code.slice(0); U.extend_headers2(builder.$options, join.builder.$options); builder.$repository = join.builder.$repository; - builder.$params = CLONE(join.builder.$params); + builder.$args = CLONE(join.builder.$args); } builder.$take = join.builder.$take; @@ -2523,7 +2523,7 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { } var date = framework_utils.isDate(value); - self.$params[key] = date ? value.getTime() : value; + self.$args[key] = date ? value.getTime() : value; if (!self.$iscache) { switch (operator) { @@ -2559,8 +2559,8 @@ DatabaseBuilder.prototype.query = function(code) { return self; }; -DatabaseBuilder.prototype.param = function(key, value) { - this.$params[key] = value; +DatabaseBuilder.prototype.arg = function(key, value) { + this.$args[key] = value; return this; }; @@ -2573,7 +2573,7 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('month', name, key, operator); @@ -2594,7 +2594,7 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('day', name, key, operator); @@ -2615,7 +2615,7 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('year', name, key, operator); @@ -2637,7 +2637,7 @@ DatabaseBuilder.prototype.hour = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('hour', name, key, operator); @@ -2659,7 +2659,7 @@ DatabaseBuilder.prototype.minute = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('minute', name, key, operator); @@ -2697,7 +2697,7 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam break; } - self.$params[key] = value; + self.$args[key] = value; if (self.$scope) code = 'if(!$is){' + code + '}'; @@ -2710,7 +2710,7 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam value = value.join(' '); value = value.toLowerCase(); } - self.$params[key] = value; + self.$args[key] = value; } return self; @@ -2744,14 +2744,14 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { value = value.toLowerCase().split(' '); } - self.$params[key] = value; + self.$args[key] = value; var count = 1; if (weight) count = ((value.length / 100) * weight) >> 0; - self.$params[key2] = count || 1; + self.$args[key2] = count || 1; if (!self.$iscache) { var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a=arg.{2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Sun, 16 Sep 2018 10:12:30 +0200 Subject: [PATCH 0645/1669] Improved NoSQL modify/update. --- changes.txt | 2 ++ nosql.js | 51 +++++++++++++++++++++++---------------------------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/changes.txt b/changes.txt index 8fe27b629..f074e107b 100755 --- a/changes.txt +++ b/changes.txt @@ -10,6 +10,8 @@ - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` - added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument +- added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code +- added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/nosql.js b/nosql.js index 80e7e03ad..f72c4bc3c 100755 --- a/nosql.js +++ b/nosql.js @@ -873,6 +873,8 @@ DP.update = function(doc, insert) { var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; builder.$options.readertype = 1; + if (typeof(data) === 'string') + data = new Function('doc', 'repository', 'arg', data.indexOf('return ') === -1 ? ('return (' + data + ')') : data); self.pending_update.push({ builder: builder, doc: data, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); return builder; @@ -889,6 +891,7 @@ DP.modify = function(doc, insert) { builder.$options.readertype = 1; if (keys.length) { + var tmp; for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { @@ -898,12 +901,18 @@ DP.modify = function(doc, insert) { case '*': case '/': !inc && (inc = {}); - var tmp = key.substring(1); + tmp = key.substring(1); inc[tmp] = key[0]; doc[tmp] = doc[key]; doc[key] = undefined; keys[i] = tmp; break; + case '$': + tmp = key.substring(1); + doc[tmp] = new Function('val', 'doc', 'repository', 'arg', doc[key].indexOf('return ') === -1 ? ('return (' + doc[key] + ')') : doc[key]); + doc[key] = undefined; + keys[i] = tmp; + break; } } self.pending_update.push({ builder: builder, doc: data, keys: keys, inc: inc, insert: insert === true ? data : insert }); @@ -1498,7 +1507,7 @@ DP.$update = function() { var val = fil.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - doc[key] = val(doc[key], doc, f.filter.repository); + doc[key] = val(doc[key], doc, f.filter.repository, f.filter.arg); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { case '!': @@ -1522,7 +1531,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository, f.filter.arg) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -2546,7 +2555,6 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { DatabaseBuilder.prototype.query = function(code) { var self = this; - if (!self.$iscache) { code = '$is=(' + code + ');'; if (self.$scope) @@ -2900,7 +2908,7 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { } var raw = self.$code.join(''); - var code = 'var repository=$F.repository,options=$F.options,arg=$F.arg,fn=$F.fn,$is=false,$tmp;var R=repository;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + (noTrimmer ? 'return doc' : 'if(options.fields){var $doc={};for(var $i=0;$i Date: Sun, 16 Sep 2018 11:23:45 +0200 Subject: [PATCH 0646/1669] Updated change log. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index f074e107b..7421e3892 100755 --- a/changes.txt +++ b/changes.txt @@ -6,10 +6,10 @@ - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` - added: support for `SameSite` cookie attribute -- added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` - added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument +- added: NOSQL/TABLE modify supports `!field` as boolean toggle - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code From 350cba956bdcf2b25f2f7583acc993acab4bfc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 18:43:31 +0200 Subject: [PATCH 0647/1669] Improved NoSQL updating. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index f72c4bc3c..09eda7b43 100755 --- a/nosql.js +++ b/nosql.js @@ -1531,7 +1531,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository, f.filter.arg) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? (fil.doc(doc, f.filter.repository, f.filter.arg) || doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -5975,7 +5975,7 @@ TP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository, f.filter.arg) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? (fil.doc(doc, f.filter.repository, f.filter.arg) || doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); From 979cfd58244fdbac138cbe12ea739a7e335cffb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 21:36:44 +0200 Subject: [PATCH 0648/1669] Improved code. --- 503.html | 1 - 1 file changed, 1 deletion(-) diff --git a/503.html b/503.html index 45ffc1e30..9b82dfe69 100644 --- a/503.html +++ b/503.html @@ -23,7 +23,6 @@ 50%{ transform:scale(1.5) rotate(180deg); color:gray } 100%{ transform:scale(1); } } - From 61120ad9bd1abc9ac4fc4517466e6cfc69239d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 22:35:29 +0200 Subject: [PATCH 0649/1669] Updated version. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 150b4840a..a4a3b79f6 100755 --- a/builders.js +++ b/builders.js @@ -21,7 +21,7 @@ /** * @module FrameworkBuilders - * @version 3.0.0 + * @version 3.1.0 */ 'use strict'; From 751523d441a7ecec32f0bb2df3a709839dfdbff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 09:51:30 +0200 Subject: [PATCH 0650/1669] Improved operations. --- builders.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index a4a3b79f6..0896483ac 100755 --- a/builders.js +++ b/builders.js @@ -4723,7 +4723,11 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.next = null; callback(error, opt.response, opt); } else { - opt.response[opt.meta.current] = value; + + if (result && (result === opt.meta.current || result === opt.name)) + opt.output = value; + + opt.response[opt.name] = value; opt.meta.prev = opt.meta.current; opt.$next(); } @@ -4742,6 +4746,7 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.repeated = 0; opt.error = new ErrorBuilder(); opt.error.path = 'operation: ' + key; + opt.name = opt.meta.current; opt.meta.index = index; opt.meta.current = key; opt.$repeat = fn.$repeat; @@ -4750,7 +4755,7 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.meta.next = name[index]; fn(opt); - }, () => callback(error.items.length ? error : null, result ? opt.response[result] : opt.response, opt)); + }, () => callback(error.items.length ? error : null, result ? opt.output : opt.response, opt)); }; function OperationOptions(error, value, options, controller) { From 59dc01721cf8796685aa943f754eadb415ccfe81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 09:58:54 +0200 Subject: [PATCH 0651/1669] Improved operation again. --- builders.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 0896483ac..b8f409d20 100755 --- a/builders.js +++ b/builders.js @@ -4675,6 +4675,7 @@ global.OPERATION = function(name, value, callback, param, controller) { global.RUN = function(name, value, callback, param, controller, result) { if (typeof(value) === 'function') { + result = controller; controller = param; param = callback; callback = value; @@ -4682,10 +4683,13 @@ global.RUN = function(name, value, callback, param, controller, result) { } if (param instanceof global.Controller || (param && param.isWebSocket)) { + result = controller; controller = param; param = EMPTYOBJECT; - } else if (param instanceof OperationOptions) + } else if (param instanceof OperationOptions) { + result = controller; controller = param.controller; + } if (typeof(name) === 'string') name = name.split(',').trim(); From 630f1566b4092e42974886700cd3d1baf0c034f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 10:10:08 +0200 Subject: [PATCH 0652/1669] Improved OPERATIONS. --- builders.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index b8f409d20..02993f563 100755 --- a/builders.js +++ b/builders.js @@ -4691,6 +4691,16 @@ global.RUN = function(name, value, callback, param, controller, result) { controller = param.controller; } + if (!result) { + if (typeof(param) === 'string') { + result = param; + param = EMPTYOBJECT; + } else if (typeof(controller) === 'string') { + result = controller; + controller = null; + } + } + if (typeof(name) === 'string') name = name.split(',').trim(); @@ -4750,9 +4760,8 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.repeated = 0; opt.error = new ErrorBuilder(); opt.error.path = 'operation: ' + key; - opt.name = opt.meta.current; opt.meta.index = index; - opt.meta.current = key; + opt.name = opt.meta.current = key; opt.$repeat = fn.$repeat; opt.$current = fn; opt.$next = next; From 893c668a8d7ec043e9bef0c90ed4b18ff4d939c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 11:24:32 +0200 Subject: [PATCH 0653/1669] Improved error handling. --- nosql.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 09eda7b43..5addc443a 100755 --- a/nosql.js +++ b/nosql.js @@ -584,7 +584,7 @@ function Table(name, filename) { if (schema && t.stringifySchema() !== schema) t.extend(schema); - }).on('error', function() { + }).on('error', function(e) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); Fs.writeFileSync(t.filename, t.stringifySchema() + NEWLINE, 'utf8'); @@ -601,7 +601,7 @@ function Table(name, filename) { t.pending_locks.length && (t.pending_locks = []); t.pending_clean.length && (t.pending_clean = []); t.pending_clear.length && (t.pending_clear = []); - t.throwReadonly(); + t.throwReadonly(e); } }); } @@ -1168,8 +1168,8 @@ DP.stream = function(fn, repository, callback) { return self; }; -DP.throwReadonly = function() { - throw new Error('Database "{0}" is readonly.'.format(this.name)); +DP.throwReadonly = function(e) { + throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : ''); }; DP.scalar = function(type, field) { From 7f8a571c8280a9ca77ce38b1ab9c98e262d41094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 11:25:54 +0200 Subject: [PATCH 0654/1669] Fixed update. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 5addc443a..2355e51b7 100755 --- a/nosql.js +++ b/nosql.js @@ -1169,7 +1169,7 @@ DP.stream = function(fn, repository, callback) { }; DP.throwReadonly = function(e) { - throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : ''); + throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : '')); }; DP.scalar = function(type, field) { From ff45097acbc618596265e45aec096f8f0410dcb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 16:42:19 +0200 Subject: [PATCH 0655/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b910f58ae..f3dca4dfd 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-10", + "version": "3.0.1-11", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 77cd81c3ada358d4954aa62b4c1cec3628962fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 19 Sep 2018 15:42:39 +0200 Subject: [PATCH 0656/1669] Fixed cleaning of `TABLE()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index bd0c44b3b..579da41a8 100755 --- a/index.js +++ b/index.js @@ -7001,7 +7001,7 @@ F.service = function(count) { keys = Object.keys(F.databasescleaner); keys.wait(function(item, next) { if (item[0] === '$') - TABLE(item).clean(next); + TABLE(item.substring(1)).clean(next); else NOSQL(item).clean(next); }); From 1174354272e2305891ecbfd9389dbcfa1c33784e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 20 Sep 2018 21:13:42 +0200 Subject: [PATCH 0657/1669] Fixed NoSQL cache for compiler. --- nosql.js | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/nosql.js b/nosql.js index 2355e51b7..c420e564c 100755 --- a/nosql.js +++ b/nosql.js @@ -2875,8 +2875,7 @@ DatabaseBuilder.prototype.random = function() { DatabaseBuilder.prototype.sort = function(name, desc) { var self = this; - if (!self.$iscache) - self.$options.sort = { name: name, asc: desc ? false : true }; + self.$options.sort = { name: name, asc: desc ? false : true }; return self; }; @@ -2903,7 +2902,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { if (key && cache) { self.$mappers = cache.mitems; self.$mappersexec = cache.mexec; - self.$options.sort = cache.sort; return cache.filter; } @@ -2916,7 +2914,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { if (cache) { self.$mappers = cache.mitems; self.$mappersexec = cache.mexec; - self.$options.sort = cache.sort; return cache.filter; } } @@ -2937,7 +2934,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { cache.filter = new Function('doc', '$F', 'index', code); cache.mexec = self.$mappersexec; cache.mitems = self.$mappers; - cache.sort = self.$options.sort; CACHE[key] = cache; return cache.filter; }; @@ -3023,17 +3019,15 @@ DatabaseBuilder.prototype.done = function() { DatabaseBuilder.prototype.fields = function() { var self = this; - if (!self.$iscache) { - var opt = self.$options; - for (var i = 0, length = arguments.length; i < length; i++) { - var name = arguments[i]; - if (name[0] === '-') { - !opt.fields2 && (opt.fields2 = {}); - opt.fields2[name.substring(1)] = 1; - } else { - !opt.fields && (opt.fields = []); - opt.fields.push(name); - } + var opt = self.$options; + for (var i = 0, length = arguments.length; i < length; i++) { + var name = arguments[i]; + if (name[0] === '-') { + !opt.fields2 && (opt.fields2 = {}); + opt.fields2[name.substring(1)] = 1; + } else { + !opt.fields && (opt.fields = []); + opt.fields.push(name); } } return self; From 58591a5c0c1e22427cfc021a1c5cd83ed240846c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 20 Sep 2018 21:14:26 +0200 Subject: [PATCH 0658/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f3dca4dfd..6e2bdc377 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-11", + "version": "3.0.1-12", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 033cd55696dede1e44455112e5217f59dd749f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 22 Sep 2018 21:33:33 +0200 Subject: [PATCH 0659/1669] Changed `Date.add()` and `Date.extend()` to UTC. --- utils.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/utils.js b/utils.js index bbe2be5c8..5723111c9 100755 --- a/utils.js +++ b/utils.js @@ -2682,44 +2682,44 @@ DP.add = function(type, value) { case 'sec': case 'second': case 'seconds': - dt.setSeconds(dt.getSeconds() + value); + dt.setUTCSeconds(dt.getUTCSeconds() + value); return dt; case 'm': case 'mm': case 'minute': case 'min': case 'minutes': - dt.setMinutes(dt.getMinutes() + value); + dt.setUTCMinutes(dt.getUTCMinutes() + value); return dt; case 'h': case 'hh': case 'hour': case 'hours': - dt.setHours(dt.getHours() + value); + dt.setUTCHours(dt.getUTCHours() + value); return dt; case 'd': case 'dd': case 'day': case 'days': - dt.setDate(dt.getDate() + value); + dt.setUTCDate(dt.getUTCDate() + value); return dt; case 'w': case 'ww': case 'week': case 'weeks': - dt.setDate(dt.getDate() + (value * 7)); + dt.setUTCDate(dt.getUTCDate() + (value * 7)); return dt; case 'M': case 'MM': case 'month': case 'months': - dt.setMonth(dt.getMonth() + value); + dt.setUTCMonth(dt.getUTCMonth() + value); return dt; case 'y': case 'yyyy': case 'year': case 'years': - dt.setFullYear(dt.getFullYear() + value); + dt.setUTCFullYear(dt.getUTCFullYear() + value); return dt; } return dt; @@ -2800,16 +2800,16 @@ DP.extend = function(date) { arr = m.split(':'); tmp = +arr[0]; - tmp >= 0 && dt.setHours(tmp); + tmp >= 0 && dt.setUTCHours(tmp); if (arr[1]) { tmp = +arr[1]; - tmp >= 0 && dt.setMinutes(tmp); + tmp >= 0 && dt.setUTCMinutes(tmp); } if (arr[2]) { tmp = +arr[2]; - tmp >= 0 && dt.setSeconds(tmp); + tmp >= 0 && dt.setUTCSeconds(tmp); } continue; @@ -2819,16 +2819,16 @@ DP.extend = function(date) { arr = m.split('-'); tmp = +arr[0]; - tmp && dt.setFullYear(tmp); + tmp && dt.setUTCFullYear(tmp); if (arr[1]) { tmp = +arr[1]; - tmp >= 0 && dt.setMonth(tmp - 1); + tmp >= 0 && dt.setUTCMonth(tmp - 1); } if (arr[2]) { tmp = +arr[2]; - tmp >= 0 && dt.setDate(tmp); + tmp >= 0 && dt.setUTCDate(tmp); } continue; @@ -2839,16 +2839,16 @@ DP.extend = function(date) { if (arr[2]) { tmp = +arr[2]; - !isNaN(tmp) && dt.setFullYear(tmp); + !isNaN(tmp) && dt.setUTCFullYear(tmp); } if (arr[1]) { tmp = +arr[1]; - !isNaN(tmp) && dt.setMonth(tmp - 1); + !isNaN(tmp) && dt.setUTCMonth(tmp - 1); } tmp = +arr[0]; - !isNaN(tmp) && dt.setDate(tmp); + !isNaN(tmp) && dt.setUTCDate(tmp); continue; } From 92c1e98404cfb62dad25e7685229d3ce72ffe579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Sep 2018 09:55:09 +0200 Subject: [PATCH 0660/1669] Fixed routing with operations. --- index.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 579da41a8..01bc6bcf8 100755 --- a/index.js +++ b/index.js @@ -868,6 +868,7 @@ function Framework() { path: 0, upload: 0, schema: 0, + operation: 0, blocked: 0, 'delete': 0, mobile: 0, @@ -14600,13 +14601,18 @@ function extend_request(PROTO) { PROTO.$total_validate = function(route, next, code) { - this.$total_schema = false; + var self = this; + self.$total_schema = false; - if (!this.$total_route.schema || this.method === 'DELETE') - return next(this, code); + if (!self.$total_route.schema || self.method === 'DELETE') + return next(self, code); - var self = this; - F.onSchema(this, this.$total_route.schema[0], this.$total_route.schema[1], function(err, body) { + if (!self.$total_route.schema[1]) { + F.stats.request.operation++; + return next(self, code); + } + + F.onSchema(self, self.$total_route.schema[0], self.$total_route.schema[1], function(err, body) { if (err) { self.$total_400(err); From 59a0a2f07190921d6707d1869e86cd729096c6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Sep 2018 15:04:14 +0200 Subject: [PATCH 0661/1669] Fixed parsing CSV. --- bin/totaljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/totaljs b/bin/totaljs index f17802003..82360f69b 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -316,7 +316,7 @@ function parse_csv(content) { return; var key = arr[0].trim().replace(/(^\")|(\"$)/g, ''); - var value = arr[1].trim().replace(/(^\")|(\"$)/g, ''); + var value = arr[2].trim().replace(/(^\")|(\"$)/g, ''); if (!key || !value) return; From c5eb0f2104b76f394c457ad0cdbec457dd23b0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Sep 2018 15:04:38 +0200 Subject: [PATCH 0662/1669] Improved code. --- bin/totaljs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index 82360f69b..ac82242e7 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -315,8 +315,8 @@ function parse_csv(content) { if (arr.length <= 1) return; - var key = arr[0].trim().replace(/(^\")|(\"$)/g, ''); - var value = arr[2].trim().replace(/(^\")|(\"$)/g, ''); + var key = arr[0].trim().replace(/(^")|("$)/g, ''); + var value = arr[2].trim().replace(/(^")|("$)/g, ''); if (!key || !value) return; @@ -810,7 +810,7 @@ function git(dir, type) { } F.path.mkdir(dir); - exec('git clone https://github.com/totaljs/{0}.git {1}'.format(type, dir), function(err) { + exec('git clone https://github.com/totaljs/{0}.git {1}'.format(type, dir), function() { F.path.mkdir(path.join(dir, '/node_modules/')); F.rmdir(path.join(dir, '.git'), function() { F.unlink(path.join(dir, '.gitignore'), function() { From 3148b630c2ccc3a44dc25f397fea72275f5da41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Sep 2018 14:50:00 +0200 Subject: [PATCH 0663/1669] Improved binary. --- bin/totaljs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/totaljs b/bin/totaljs index ac82242e7..04d7be51d 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -307,10 +307,16 @@ function parse_csv(content) { var output = {}; var max = 0; - content.split('\n').forEach(function(line) { + content.split('\n').forEach(function(line, index) { + + if (!index) + return; + line = line.trim(); + if (!line) return; + var arr = line.split(';'); if (arr.length <= 1) return; From 9d837902a3cda8a2ed70015e143799804d8f6fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Sep 2018 14:50:14 +0200 Subject: [PATCH 0664/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6e2bdc377..68f13a2f5 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-12", + "version": "3.0.1-13", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From eb1295c3a0f7bdc81c57121462ee0cc374993a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 26 Sep 2018 19:43:01 +0200 Subject: [PATCH 0665/1669] Updated `stop` to `true` (by default) in `NEWOPERATION()`. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 02993f563..f012f0828 100755 --- a/builders.js +++ b/builders.js @@ -4610,7 +4610,7 @@ global.NEWOPERATION = function(name, fn, repeat, stop) { operations[name].$owner = F.$owner(); operations[name].$newversion = REGEXP_NEWOPERATION.test(fn.toString()); operations[name].$repeat = repeat; - operations[name].$stop = stop; + operations[name].$stop = stop !== false; return this; }; From 42744e6480c690600255b66f258e035df878f49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 26 Sep 2018 19:52:54 +0200 Subject: [PATCH 0666/1669] Added `default-restbuilder-timeout`. --- changes.txt | 1 + index.js | 2 ++ utils.js | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 7421e3892..f737af71a 100755 --- a/changes.txt +++ b/changes.txt @@ -12,6 +12,7 @@ - added: NOSQL/TABLE modify supports `!field` as boolean toggle - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code +- added: a new config item `default-restbuilder-timeout : 10000` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/index.js b/index.js index 01bc6bcf8..e65b9a415 100755 --- a/index.js +++ b/index.js @@ -709,6 +709,7 @@ function Framework() { // in milliseconds 'default-request-timeout': 3000, 'default-dependency-timeout': 1500, + 'default-restbuilder-timeout': 10000, // otherwise is used ImageMagick (Heroku supports ImageMagick) // gm = graphicsmagick or im = imagemagick @@ -8625,6 +8626,7 @@ F.$configure_configs = function(arr, rewrite) { case 'default-interval-websocket-ping': case 'default-interval-clear-dnscache': case 'default-dependency-timeout': + case 'default-restbuilder-timeout': case 'nosql-cleaner': obj[name] = U.parseInt(value); break; diff --git a/utils.js b/utils.js index 5723111c9..67204fb17 100755 --- a/utils.js +++ b/utils.js @@ -485,7 +485,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (callback === NOOP) callback = null; - var options = { length: 0, timeout: timeout || 10000, evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; + var options = { length: 0, timeout: timeout || CONF['default-restbuilder-timeout'], evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; var method; var type = 0; var isCookies = false; From b8d954f0125f6fb7a854f940b2b47451908d0c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 27 Sep 2018 23:28:03 +0200 Subject: [PATCH 0667/1669] Added URI validation in `REQUEST`. --- utils.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils.js b/utils.js index 67204fb17..4ad5fe530 100755 --- a/utils.js +++ b/utils.js @@ -625,6 +625,12 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, } var uri = Url.parse(url); + + if (!uri.hostname || !uri.host) { + callback && callback(new Error('URL doesn\'t contain a hostname'), '', 0); + return; + } + uri.method = method; uri.headers = headers; options.uri = uri; From aa4700a97be07b0b272f146a72b1b71b8f139f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Sep 2018 19:37:45 +0200 Subject: [PATCH 0668/1669] Added `logger`. --- builders.js | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++-- changes.txt | 1 + index.js | 77 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 165 insertions(+), 6 deletions(-) diff --git a/builders.js b/builders.js index f012f0828..837d7542c 100755 --- a/builders.js +++ b/builders.js @@ -1161,6 +1161,10 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, controller = model.$$controller; var builder = new ErrorBuilder(); + var $now; + + if (CONF.logger) + $now = Date.now(); self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -1168,10 +1172,12 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, if (!isGenerator(self, $type, self[TYPE])) { if (self[TYPE].$newversion) self[TYPE](new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, model, $type, undefined, builder, res, callback, controller); }, controller)); else self[TYPE](builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, model, $type, undefined, builder, res, callback, controller); }, controller, skip !== true); return self; @@ -1189,6 +1195,9 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, }; var onCallback = function(res) { + + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); + if (callback.success) return; @@ -1236,6 +1245,7 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio var self = this; var builder = new ErrorBuilder(); + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -1243,16 +1253,21 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio if (controller instanceof SchemaOptions) controller = controller.controller; + if (CONF.logger) + $now = Date.now(); + var output = self.default(); var $type = 'get'; if (!isGenerator(self, $type, self.onGet)) { if (self.onGet.$newversion) self.onGet(new SchemaOptions(builder, output, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, 'get'), controller, $now); self.$process(arguments, output, $type, undefined, builder, res, callback, controller); }, controller)); else self.onGet(builder, output, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, 'get'), controller, $now); self.$process(arguments, output, $type, undefined, builder, res, callback, controller); }, controller); return self; @@ -1270,6 +1285,9 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio }; var onCallback = function(res) { + + CONF.logger && F.ilogger(self.getLoggerName($type, 'get'), controller, $now); + if (callback.success) return; @@ -1309,6 +1327,7 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { var self = this; var builder = new ErrorBuilder(); var $type = 'remove'; + var $now; if (!self.onRemove) return callback(new Error('Operation "{0}/{1}" not found'.format(self.name, $type))); @@ -1316,16 +1335,21 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { if (controller instanceof SchemaOptions) controller = controller.controller; + if (CONF.logger) + $now = Date.now(); + self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); if (!isGenerator(self, $type, self.onRemove)) { if (self.onRemove.$newversion) self.onRemove(new SchemaOptions(builder, undefined, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller)); else self.onRemove(builder, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller); return self; @@ -1344,6 +1368,8 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); + if (callback.success) return; @@ -1386,17 +1412,23 @@ SchemaBuilderEntity.prototype.query = function(options, callback, controller) { var self = this; var builder = new ErrorBuilder(); var $type = 'query'; + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); + if (CONF.logger) + $now = Date.now(); + if (!isGenerator(self, $type, self.onQuery)) { if (self.onQuery.$newversion) self.onQuery(new SchemaOptions(builder, undefined, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller)); else self.onQuery(builder, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller); return self; @@ -1415,6 +1447,8 @@ SchemaBuilderEntity.prototype.query = function(options, callback, controller) { var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); + if (callback.success) return; @@ -2159,6 +2193,10 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk self.resourcePrefix && builder.setPrefix(self.resourcePrefix); var output = []; + var $now; + + if (CONF.logger) + $now = Date.now(); async_wait(hook, function(item, next) { if (item.fn.$newversion) @@ -2172,6 +2210,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk next(); }, controller, skip !== true); }, function() { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process_hook(model, $type, name, builder, output, callback); }, 0); @@ -2187,19 +2226,25 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk var builder = new ErrorBuilder(); var output = []; + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); + if (CONF.logger) + $now = Date.now(); + async_wait(hook, function(item, next, index) { if (!isGenerator(self, 'hook.' + name + '.' + index, item.fn)) { if (item.fn.$newversion) { item.fn.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res === undefined ? model : res); next(); }, controller)); } else { item.fn.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res === undefined ? model : res); next(); }, controller, skip !== true); @@ -2216,6 +2261,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk builder.push(err); next(); }, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res == undefined ? model : res); next(); }, controller)); @@ -2226,6 +2272,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk builder.push(err); next(); }, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res == undefined ? model : res); next(); }, controller, skip !== true); @@ -2274,12 +2321,16 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca var ref = self[type + 's']; var item = ref ? ref[name] : undefined; + var $now; if (!item) { callback(new ErrorBuilder().push('', type.capitalize() + ' "{0}" not found.'.format(name))); return self; } + if (CONF.logger) + $now = Date.now(); + if (controller instanceof SchemaOptions) controller = controller.controller; @@ -2292,10 +2343,12 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca self.resourcePrefix && builder.setPrefix(self.resourcePrefix); if (item.$newversion) item.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller)); else item.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller, skip !== true); return self; @@ -2319,10 +2372,12 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca if (!isGenerator(self, type + '.' + name, item)) { if (item.$newversion) item.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller)); else item.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller); return; @@ -2341,6 +2396,8 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); + if (callback.success) return; @@ -2365,6 +2422,10 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca return self; }; +SchemaBuilderEntity.prototype.getLoggerName = function(type, name) { + return this.name + '.' + type + (name ? ('(\'' + name + '\')') : '()'); +}; + /** * Run a workflow * @param {String} name @@ -2417,6 +2478,7 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac var builder = new ErrorBuilder(); var $type = 'operation'; + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2427,13 +2489,18 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac if (model && !controller && model.$$controller) controller = model.$$controller; + if (CONF.logger) + $now = Date.now(); + if (!isGenerator(self, 'operation.' + name, operation)) { if (operation.$newversion) { operation.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, model, $type, name, builder, res, callback, controller); }, controller)); } else operation.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, model, $type, name, builder, res, callback, controller); }, controller, skip !== true); return self; @@ -2452,6 +2519,8 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); + if (callback.success) return; @@ -4615,6 +4684,10 @@ global.NEWOPERATION = function(name, fn, repeat, stop) { return this; }; +function getLoggerNameOperation(name) { + return 'OPERATION(\'' + name + '\')'; +} + global.OPERATION = function(name, value, callback, param, controller) { if (typeof(value) === 'function') { @@ -4624,15 +4697,24 @@ global.OPERATION = function(name, value, callback, param, controller) { value = EMPTYOBJECT; } + if (param instanceof Controller) { + controller = param; + param = undefined; + } + var fn = operations[name]; var error = new ErrorBuilder(); + var $now; + + if (CONF.logger) + $now = Date.now(); if (fn) { if (fn.$newversion) { var self = new OperationOptions(error, value, param, controller); self.$repeat = fn.$repeat; self.callback = function(value) { - + CONF.logger && F.ilogger(getLoggerNameOperation(name), controller, $now); if (arguments.length > 1) { if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { self.error.push(value); @@ -4653,11 +4735,10 @@ global.OPERATION = function(name, value, callback, param, controller) { return self; }; - fn(self); - } else fn(error, value, function(value) { + CONF.logger && F.ilogger(getLoggerNameOperation(name), controller, $now); if (callback) { if (value instanceof Error) { error.push(value); @@ -4714,6 +4795,8 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.callback = function(value) { + CONF.logger && F.ilogger(getLoggerNameOperation(opt.name), controller, opt.duration); + if (arguments.length > 1) { if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { opt.error.push(value); @@ -4766,6 +4849,10 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.$current = fn; opt.$next = next; opt.meta.next = name[index]; + + if (CONF.logger) + opt.duration = Date.now(); + fn(opt); }, () => callback(error.items.length ? error : null, result ? opt.output : opt.response, opt)); diff --git a/changes.txt b/changes.txt index f737af71a..99f20ccf8 100755 --- a/changes.txt +++ b/changes.txt @@ -13,6 +13,7 @@ - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - added: a new config item `default-restbuilder-timeout : 10000` +- added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/index.js b/index.js index e65b9a415..4dfb12433 100755 --- a/index.js +++ b/index.js @@ -739,6 +739,7 @@ function Framework() { 'nosql-inmemory': null, // String Array 'nosql-cleaner': 1440, 'nosql-logger': true, + logger: false, // Used in F.service() // All values are in minutes @@ -12004,6 +12005,8 @@ Controller.prototype.callback = function(view) { var self = this; return function(err, data) { + CONF.logger && self.req.$logger && F.ilogger(null, self.req); + if (self.res && self.res.success) return; @@ -16425,6 +16428,10 @@ function existsSync(filename, file) { } } +function getLoggerMiddleware(name) { + return 'MIDDLEWARE("' + name + '")'; +} + function async_middleware(index, req, res, middleware, callback, options, controller) { if (res.success || res.headersSent || res.finished) { @@ -16444,10 +16451,13 @@ function async_middleware(index, req, res, middleware, callback, options, contro } var output; + var $now; + + if (CONF.logger) + $now = Date.now(); if (item.$newversion) { var opt = req.$total_middleware; - if (!index || !opt) { opt = req.$total_middleware = new MiddlewareOptions(); opt.req = req; @@ -16457,6 +16467,7 @@ function async_middleware(index, req, res, middleware, callback, options, contro opt.controller = controller; opt.callback2 = callback; opt.next = function(err) { + CONF.logger && F.ilogger(getLoggerMiddleware(name), req, $now); var mid = req.$total_middleware; if (err === false) { req.$total_route && req.$total_success(); @@ -16476,6 +16487,7 @@ function async_middleware(index, req, res, middleware, callback, options, contro } else { output = item.call(framework, req, res, function(err) { + CONF.logger && F.ilogger(getLoggerMiddleware(name), req, $now); if (err === false) { req.$total_route && req.$total_success(); callback = null; @@ -16630,10 +16642,11 @@ function controller_json_workflow(id) { self.id = id; var w = self.route.workflow; + CONF.logger && (self.req.$logger = []); + if (w instanceof Object) { if (!w.type) { - // IS IT AN OPERATION? if (!self.route.schema.length) { OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); @@ -16680,6 +16693,9 @@ function controller_json_workflow_multiple(id) { var self = this; self.id = id; var w = self.route.workflow; + + CONF.logger && (self.req.$logger = []); + if (w instanceof Object) { if (!w.type) { @@ -16743,6 +16759,61 @@ function parseSchema(name) { return schema; } +function ilogger(body) { + F.path.verify('logs'); + U.queue('F.ilogger', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], 'logger.log'), body, next)); +} + +F.ilogger = function(name, req, ts) { + + if (req && req instanceof Controller) + req = req.req; + + var isc = CONF.logger === 'console'; + var divider = ''; + + for (var i = 0; i < (isc ? 60 : 220); i++) + divider += '-'; + + var msg; + + if (req && !name && req.$logger && req.$logger.length) { + + msg = req.method + ' ' + req.url; + + req.$logger.unshift(msg); + req.$logger.push(divider + '\n'); + + if (isc) + console.log(req.$logger.join('\n')); + else + ilogger(req.$logger.join('\n')); + + req.$logger = null; + return; + } + + if (!name) + return; + + var dt = new Date(); + + msg = dt.format('yyyy-MM-dd HH:mm:ss') + ' | ' + name.padRight(40, ' ') + ' | ' + (((dt.getTime() - ts) / 1000).format(3) + ' sec.').padRight(12) + ' | ' + (req ? (req.method + ' ' + req.url).max(70) : '').padRight(70); + + if (isc) { + if (req && req.$logger) + req.$logger.push(msg); + else + console.log(msg + '\n' + divider); + } else { + msg = msg + ' | ' + (req ? (req.ip || '') : '').padRight(20) + ' | ' + (req && req.headers ? (req.headers['user-agent'] || '') : ''); + if (req && req.$logger) + req.$logger.push(msg); + else + ilogger(msg + '\n' + divider + '\n'); + } +}; + // Because of controller prototypes // It's used in F.view() and F.viewCompile() const EMPTYCONTROLLER = new Controller('', null, null, ''); @@ -16753,4 +16824,4 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; +global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file From 70f8cc81403fdc410ef7defcad867af4f47e35f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Sep 2018 19:43:04 +0200 Subject: [PATCH 0669/1669] Improved logging. --- index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 4dfb12433..eb9ab2b9f 100755 --- a/index.js +++ b/index.js @@ -16772,7 +16772,7 @@ F.ilogger = function(name, req, ts) { var isc = CONF.logger === 'console'; var divider = ''; - for (var i = 0; i < (isc ? 60 : 220); i++) + for (var i = 0; i < (isc ? 64 : 220); i++) divider += '-'; var msg; @@ -16782,12 +16782,14 @@ F.ilogger = function(name, req, ts) { msg = req.method + ' ' + req.url; req.$logger.unshift(msg); - req.$logger.push(divider + '\n'); + req.$logger.push(divider); if (isc) console.log(req.$logger.join('\n')); - else + else { + req.$logger.push(''); ilogger(req.$logger.join('\n')); + } req.$logger = null; return; From 72b798bccf0e506f289ba750badc3ca2416ae6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 29 Sep 2018 22:22:47 +0200 Subject: [PATCH 0670/1669] Fixed operation routing. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index eb9ab2b9f..7f6a8ac6b 100755 --- a/index.js +++ b/index.js @@ -16649,7 +16649,7 @@ function controller_json_workflow(id) { // IS IT AN OPERATION? if (!self.route.schema.length) { - OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); + OPERATION(w.id, self.body, w.view ? self.callback(w.view) : self.callback(), self); return; } From 2832d3aec15574bb1574fc4ec9621c4d2e6f1fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 11:18:19 +0200 Subject: [PATCH 0671/1669] Fixed rendering groups of components. --- changes.txt | 1 + index.js | 19 +++++++++++++++---- internal.js | 50 ++++++++------------------------------------------ 3 files changed, 24 insertions(+), 46 deletions(-) diff --git a/changes.txt b/changes.txt index 99f20ccf8..40f1039c9 100755 --- a/changes.txt +++ b/changes.txt @@ -30,6 +30,7 @@ - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) - fixed: subdomain routing for `localhost` - fixed: service for database cleaner +- fixed: rendering group of components - improved: NoSQL reader diff --git a/index.js b/index.js index 7f6a8ac6b..2f7ea35d7 100755 --- a/index.js +++ b/index.js @@ -11312,10 +11312,19 @@ Controller.prototype.head = function() { var self = this; if (!arguments.length) { - // OBSOLETE: this is useless - // F.emit('controller-render-head', self); var author = self.repository[REPOSITORY_META_AUTHOR] || self.config.author; - return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || ''); + var plus = ''; + if (self.$hasComponents) { + if (self.$hasComponents instanceof Array) { + for (var i = 0; i < self.$hasComponents.length; i++) { + var group = F.components.groups[self.$hasComponents[i]]; + if (group) + plus += group.links; + } + } else + plus = F.components.links; + } + return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || '') + plus; } var header = (self.repository[REPOSITORY_HEAD] || ''); @@ -11532,7 +11541,9 @@ Controller.prototype.$import = function() { } if (filename === 'components' && F.components.has) { - builder += F.components.links; + // Generated in controller.head() + // self.$hasComponents = true; + // builder += F.components.links; continue; } diff --git a/internal.js b/internal.js index 462cdbb0c..af11d69ca 100755 --- a/internal.js +++ b/internal.js @@ -55,8 +55,6 @@ const REG_BLOCK_BEG = /@\{block.*?\}/i; const REG_BLOCK_END = /@\{end\}/i; const REG_SKIP_1 = /\('|"/; const REG_SKIP_2 = /,(\s)?\w+/; -const REG_HEAD = /<\/head>/i; -const REG_COMPONENTS = /@{(\s)?(component|components)(\s)?\(/i; const REG_COMPONENTS_GROUP = /('|")[a-z0-9]+('|")/i; const HTTPVERBS = { 'get': true, 'post': true, 'options': true, 'put': true, 'delete': true, 'patch': true, 'upload': true, 'head': true, 'trace': true, 'propfind': true }; const RENDERNOW = ['self.$import(', 'self.route', 'self.$js(', 'self.$css(', 'self.$favicon(', 'self.$script(', '$STRING(self.resource(', '$STRING(RESOURCE(', 'self.translate(', 'language', 'self.sitemap_url(', 'self.sitemap_name(', '$STRING(CONFIG(', '$STRING(config.', '$STRING(config[', '$STRING(CONF.', '$STRING(CONF[', '$STRING(config(']; @@ -1724,39 +1722,6 @@ function view_parse(content, minify, filename, controller) { var index = 0; var isCookie = false; - if (!controller.$hasComponents) - controller.$hasComponents = REG_COMPONENTS.test(content) && REG_HEAD.test(content); - - if (controller.$hasComponents) { - - index = content.indexOf('@{import('); - - var add = true; - while (index !== -1) { - var str = content.substring(index, content.indexOf(')', index)); - if (str.indexOf('components') !== -1) { - add = false; - break; - } else - index = content.indexOf('@{import(', index + str.length); - } - - if (add && controller.$hasComponents) { - if (controller.$hasComponents instanceof Array) { - content = content.replace(REG_HEAD, function(text) { - var str = ''; - for (var i = 0; i < controller.$hasComponents.length; i++) { - var group = F.components.groups[controller.$hasComponents[i]]; - if (group) - str += group.links; - } - return str + text; - }); - } else - content = content.replace(REG_HEAD, text => F.components.links + text); - } - } - function escaper(value) { var is = REG_TAGREMOVE.test(value); @@ -1938,24 +1903,24 @@ function view_parse(content, minify, filename, controller) { if (!a) { var isMeta = tmp.indexOf('\'meta\'') !== -1; var isHead = tmp.indexOf('\'head\'') !== -1; - var isComponent = tmp.indexOf('\'components\'') !== -1; - tmp = tmp.replace(/'(meta|head|components)',/g, '').replace(/(,,|,\)|\s{1,})/g, ''); - if (isMeta || isHead || isComponent) { + tmp = tmp.replace(/'(meta|head)',/g, '').replace(/(,,|,\)|\s{1,})/g, ''); + if (isMeta || isHead) { var tmpimp = ''; if (isMeta) tmpimp += (isMeta ? '\'meta\'' : ''); if (isHead) tmpimp += (tmpimp ? ',' : '') + (isHead ? '\'head\'' : ''); - if (isComponent) - tmpimp += (tmpimp ? ',' : '') + (isComponent ? '\'components\'' : ''); builder += '+self.$import(' + tmpimp + ')'; } } + if (tmp.indexOf('components') !== -1) + controller.$hasComponents = true; can = true; break; } } - } + } else if (!controller.$hasComponents && tmp.indexOf('components') !== -1) + controller.$hasComponents = true; if (can && !counter) { try { @@ -2000,7 +1965,8 @@ function view_parse(content, minify, filename, controller) { if (RELEASE) builder = builder.replace(/(\+\$EMPTY\+)/g, '+').replace(/(\$output=\$EMPTY\+)/g, '$output=').replace(/(\$output\+=\$EMPTY\+)/g, '$output+=').replace(/(\}\$output\+=\$EMPTY)/g, '}').replace(/(\{\$output\+=\$EMPTY;)/g, '{').replace(/(\+\$EMPTY\+)/g, '+').replace(/(>'\+'<)/g, '><').replace(/'\+'/g, ''); - var fn = '(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'; + var fn = ('(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){self.$hasComponents=' + (controller.$hasComponents instanceof Array ? JSON.stringify(controller.$hasComponents).replace(/"/g, '\'') : controller.$hasComponents === true ? 'true' : 'null') + ';var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'); + try { fn = eval(fn); } catch (e) { From 3a98f227f4def53c0f6455b4165e596ae0a90460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 13:56:00 +0200 Subject: [PATCH 0672/1669] Added a new config option `default-request-maxkeys : 33` --- changes.txt | 1 + index.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/changes.txt b/changes.txt index 40f1039c9..e54f67f86 100755 --- a/changes.txt +++ b/changes.txt @@ -13,6 +13,7 @@ - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - added: a new config item `default-restbuilder-timeout : 10000` +- added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` diff --git a/index.js b/index.js index 2f7ea35d7..bb88e4db2 100755 --- a/index.js +++ b/index.js @@ -691,6 +691,7 @@ function Framework() { 'default-layout': 'layout', 'default-theme': '', 'default-proxy': '', + 'default-request-maxkeys': 33, // default maximum request size / length // default 10 kB @@ -8619,6 +8620,7 @@ F.$configure_configs = function(arr, rewrite) { case 'default-cors-maxage': case 'default-request-timeout': case 'default-request-maxlength': + case 'default-request-maxkeys': case 'default-websocket-maxlength': case 'default-interval-clear-cache': case 'default-interval-clear-resources': @@ -8778,6 +8780,8 @@ F.$configure_configs = function(arr, rewrite) { if (F.config['allow-performance']) http.globalAgent.maxSockets = 9999; + QUERYPARSEROPTIONS.maxKeys = F.config['default-request-maxkeys'] || 33; + var xpowered = F.config['default-xpoweredby']; Object.keys(HEADERS).forEach(function(key) { From 9bc76f35502b8783f268f7e5006f5670f88190a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 13:56:29 +0200 Subject: [PATCH 0673/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 68f13a2f5..ff6b89ea7 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-13", + "version": "3.0.1-14", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 26b4f5c493be339db5f917f86ba612479f631f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 22:59:08 +0200 Subject: [PATCH 0674/1669] Added a multiple method declaration. --- changes.txt | 1 + index.js | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index e54f67f86..975140b0b 100755 --- a/changes.txt +++ b/changes.txt @@ -19,6 +19,7 @@ - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` +- updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function diff --git a/index.js b/index.js index bb88e4db2..865590644 100755 --- a/index.js +++ b/index.js @@ -1854,12 +1854,17 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.substring(index + 1).trim(); } - url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; }); + + if (method.indexOf(',') !== -1) { + !flags && (flags = []); + method.split(',').forEach(m => flags.push(m.trim())); + method = ''; + } } if (url[0] === '#') { From 738b04d9cb1a9672e157724329de1980f58ae8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 08:37:01 +0200 Subject: [PATCH 0675/1669] Fixed sending data. --- changes.txt | 1 + utils.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 975140b0b..8b8e5a01d 100755 --- a/changes.txt +++ b/changes.txt @@ -33,6 +33,7 @@ - fixed: subdomain routing for `localhost` - fixed: service for database cleaner - fixed: rendering group of components +- fixed: RESTBuilder - JSON request without param sends an empty object - improved: NoSQL reader diff --git a/utils.js b/utils.js index 4ad5fe530..0d7ec33c2 100755 --- a/utils.js +++ b/utils.js @@ -606,6 +606,9 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data.length && url.indexOf('?') === -1 && (url += '?' + data); data = ''; } + + if (type === 1 && (data === EMPTYOBJECT || !data)) + data = BUFEMPTYJSON; } if (data && type !== 4) { @@ -6187,5 +6190,7 @@ exports.reader = function() { return new Reader(); }; +const BUFEMPTYJSON = exports.createBuffer('{}'); + global.WAIT = exports.wait; !global.F && require('./index'); \ No newline at end of file From b66beab6fd1b12254f63190646ea02baaa2f36a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 08:38:37 +0200 Subject: [PATCH 0676/1669] Fixed data serialization. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 0d7ec33c2..d392150bf 100755 --- a/utils.js +++ b/utils.js @@ -607,7 +607,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && (data === EMPTYOBJECT || !data)) + if (type === 1 && (data === EMPTYOBJECT || data === '' || data === undefined)) data = BUFEMPTYJSON; } From 182798a62bf9be95be3c204f3287ea28ae6edc90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 08:39:44 +0200 Subject: [PATCH 0677/1669] Improved condition. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index d392150bf..3b572616c 100755 --- a/utils.js +++ b/utils.js @@ -607,7 +607,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && (data === EMPTYOBJECT || data === '' || data === undefined)) + if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined)) data = BUFEMPTYJSON; } From 959344db068b08fcdfe7a0df57859ede19b8fb66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 10:52:58 +0200 Subject: [PATCH 0678/1669] Fixed `$MAKE()` with `callback`. --- changes.txt | 1 + index.js | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/changes.txt b/changes.txt index 8b8e5a01d..13fb451b4 100755 --- a/changes.txt +++ b/changes.txt @@ -34,6 +34,7 @@ - fixed: service for database cleaner - fixed: rendering group of components - fixed: RESTBuilder - JSON request without param sends an empty object +- fixed: `$MAKE()` with `callback` - improved: NoSQL reader diff --git a/index.js b/index.js index 865590644..5b4b4d421 100755 --- a/index.js +++ b/index.js @@ -358,6 +358,12 @@ global.$MAKE = function(schema, model, filter, callback, novalidate, argument) { var o = framework_builders.getschema(schema[0], schema[1]); var w = null; + if (typeof(filter) === 'function') { + var tmp = callback; + callback = filter; + filter = tmp; + } + if (filter instanceof Array) { w = {}; for (var i = 0; i < filter.length; i++) From 145d873aa5cff661d63348aaa04fcfbcf5333c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:23:35 +0200 Subject: [PATCH 0679/1669] Fixed `String.slug()`. --- changes.txt | 1 + utils.js | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 13fb451b4..ed5b82cca 100755 --- a/changes.txt +++ b/changes.txt @@ -35,6 +35,7 @@ - fixed: rendering group of components - fixed: RESTBuilder - JSON request without param sends an empty object - fixed: `$MAKE()` with `callback` +- fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars - improved: NoSQL reader diff --git a/utils.js b/utils.js index 3b572616c..22671ee81 100755 --- a/utils.js +++ b/utils.js @@ -275,7 +275,6 @@ exports.wait = function(fnValid, fnCallback, timeout, interval) { clearInterval(id_interval); clearTimeout(id_timeout); fnCallback && fnCallback(null, true); - return; } }, interval || 500); @@ -4048,6 +4047,11 @@ SP.slug = SP.toSlug = SP.toLinker = SP.linker = function(max) { var c = self[i]; var code = self.charCodeAt(i); + if (code > 540){ + builder = ''; + break; + } + if (builder.length >= max) break; @@ -4060,8 +4064,17 @@ SP.slug = SP.toSlug = SP.toLinker = SP.linker = function(max) { if ((code > 47 && code < 58) || (code > 94 && code < 123)) builder += c; } - var l = builder.length - 1; - return builder[l] === '-' ? builder.substring(0, l) : builder; + + if (builder.length > 1) { + length = builder.length - 1; + return builder[length] === '-' ? builder.substring(0, length) : builder; + } else if (!length) + return ''; + + length = self.length; + self = self.replace(/\s/g, ''); + builder = self.crc32(true).toString(36) + ''; + return self[0].charCodeAt(0).toString(32) + builder + self[self.length - 1].charCodeAt(0).toString(32) + length; }; SP.pluralize = function(zero, one, few, other) { From 191c5072ba53745748c753c976d02ce75e214804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:39:08 +0200 Subject: [PATCH 0680/1669] Fixed component rendering. --- index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/index.js b/index.js index 5b4b4d421..497d9ba6d 100755 --- a/index.js +++ b/index.js @@ -12825,10 +12825,8 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, obj.options = obj.settings = item.settings; obj.next = obj.callback = function(model) { - if (arguments.length > 1) model = arguments[1]; - item.value = self.component(item.name, item.settings, model); value = value.replace(item.replace, item.value); if (isLayout && self.precache) @@ -12892,7 +12890,7 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, done.callback && done.callback(null, value); } - }, 3); + }); return cachekey ? value : (partial ? (fn => done.callback = fn) : self); } From ffc852cbb955285de79bc49f2b460aa084449ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:39:44 +0200 Subject: [PATCH 0681/1669] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index ed5b82cca..2bd9f4eaa 100755 --- a/changes.txt +++ b/changes.txt @@ -36,6 +36,7 @@ - fixed: RESTBuilder - JSON request without param sends an empty object - fixed: `$MAKE()` with `callback` - fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars +- fixed: async rendering of `components` - improved: NoSQL reader From d20978d4eb85d0a7ef84739ecd3f51cca834eacd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:41:01 +0200 Subject: [PATCH 0682/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ff6b89ea7..4ae2bf8ea 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-14", + "version": "3.0.1-15", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From c96aa515bbe2d5b2e3b37dcba2be4363698b6d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 6 Oct 2018 00:17:14 +0200 Subject: [PATCH 0683/1669] Improved `UID()`. --- changes.txt | 1 + index.js | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 2bd9f4eaa..acb2538a3 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,7 @@ - fixed: async rendering of `components` - improved: NoSQL reader +- improved: `UID()` -> now it changes a random hash each minute ======= 3.0.0 diff --git a/index.js b/index.js index 497d9ba6d..1387bb017 100755 --- a/index.js +++ b/index.js @@ -605,9 +605,14 @@ var directory = U.$normalize(require.main ? Path.dirname(require.main.filename) // F.service() changes the values below: var DATE_EXPIRES = new Date().add('y', 1).toUTCString(); +const _randomstring = 'abcdefghijklmnoprstuwxy'.split(''); +function random3string() { + return _randomstring[(Math.random() * _randomstring.length) >> 0] + _randomstring[(Math.random() * _randomstring.length) >> 0] + _randomstring[(Math.random() * _randomstring.length) >> 0]; +} + const WEBSOCKET_COMPRESS = U.createBuffer([0x00, 0x00, 0xFF, 0xFF]); const WEBSOCKET_COMPRESS_OPTIONS = { windowBits: Zlib.Z_DEFAULT_WINDOWBITS }; -const UIDGENERATOR = { date: new Date().format('yyMMddHHmm'), instance: 'abcdefghijklmnoprstuwxy'.split('').random().join('').substring(0, 3), index: 1, types: {} }; +const UIDGENERATOR = { date: new Date().format('yyMMddHHmm'), instance: random3string(), index: 1, types: {} }; const EMPTYBUFFER = U.createBufferSize(0); global.EMPTYBUFFER = EMPTYBUFFER; @@ -6880,6 +6885,7 @@ F.service = function(count) { UIDGENERATOR.date = NOW.format('yyMMddHHmm'); UIDGENERATOR.index = 1; + UIDGENERATOR.instance = random3string(); var keys; From a4063ca809a30157702f06fe9d5f2ba6db28f09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 15:52:40 +0200 Subject: [PATCH 0684/1669] Fixed `cache` in RESTBuilder. --- builders.js | 4 ++-- changes.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index 837d7542c..298466d97 100755 --- a/builders.js +++ b/builders.js @@ -4610,13 +4610,13 @@ RESTBuilder.prototype.exec = function(callback) { return callback(err, EMPTYOBJECT, output); self.$schema.make(self.maketransform(output.value, output), function(err, model) { - !err && key && F.cache.add(key, output, self.$cache_expire); + !err && key && output.status === 200 && F.cache.add(key, output, self.$cache_expire); callback(err, err ? EMPTYOBJECT : model, output); output.cache = true; }); } else { - !err && key && F.cache.add(key, output, self.$cache_expire); + !err && key && output.status === 200 && F.cache.add(key, output, self.$cache_expire); callback(err, self.maketransform(output.value, output), output); output.cache = true; } diff --git a/changes.txt b/changes.txt index acb2538a3..567c89324 100755 --- a/changes.txt +++ b/changes.txt @@ -37,6 +37,7 @@ - fixed: `$MAKE()` with `callback` - fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars - fixed: async rendering of `components` +- fixed: RESTBuilder cache works only if the response status is `200` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute From f599ef5d3767348d5f2fd5448b6b74fb23c63e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 17:38:48 +0200 Subject: [PATCH 0685/1669] Clean code. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 1387bb017..face150d2 100755 --- a/index.js +++ b/index.js @@ -14918,7 +14918,7 @@ function extend_response(PROTO) { options = {}; options.path = options.path || '/'; - expires && builder.push('Expires=' + expires.toUTCString()); + expires && builder.push('Expires=' + expires.toUTCString()); options.domain && builder.push('Domain=' + options.domain); options.path && builder.push('Path=' + options.path); options.secure && builder.push('Secure'); From 256a620f1d0c108d870b28a599d78e8c0621f0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 18:37:50 +0200 Subject: [PATCH 0686/1669] Added a new global alias `FUNC`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 567c89324..31bc59294 100755 --- a/changes.txt +++ b/changes.txt @@ -8,6 +8,7 @@ - added: support for `SameSite` cookie attribute - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` +- added: a new global alias `FUNC` (it's a reference to functions) for `F.functions` - added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument - added: NOSQL/TABLE modify supports `!field` as boolean toggle - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code diff --git a/index.js b/index.js index face150d2..9a089ee5f 100755 --- a/index.js +++ b/index.js @@ -767,7 +767,7 @@ function Framework() { self.$bundling = true; self.resources = {}; self.connections = {}; - self.functions = {}; + global.FUNC = self.functions = {}; self.themes = {}; self.versions = null; self.workflows = {}; From 446dbb5ca0cdf3f45fbad88639c93a3e0a769fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 22:51:56 +0200 Subject: [PATCH 0687/1669] Fixed `versions` with `auto` (added cleaning cache) --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 9a089ee5f..fd833c7b1 100755 --- a/index.js +++ b/index.js @@ -8486,6 +8486,10 @@ F.$configure_versions = function(arr, clean) { var index = key.lastIndexOf('.'); filename = key.substring(0, index) + '-' + hash + key.substring(index); F.versions[key] = filename; + var keys = Object.keys(F.temporary.other); + for (var i = 0; i < keys[i]; i++) + if (keys[i].substring(0, 4) === 'view') + delete F.temporary.other[keys[i]]; } }); }); From c3ab193724b29559386dc193fcc4b3c6a945f04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 00:01:02 +0200 Subject: [PATCH 0688/1669] Fixed compressing CSS. --- changes.txt | 1 + internal.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 31bc59294..457bc7702 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,7 @@ - fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars - fixed: async rendering of `components` - fixed: RESTBuilder cache works only if the response status is `200` +- fixed: compressing CSS with `\t` tabs - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/internal.js b/internal.js index af11d69ca..e9bfb093a 100755 --- a/internal.js +++ b/internal.js @@ -63,7 +63,7 @@ const REG_NOCOMPRESS = /@\{nocompress\s\w+}/gi; const REG_TAGREMOVE = /[^>](\r)\n\s{1,}$/; const REG_HELPERS = /helpers\.[a-z0-9A-Z_$]+\(.*?\)+/g; const REG_SITEMAP = /\s+(sitemap_navigation\(|sitemap\()+/g; -const REG_CSS_0 = /\s{2,}/g; +const REG_CSS_0 = /\s{2,}|\t/g; const REG_CSS_1 = /\n/g; const REG_CSS_2 = /\s?\{\s{1,}/g; const REG_CSS_3 = /\s?\}\s{1,}/g; From 8988b6ec7145b602f05a165314e3da84123130ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 00:10:46 +0200 Subject: [PATCH 0689/1669] Fixed cache for `versions` and `auto` value. --- index.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index fd833c7b1..993932f1a 100755 --- a/index.js +++ b/index.js @@ -2557,10 +2557,12 @@ global.MERGE = F.merge = function(url) { if (url[0] !== '/') url = '/' + url; - var filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'merged_' + createTemporaryKey(url)); + var key = createTemporaryKey(url); + var filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'merged_' + key); F.routes.merge[url] = { filename: filename.replace(/\.(js|css)$/g, ext => '.min' + ext), files: arr }; Fs.unlink(F.routes.merge[url].filename, NOOP); F.owners.push({ type: 'merge', owner: _owner, id: url }); + delete F.temporary.notfound[key]; return F; }; @@ -8486,10 +8488,9 @@ F.$configure_versions = function(arr, clean) { var index = key.lastIndexOf('.'); filename = key.substring(0, index) + '-' + hash + key.substring(index); F.versions[key] = filename; - var keys = Object.keys(F.temporary.other); - for (var i = 0; i < keys[i]; i++) - if (keys[i].substring(0, 4) === 'view') - delete F.temporary.other[keys[i]]; + F.temporary.views = {}; + F.temporary.other = {}; + global.$VIEWCACHE = []; } }); }); From b8280bb45c94d0fa68f6ef7665feef6ee5f98c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 13:35:58 +0200 Subject: [PATCH 0690/1669] Fixed `controller.autoclear()`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 457bc7702..979e8d213 100755 --- a/changes.txt +++ b/changes.txt @@ -40,6 +40,7 @@ - fixed: async rendering of `components` - fixed: RESTBuilder cache works only if the response status is `200` - fixed: compressing CSS with `\t` tabs +- fixed: `controller.autoclear()` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/index.js b/index.js index 993932f1a..4f2f5c040 100755 --- a/index.js +++ b/index.js @@ -12089,7 +12089,7 @@ Controller.prototype.noClear = function(enable) { }; Controller.prototype.autoclear = function(enable) { - this.req._manual = enable === true; + this.req._manual = enable === false; return this; }; From 3596979f6e4d1644cc0b5909ecb4549aae59da07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 18:21:22 +0200 Subject: [PATCH 0691/1669] Updated test. --- test/test-css.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-css.js b/test/test-css.js index eff390911..c3e534726 100755 --- a/test/test-css.js +++ b/test/test-css.js @@ -13,7 +13,7 @@ buffer.push('div{background:linear-gradient(90deg, #000000, #FFFFFF);}'); var css = buffer.join('\n'); assert.ok(internal.compile_css(css) === 'b{border-radius:1px}a{border-radius:1px 2px 3px 4px}a{text-overflow:ellipsis}span{opacity:0;filter:alpha(opacity=0)}@keyframes test{border-radius:5px}@-webkit-keyframes test{border-radius:5px}@-moz-keyframes test{border-radius:5px}@-o-keyframes test{border-radius:5px}div{background:-webkit-linear-gradient(90deg,#000000,#FFFFFF);background:-moz-linear-gradient(90deg,#000000,#FFFFFF);background:-ms-linear-gradient(90deg,#000000,#FFFFFF);background:linear-gradient(90deg,#000000,#FFFFFF)}', 'automated CSS vendor prefixes'); -css = '.input{ }, .input:disabled, .input:hover { background-color: red; } .required{content:"This, field is required"}'; +css = '.input{ }, .input:disabled, .input:hover { background-color: red; } .required{content:"This, field is required"}'; assert.ok(internal.compile_css(css) === '.input{},.input:disabled,.input:hover{background-color:red}.required{content:"This, field is required"}', 'Problem with content.'); buffer = []; From 3eea20693bcf98b87845af75bc8f0313a6f6b49a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 18:32:38 +0200 Subject: [PATCH 0692/1669] Updated `NEWOPERATION()`. --- builders.js | 11 +++++++++-- changes.txt | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/builders.js b/builders.js index 298466d97..dbd46b4b7 100755 --- a/builders.js +++ b/builders.js @@ -4664,10 +4664,11 @@ function $decodeURIComponent(value) { } } -global.NEWOPERATION = function(name, fn, repeat, stop) { +global.NEWOPERATION = function(name, fn, repeat, stop, binderror) { // @repeat {Number} How many times will be the operation repeated after error? // @stop {Boolean} Stop when the error is thrown + // @binderror {Boolean} Binds error when chaining of operations // Remove operation if (fn == null) { @@ -4680,6 +4681,7 @@ global.NEWOPERATION = function(name, fn, repeat, stop) { operations[name].$newversion = REGEXP_NEWOPERATION.test(fn.toString()); operations[name].$repeat = repeat; operations[name].$stop = stop !== false; + operations[name].$binderror = binderror === true; return this; }; @@ -4814,7 +4816,12 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.repeated++; setImmediate(opt => opt.$current(opt), opt); } else { - opt.error.items.length && error.push(opt.error); + if (opt.error.items.length) { + error.push(opt.error); + if (opt.$current.$binderror) + value = opt.error.output(false); + } + if (opt.error.items.length && opt.$current.$stop) { name = null; opt.next = null; diff --git a/changes.txt b/changes.txt index 979e8d213..7970aea75 100755 --- a/changes.txt +++ b/changes.txt @@ -18,7 +18,7 @@ - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` -- updated: `NEWOPERATION()` supports `repeat` and `stop` argument +- updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` From 66071d4acf8a79aff5c2448e66171663ac3f35ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Oct 2018 09:06:01 +0200 Subject: [PATCH 0693/1669] Fixed JavaScript minificator. --- changes.txt | 1 + internal.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 7970aea75..d669bba59 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` +- fixed: a critical bug with JavaScript minificator - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories diff --git a/internal.js b/internal.js index e9bfb093a..7d25cd636 100755 --- a/internal.js +++ b/internal.js @@ -1202,7 +1202,7 @@ function minify_javascript(data) { } } - if (c === '+' || c === '-' && next === ' ') { + if ((c === '+' || c === '-') && next === ' ') { if (data[index + 1] === c) { index += 2; output.push(c); From 2fed4603fc5e0a3f179fa6df204df32354b21ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Oct 2018 09:29:41 +0200 Subject: [PATCH 0694/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ae2bf8ea..92287866c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-15", + "version": "3.0.1-16", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 95114107d864a86aef2e3f9b6f511a7da7f6db32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Oct 2018 13:22:31 +0200 Subject: [PATCH 0695/1669] Added `$.cancel()` for Schemas/Operations. --- builders.js | 20 ++++++++++++++++++++ changes.txt | 1 + 2 files changed, 21 insertions(+) diff --git a/builders.js b/builders.js index dbd46b4b7..57d880cfa 100755 --- a/builders.js +++ b/builders.js @@ -91,6 +91,16 @@ SchemaOptions.prototype = { } }; +SchemaOptions.prototype.cancel = function() { + var self = this; + self.callback = self.next = null; + self.error = null; + self.controller = null; + self.model = null; + self.options = null; + return self; +}; + SchemaOptions.prototype.clean = function() { return this.model.$clean(); }; @@ -4918,6 +4928,16 @@ OperationOptions.prototype = { } }; +OperationOptions.prototype.cancel = function() { + var self = this; + self.callback = null; + self.error = null; + self.controller = null; + self.options = null; + self.model = self.value = null; + return self; +}; + OperationOptions.prototype.DB = function() { return F.database(this.error); }; diff --git a/changes.txt b/changes.txt index d669bba59..5eba13522 100755 --- a/changes.txt +++ b/changes.txt @@ -16,6 +16,7 @@ - added: a new config item `default-restbuilder-timeout : 10000` - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations +- added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) From eb0af55c6e2dfbaa1a9a22e231d9459d3ea875a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 12 Oct 2018 11:11:26 +0200 Subject: [PATCH 0696/1669] Improved `CORS` + added `default-cors` to `config`. --- changes.txt | 2 + index.js | 72 +++++++++++++++++++++++++++------- test/test-framework-debug.js | 2 +- test/test-framework-release.js | 2 +- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/changes.txt b/changes.txt index 5eba13522..5c4ccdb18 100755 --- a/changes.txt +++ b/changes.txt @@ -14,6 +14,7 @@ - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - added: a new config item `default-restbuilder-timeout : 10000` +- added: a new config item `default-cors : https://www.totaljs.com, https://www.componentator.com` which allows originators for `CORS()` method - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method @@ -46,6 +47,7 @@ - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute +- improved: CORS ======= 3.0.0 diff --git a/index.js b/index.js index 4f2f5c040..088b4b220 100755 --- a/index.js +++ b/index.js @@ -715,6 +715,9 @@ function Framework() { 'default-response-maxage': '11111111', 'default-errorbuilder-status': 200, + // Default originators + 'default-cors': null, + // Seconds (2 minutes) 'default-cors-maxage': 120, @@ -1660,7 +1663,7 @@ global.CORS = F.cors = function(url, flags, credentials) { } var route = {}; - var origins = []; + var origin = []; var methods = []; var headers = []; var age; @@ -1683,8 +1686,14 @@ global.CORS = F.cors = function(url, flags, credentials) { continue; } + if (flag.substring(0, 2) === '//') { + origin.push('http:' + flag); + origin.push('https:' + flag); + continue; + } + if (flag.startsWith('http://') || flag.startsWith('https://')) { - origins.push(flag); + origin.push(flag); continue; } @@ -1713,6 +1722,9 @@ global.CORS = F.cors = function(url, flags, credentials) { if (!methods.length) methods = 'POST,PUT,GET,DELETE,PATCH,GET,HEAD'.split(','); + if (!origin.length && CONF['default-cors']) + origin = CONF['default-cors']; + route.isWILDCARD = url.lastIndexOf('*') !== -1; var index = url.indexOf('{'); @@ -1731,7 +1743,7 @@ global.CORS = F.cors = function(url, flags, credentials) { route.hash = url.hash(); route.owner = _owner; route.url = framework_internal.routeSplitCreate(url); - route.origins = origins.length ? origins : null; + route.origin = origin.length ? origin : null; route.methods = methods.length ? methods : null; route.headers = headers.length ? headers : null; route.credentials = credentials; @@ -1744,10 +1756,10 @@ global.CORS = F.cors = function(url, flags, credentials) { if (e) { // Extends existing - if (route.origins && e.origins) - corsextend(route.origins, e.origins); - else if (e.origins && !route.origins) - e.origins = null; + if (route.origin && e.origin) + corsextend(route.origin, e.origin); + else if (e.origin && !route.origin) + e.origin = null; if (route.methods && e.methods) corsextend(route.methods, e.methods); @@ -2987,7 +2999,7 @@ global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { flag = flag.toString().toLowerCase(); - // Origins + // Origin if (flag.startsWith('http://') || flag.startsWith('https://')) { !allow && (allow = []); allow.push(flag); @@ -7258,7 +7270,7 @@ F.$requestcontinue = function(req, res, headers) { req.flags = flags; F.$events['request-begin'] && F.emit('request-begin', req, res); - var isCORS = (F._length_cors || F.routes.corsall) && req.headers['origin'] != null; + var isCORS = (F._length_cors || F.routes.corsall) && req.headers.origin != null; switch (first) { case 'G': @@ -7337,7 +7349,9 @@ F.$cors = function(req, res, fn, arg) { var isAllowed = F.routes.corsall; var cors, origin; var headers = req.headers; + if (!isAllowed) { + for (var i = 0; i < F._length_cors; i++) { cors = F.routes.cors[i]; if (framework_internal.routeCompare(req.path, cors.url, false, cors.isWILDCARD)) { @@ -7383,11 +7397,11 @@ F.$cors = function(req, res, fn, arg) { } } - origin = headers['origin'].toLowerCase(); - if (!stop && cors.origins) { + if (!stop && cors.origin) { + origin = headers.origin.toLowerCase(); isAllowed = false; - for (var i = 0, length = cors.origins.length; i < length; i++) { - if (cors.origins[i].indexOf(origin) !== -1) { + for (var i = 0, length = cors.origin.length; i < length; i++) { + if (cors.origin[i].indexOf(origin) !== -1) { isAllowed = true; break; } @@ -7395,9 +7409,20 @@ F.$cors = function(req, res, fn, arg) { if (!isAllowed) stop = true; } + } else if (CONF['default-cors']) { + origin = headers.origin.toLowerCase(); + if (CONF['default-cors'].indexOf(origin) === -1) { + stop = true; + isAllowed = false; + } } - res.setHeader('Access-Control-Allow-Origin', F.routes.corsall ? headers['origin'] : (cors.origins ? cors.origins : cors.credentials ? isAllowed ? origin : cors.origins ? cors.origins : origin : headers['origin'])); + if (stop) + origin = 'null'; + else + origin = headers.origin; + + res.setHeader('Access-Control-Allow-Origin', origin); if (!cors || cors.credentials) res.setHeader('Access-Control-Allow-Credentials', 'true'); @@ -8692,6 +8717,23 @@ F.$configure_configs = function(arr, rewrite) { obj[tmp] = value; break; + case 'default-cors': + value = value.replace(/,/g, ' ').split(' '); + tmp = []; + for (var j = 0; j < value.length; j++) { + var co = (value[j] || '').trim(); + if (co) { + co = co.toLowerCase(); + if (co.substring(0, 2) === '//') { + tmp.push('http:' + co); + tmp.push('https:' + co); + } else + tmp.push(co); + } + } + obj[name] = tmp.length ? tmp : null; + break; + case 'allow-handle-static-files': OBSOLETE('config["allow-handle-static-files"]', 'The key has been renamed to "allow-static-files"'); obj['allow-static-files'] = true; @@ -13680,7 +13722,7 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { self.length = length; - var origin = self.req.headers['origin'] || ''; + var origin = self.req.headers.origin || ''; var length = allow.length; if (length && allow.indexOf('*') === -1) { diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index cbcade096..5f723711b 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -140,7 +140,7 @@ function test_routing(next) { // "access-control-allow-origin" doesn't support * (wildcard) when "access-control-allow-credentials" is set to true // node.js doesn't support duplicates headers - assert.ok(headers['access-control-allow-origin'] === 'http://www.petersirka.eu', 'CORS, headers problem 1'); + assert.ok(headers['access-control-allow-origin'] === 'null', 'CORS, headers problem 1'); assert.ok(headers['access-control-allow-credentials'] === 'true', 'CORS, headers problem 2'); assert.ok(headers['access-control-allow-methods'] === 'POST, PUT, DELETE, OPTIONS', 'CORS, headers problem 3'); assert.ok(headers['access-control-allow-headers'] === 'x-ping', 'CORS, headers problem 4'); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index e47ac4ad0..3cfccb3ac 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -140,7 +140,7 @@ function test_routing(next) { // "access-control-allow-origin" doesn't support * (wildcard) when "access-control-allow-credentials" is set to true // node.js doesn't support duplicates headers - assert.ok(headers['access-control-allow-origin'] === 'http://www.petersirka.eu', 'CORS, headers problem 1'); + assert.ok(headers['access-control-allow-origin'] === 'null', 'CORS, headers problem 1'); assert.ok(headers['access-control-allow-credentials'] === 'true', 'CORS, headers problem 2'); assert.ok(headers['access-control-allow-methods'] === 'POST, PUT, DELETE, OPTIONS', 'CORS, headers problem 3'); assert.ok(headers['access-control-allow-headers'] === 'x-ping', 'CORS, headers problem 4'); From 3c4f49053a58afe3096098c0782021e2139fbd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 13 Oct 2018 00:28:02 +0200 Subject: [PATCH 0697/1669] Fixed CORS. --- index.js | 83 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/index.js b/index.js index 088b4b220..b74dcbeef 100755 --- a/index.js +++ b/index.js @@ -1687,13 +1687,12 @@ global.CORS = F.cors = function(url, flags, credentials) { } if (flag.substring(0, 2) === '//') { - origin.push('http:' + flag); - origin.push('https:' + flag); + origin.push(flag.substring(2)); continue; } if (flag.startsWith('http://') || flag.startsWith('https://')) { - origin.push(flag); + origin.push(flag.substring(flag.indexOf('/') + 2)); continue; } @@ -7349,6 +7348,7 @@ F.$cors = function(req, res, fn, arg) { var isAllowed = F.routes.corsall; var cors, origin; var headers = req.headers; + var key; if (!isAllowed) { @@ -7365,29 +7365,18 @@ F.$cors = function(req, res, fn, arg) { var stop = false; - if (!isAllowed) - stop = true; + key = 'cors' + cors.hash + '_' + headers.origin; - isAllowed = false; + if (F.temporary.other[key]) { + stop = F.temporary.other[key] === 2; + } else { - if (!stop && cors.headers) { isAllowed = false; - for (var i = 0, length = cors.headers.length; i < length; i++) { - if (headers[cors.headers[i]]) { - isAllowed = true; - break; - } - } - if (!isAllowed) - stop = true; - } - if (!stop && cors.methods) { - isAllowed = false; - var current = headers['access-control-request-method'] || req.method; - if (current !== 'OPTIONS') { - for (var i = 0, length = cors.methods.length; i < length; i++) { - if (current === cors.methods[i]) { + if (cors.headers) { + isAllowed = false; + for (var i = 0, length = cors.headers.length; i < length; i++) { + if (headers[cors.headers[i]]) { isAllowed = true; break; } @@ -7395,25 +7384,47 @@ F.$cors = function(req, res, fn, arg) { if (!isAllowed) stop = true; } - } - if (!stop && cors.origin) { - origin = headers.origin.toLowerCase(); - isAllowed = false; - for (var i = 0, length = cors.origin.length; i < length; i++) { - if (cors.origin[i].indexOf(origin) !== -1) { - isAllowed = true; - break; + if (!stop && cors.methods) { + isAllowed = false; + var current = headers['access-control-request-method'] || req.method; + if (current !== 'OPTIONS') { + for (var i = 0, length = cors.methods.length; i < length; i++) { + if (current === cors.methods[i]) { + isAllowed = true; + break; + } + } + if (!isAllowed) + stop = true; + } + } + + if (!stop && cors.origin) { + origin = headers.origin.toLowerCase().substring(headers.origin.indexOf('/') + 2); + if (origin !== headers.host) { + isAllowed = false; + for (var i = 0, length = cors.origin.length; i < length; i++) { + if (cors.origin[i].indexOf(origin) !== -1) { + isAllowed = true; + break; + } + } + if (!isAllowed) + stop = true; } } - if (!isAllowed) - stop = true; + + F.temporary.other[key] = stop ? 2 : 1; } } else if (CONF['default-cors']) { - origin = headers.origin.toLowerCase(); - if (CONF['default-cors'].indexOf(origin) === -1) { - stop = true; - isAllowed = false; + key = headers.origin; + if (F.temporary.other[key]) { + stop = F.temporary.other[key] === 2; + } else { + origin = key.toLowerCase().substring(key.indexOf('/') + 2); + stop = origin !== headers.host && CONF['default-cors'].indexOf(origin) === -1; + F.temporary.other[key] = stop ? 2 : 1; } } From a229c9c49f39fed35a1346eb47bf494aad48df42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 13 Oct 2018 21:00:08 +0200 Subject: [PATCH 0698/1669] Fixed CORS. --- index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index b74dcbeef..313ea00ac 100755 --- a/index.js +++ b/index.js @@ -8736,10 +8736,9 @@ F.$configure_configs = function(arr, rewrite) { if (co) { co = co.toLowerCase(); if (co.substring(0, 2) === '//') { - tmp.push('http:' + co); - tmp.push('https:' + co); - } else tmp.push(co); + } else + tmp.push(co.substring(co.indexOf('/') + 2)); } } obj[name] = tmp.length ? tmp : null; From cccc0d5cd5eb233d2f90305b7b3d0e21621d1e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 12:48:43 +0200 Subject: [PATCH 0699/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 92287866c..8dc06068c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-16", + "version": "3.0.1-17", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 273da270b5c6d10089bedbe41c94910c7f3136f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 19:05:07 +0200 Subject: [PATCH 0700/1669] Fixed `REQUEST()` and `controller.proxy()`. --- changes.txt | 2 ++ utils.js | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 5c4ccdb18..750fce167 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` +- updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*`. - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator @@ -44,6 +45,7 @@ - fixed: RESTBuilder cache works only if the response status is `200` - fixed: compressing CSS with `\t` tabs - fixed: `controller.autoclear()` +- fixed: `controller.proxy()` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/utils.js b/utils.js index 22671ee81..ef49b8004 100755 --- a/utils.js +++ b/utils.js @@ -89,6 +89,7 @@ const REG_ROOT = /@\{#\}(\/)?/g; const REG_NOREMAP = /@\{noremap\}(\n)?/g; const REG_REMAP = /href=".*?"|src=".*?"/gi; const REG_URLEXT = /(https|http|wss|ws|file):\/\/|\/\/[a-z0-9]|[a-z]:/i; +const REG_TEXTAPPLICATION = /text|application/i; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -973,17 +974,23 @@ function request_response(res, uri, options) { } var self = this; - var str = self._buffer ? self._buffer.toString(options.encoding) : ''; + var data; + + if (!self.headers['content-type'] || REG_TEXTAPPLICATION.test(self.headers['content-type'])) + data = self._buffer ? self._buffer.toString(options.encoding) : ''; + else + data = self._buffer; + self._buffer = undefined; if (options.evt) { - options.evt.$events.end && options.evt.emit('end', str, self.statusCode, self.headers, uri.host, options.cookies); + options.evt.$events.end && options.evt.emit('end', data, self.statusCode, self.headers, uri.host, options.cookies); options.evt.removeAllListeners(); options.evt = null; } if (options.callback) { - options.callback(null, uri.method === 'HEAD' ? self.headers : str, self.statusCode, self.headers, uri.host, options.cookies); + options.callback(null, uri.method === 'HEAD' ? self.headers : data, self.statusCode, self.headers, uri.host, options.cookies); options.callback = null; } From 4b9938ef2d16384af246225c1649569d6a1b341d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 19:29:50 +0200 Subject: [PATCH 0701/1669] Fixed `controller.proxy()`. --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 313ea00ac..a507e78e0 100755 --- a/index.js +++ b/index.js @@ -12661,7 +12661,11 @@ Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callbac } else { self.status = code; callback && callback(err, data, code, headers); - self.content(data, (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, '')); + var ct = (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, ''); + if (data instanceof Buffer) + self.binary(data, ct); + else + self.content(data, ct); } }, null, h, ENCODING, timeout || 10000); From 5ed1e16e6fe838d01396f2c4f796e696e5d28681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 22:02:35 +0200 Subject: [PATCH 0702/1669] Component update. --- index.js | 9 +++++++-- internal.js | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index a507e78e0..387345d23 100755 --- a/index.js +++ b/index.js @@ -10280,9 +10280,13 @@ Controller.prototype.getSchema = function() { Controller.prototype.component = function(name, settings, model) { var filename = F.components.views[name]; if (filename) { - var generator = framework_internal.viewEngine(name, filename, this); + var self = this; + var tmp = F.components.instances[name]; + self.$hasComponents && (self.$hasComponents = []); + self.$hasComponents.push(tmp.group); + var generator = framework_internal.viewEngine(name, filename, self); if (generator) - return generator.call(this, this, this.repository, model || this.$model, this.session, this.query, this.body, this.url, F.global, F.helpers, this.user, this.config, F.functions, 0, this.outputPartial, this.req.files, this.req.mobile, settings || EMPTYOBJECT); + return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, self.config, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); } return ''; }; @@ -11016,6 +11020,7 @@ Controller.prototype.$view = function(name, model, expire, key) { } var value = self.view(name, model, null, true, true, cache); + if (!value) return ''; diff --git a/internal.js b/internal.js index 7d25cd636..5999721a4 100755 --- a/internal.js +++ b/internal.js @@ -1913,14 +1913,11 @@ function view_parse(content, minify, filename, controller) { builder += '+self.$import(' + tmpimp + ')'; } } - if (tmp.indexOf('components') !== -1) - controller.$hasComponents = true; can = true; break; } } - } else if (!controller.$hasComponents && tmp.indexOf('components') !== -1) - controller.$hasComponents = true; + } if (can && !counter) { try { @@ -2184,12 +2181,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { if (!controller.$hasComponents) controller.$hasComponents = []; - if (controller.$hasComponents instanceof Array) { - var group = command.match(REG_COMPONENTS_GROUP); - if (group && group.length) { - group = group[0].toString().replace(/'|"'/g, ''); - controller.$hasComponents.indexOf(group) === -1 && controller.$hasComponents.push(group); - } + var group = command.match(REG_COMPONENTS_GROUP); + if (group && group.length) { + group = group[0].toString().replace(/'|"'/g, ''); + controller.$hasComponents.indexOf(group) === -1 && controller.$hasComponents.push(group); } return 'self.$' + command + (command.indexOf('(') === -1 ? '()' : ''); @@ -2199,7 +2194,6 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'component': - controller.$hasComponents = true; tmp = command.indexOf('\''); var is = false; @@ -2216,8 +2210,14 @@ function view_prepare(command, dynamicCommand, functions, controller) { is = true; } - if (is) { + + if (tmp.group) { + !controller.$hasComponents && (controller.$hasComponents = []); + controller.$hasComponents.push(tmp.group); + } else + controller.$hasComponents = true; + var settings = command.substring(11 + name.length + 2, command.length - 1).trim(); if (settings === ')') settings = ''; From a7648642476a4e033444e37d256fb52284480baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 09:14:32 +0200 Subject: [PATCH 0703/1669] Improved components. --- index.js | 71 ++++++++++++++++++++++--------------- internal.js | 29 +++++++-------- test/controllers/default.js | 2 +- test/views/params.html | 2 +- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/index.js b/index.js index 387345d23..b1cdf0bca 100755 --- a/index.js +++ b/index.js @@ -82,6 +82,7 @@ const REPOSITORY_META_AUTHOR = '$author'; const REPOSITORY_META_IMAGE = '$image'; const REPOSITORY_PLACE = '$place'; const REPOSITORY_SITEMAP = '$sitemap'; +const REPOSITORY_COMPONENTS = '$components'; const ATTR_END = '"'; const ETAG = '858'; const CONCAT = [null, null]; @@ -4113,7 +4114,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } } - if (obj && obj.group) { + if (obj) { + + if (!obj.group) + obj.group = 'default'; + key = obj.group.crc32(true); temporary += '_g' + key; tmp = F.components.groups[obj.group]; @@ -4130,8 +4135,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp.css = true; } - tmp.version = NOW.getTime(); - tmp.links = (tmp.js ? ''.format(link, tmp.version, key) : '') + (tmp.css ? ''.format(link, tmp.version, key) : ''); + tmp.version = GUID(5); + tmp.links = (tmp.js ? ''.format(link, tmp.version, key) : '') + (tmp.css ? ''.format(link, tmp.version, key) : ''); } !skipEmit && setTimeout(function() { @@ -10281,12 +10286,16 @@ Controller.prototype.component = function(name, settings, model) { var filename = F.components.views[name]; if (filename) { var self = this; - var tmp = F.components.instances[name]; - self.$hasComponents && (self.$hasComponents = []); - self.$hasComponents.push(tmp.group); - var generator = framework_internal.viewEngine(name, filename, self); - if (generator) + var generator = framework_internal.viewEngine(name, filename, self, true); + if (generator) { + if (generator.components.length) { + if (!self.repository[REPOSITORY_COMPONENTS]) + self.repository[REPOSITORY_COMPONENTS] = {}; + for (var i = 0; i < generator.components.length; i++) + self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; + } return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, self.config, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); + } } return ''; }; @@ -11012,11 +11021,8 @@ Controller.prototype.$view = function(name, model, expire, key) { if (expire) { cache = '$view.' + name + '.' + (key || ''); var output = self.cache.read2(cache); - if (output) { - if (output.components) - self.$hasComponents = true; + if (output) return output.body; - } } var value = self.view(name, model, null, true, true, cache); @@ -11397,15 +11403,11 @@ Controller.prototype.head = function() { if (!arguments.length) { var author = self.repository[REPOSITORY_META_AUTHOR] || self.config.author; var plus = ''; - if (self.$hasComponents) { - if (self.$hasComponents instanceof Array) { - for (var i = 0; i < self.$hasComponents.length; i++) { - var group = F.components.groups[self.$hasComponents[i]]; - if (group) - plus += group.links; - } - } else - plus = F.components.links; + var components = self.repository[REPOSITORY_COMPONENTS]; + if (components) { + var keys = Object.keys(components); + for (var i = 0; i < keys.length; i++) + plus += F.components.groups[keys[i]].links; } return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || '') + plus; } @@ -11625,8 +11627,6 @@ Controller.prototype.$import = function() { if (filename === 'components' && F.components.has) { // Generated in controller.head() - // self.$hasComponents = true; - // builder += F.components.links; continue; } @@ -12847,7 +12847,16 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, var helpers = F.helpers; try { + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); + + if (generator.components.length) { + if (!self.repository[REPOSITORY_COMPONENTS]) + self.repository[REPOSITORY_COMPONENTS] = {}; + for (var i = 0; i < generator.components.length; i++) + self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; + } + } catch (ex) { err = new Error('View "' + filename + '": ' + ex.message); @@ -13048,9 +13057,6 @@ Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { self.layoutName = output.layout; self.themeName = output.theme; - if (output.components) - self.$hasComponents = true; - var res = self.res; res.options.code = self.status || 200; @@ -13117,7 +13123,7 @@ Controller.prototype.$memorize_prepare = function(key, expires, disabled, fnTo, return; } - var options = { content: value, type: contentType || CT_TEXT, layout: self.layoutName, theme: self.themeName, components: (self.$viewasync ? true : false) || (self.$hasComponents ? true : false) }; + var options = { content: value, type: contentType || CT_TEXT, layout: self.layoutName, theme: self.themeName }; if (headers) options.headers = headers; @@ -15489,7 +15495,13 @@ function extend_response(PROTO) { if (F.components.has && F.components[req.extension] && req.uri.pathname === F.config['static-url-components'] + req.extension) { res.noCompress = true; - filename = F.path.temp('components' + (req.query.group ? '_g' + req.query.group : '') + '.' + req.extension); + res.options.components = true; + var g = req.query.group ? req.query.group.substring(0, req.query.group.length - 6) : ''; + filename = F.path.temp('components' + (g ? '_g' + g : '') + '.' + req.extension); + if (g) + req.$key = 'components_' + g + '.' + req.extension; + else + req.$key = 'components.' + req.extension; } res.options.filename = filename; @@ -15663,7 +15675,8 @@ function extend_response(PROTO) { return res; } - (DEBUG || res.$nocache) && F.isProcessed(req.$key) && (F.temporary.path[req.$key] = undefined); + if (!res.options.components && (DEBUG || res.$nocache)) + F.isProcessed(req.$key) && (F.temporary.path[req.$key] = undefined); if (name[1] && !compress) headers[HEADER_LENGTH] = name[1]; diff --git a/internal.js b/internal.js index 5999721a4..47f88241c 100755 --- a/internal.js +++ b/internal.js @@ -1769,6 +1769,7 @@ function view_parse(content, minify, filename, controller) { var isCOMPILATION = false; var builderTMP = ''; var sectionName = ''; + var components = {}; var text; while (command) { @@ -1893,7 +1894,7 @@ function view_parse(content, minify, filename, controller) { builder += '}$output+=$EMPTY'; } else { - tmp = view_prepare(command.command, newCommand, functionsName, controller, filename); + tmp = view_prepare(command.command, newCommand, functionsName, controller, components); var can = false; // Inline rendering is supported only in release mode @@ -1962,10 +1963,11 @@ function view_parse(content, minify, filename, controller) { if (RELEASE) builder = builder.replace(/(\+\$EMPTY\+)/g, '+').replace(/(\$output=\$EMPTY\+)/g, '$output=').replace(/(\$output\+=\$EMPTY\+)/g, '$output+=').replace(/(\}\$output\+=\$EMPTY)/g, '}').replace(/(\{\$output\+=\$EMPTY;)/g, '{').replace(/(\+\$EMPTY\+)/g, '+').replace(/(>'\+'<)/g, '><').replace(/'\+'/g, ''); - var fn = ('(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){self.$hasComponents=' + (controller.$hasComponents instanceof Array ? JSON.stringify(controller.$hasComponents).replace(/"/g, '\'') : controller.$hasComponents === true ? 'true' : 'null') + ';var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'); + var fn = ('(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'); try { fn = eval(fn); + fn.components = Object.keys(components); } catch (e) { throw new Error(filename + ': ' + e.message.toString()); } @@ -1985,7 +1987,7 @@ function view_parse_plus(builder) { return c !== '!' && c !== '?' && c !== '+' && c !== '.' && c !== ':'; } -function view_prepare(command, dynamicCommand, functions, controller) { +function view_prepare(command, dynamicCommand, functions, controller, components) { var a = command.indexOf('.'); var b = command.indexOf('('); @@ -2050,8 +2052,6 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'G': case 'model': case 'repository': - case 'get': - case 'post': case 'query': case 'global': case 'session': @@ -2178,13 +2178,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'components': - if (!controller.$hasComponents) - controller.$hasComponents = []; - var group = command.match(REG_COMPONENTS_GROUP); if (group && group.length) { group = group[0].toString().replace(/'|"'/g, ''); - controller.$hasComponents.indexOf(group) === -1 && controller.$hasComponents.push(group); + components[group] = 1; } return 'self.$' + command + (command.indexOf('(') === -1 ? '()' : ''); @@ -2210,17 +2207,15 @@ function view_prepare(command, dynamicCommand, functions, controller) { is = true; } - if (is) { + if (tmp) + components[tmp.group] = 1; - if (tmp.group) { - !controller.$hasComponents && (controller.$hasComponents = []); - controller.$hasComponents.push(tmp.group); - } else - controller.$hasComponents = true; + if (is) { var settings = command.substring(11 + name.length + 2, command.length - 1).trim(); if (settings === ')') settings = ''; + $VIEWASYNC++; return '\'@{-{0}-}\'+(function(index){!controller.$viewasync&&(controller.$viewasync=[]);controller.$viewasync.push({replace:\'@{-{0}-}\',name:\'{1}\',settings:{2}});return $EMPTY})({0})'.format($VIEWASYNC, name, settings || 'null'); } @@ -2988,7 +2983,7 @@ function modificators(value, filename, type) { return value; } -function viewengine_load(name, filename, controller) { +function viewengine_load(name, filename, controller, component) { var precompiled = F.routes.views[name]; if (precompiled) @@ -3004,7 +2999,7 @@ function viewengine_load(name, filename, controller) { generator = viewengine_read(filename, controller); - if (!F.isDebug) + if (component || !F.isDebug) F.temporary.views[key] = generator; return generator; diff --git a/test/controllers/default.js b/test/controllers/default.js index 6e74ebade..c535d0328 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -183,7 +183,7 @@ function plain_post_parse() { var self = this; self.layout(''); var output = self.view('params', null, true); - assert.ok(output === '--body=total.js--query=query--post=total.js--get=query--', 'Problem with getting values from request body and URL.'); + assert.ok(output === '--body=total.js--query=query--', 'Problem with getting values from request body and URL.'); self.body.type = 'parse'; self.json(self.body); } diff --git a/test/views/params.html b/test/views/params.html index d9fe03ba6..52d79dbdc 100644 --- a/test/views/params.html +++ b/test/views/params.html @@ -1 +1 @@ ---body=@{body.name}--query=@{query.value}--post=@{post.name}--get=@{get.value}-- \ No newline at end of file +--body=@{body.name}--query=@{query.value}-- \ No newline at end of file From 3708e137e03090554fb098a35f799df14f594a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 09:34:21 +0200 Subject: [PATCH 0704/1669] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 750fce167..1c6590021 100755 --- a/changes.txt +++ b/changes.txt @@ -50,6 +50,7 @@ - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute - improved: CORS +- improved: rendering of components ======= 3.0.0 From a8dd5fc977002788b84b6f113b3e14b8dc6c6500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 13:01:04 +0200 Subject: [PATCH 0705/1669] Fixed rendering of group of components. --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index b1cdf0bca..e143d803f 100755 --- a/index.js +++ b/index.js @@ -12848,8 +12848,6 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, try { - value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); - if (generator.components.length) { if (!self.repository[REPOSITORY_COMPONENTS]) self.repository[REPOSITORY_COMPONENTS] = {}; @@ -12857,6 +12855,8 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; } + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); + } catch (ex) { err = new Error('View "' + filename + '": ' + ex.message); From 57e0533d2fdfdca96130bd3342fe0be50753ebcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 13:51:01 +0200 Subject: [PATCH 0706/1669] Fixed components rendering. --- index.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index e143d803f..9ac80ec65 100755 --- a/index.js +++ b/index.js @@ -11406,8 +11406,11 @@ Controller.prototype.head = function() { var components = self.repository[REPOSITORY_COMPONENTS]; if (components) { var keys = Object.keys(components); - for (var i = 0; i < keys.length; i++) - plus += F.components.groups[keys[i]].links; + for (var i = 0; i < keys.length; i++) { + var com = F.components.groups[keys[i]]; + if (com) + plus += com.links; + } } return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || '') + plus; } From c311eedbbc55799a61d2bd14b41fb4606a8debe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 18 Oct 2018 22:48:27 +0200 Subject: [PATCH 0707/1669] Added `CACHE()`. --- changes.txt | 1 + index.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 1c6590021..97e65cd37 100755 --- a/changes.txt +++ b/changes.txt @@ -18,6 +18,7 @@ - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method +- added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index 9ac80ec65..3c6747aa2 100755 --- a/index.js +++ b/index.js @@ -293,6 +293,11 @@ global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmem global.CONFIG = function(name, val) { return arguments.length === 1 ? F.config[name] : (F.config[name] = val); }; + +global.CACHE = function(name, value, expire, persistent) { + return arguments.length === 1 ? F.cache.get2(name) : F.cache.set(name, value, expire, null, persistent); +}; + global.UPTODATE = (type, url, options, interval, callback) => F.uptodate(type, url, options, interval, callback); global.INSTALL = (type, name, declaration, options, callback) => F.install(type, name, declaration, options, callback); global.UNINSTALL = (type, name, options) => F.uninstall(type, name, options); From 61ea98e393280482ff15e8527a8116656924366b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 19 Oct 2018 14:37:21 +0200 Subject: [PATCH 0708/1669] New beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8dc06068c..6ea024b0c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-17", + "version": "3.0.1-18", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 1314c44fc6591518ec9264e6527f85bbd00a68da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Oct 2018 11:21:24 +0200 Subject: [PATCH 0709/1669] Renamed config keys. --- bin/totaljs | 10 +- builders.js | 8 +- bundles.js | 36 +- debug.js | 42 +- image.js | 6 +- index.js | 983 ++++++++++++++++++++---------------- internal.js | 30 +- mail.js | 18 +- nosql.js | 26 +- package.json | 2 +- test/modules/inline-view.js | 10 +- test/modules/test.js | 6 +- test/test-tmp.js | 2 +- utils.js | 28 +- 14 files changed, 657 insertions(+), 550 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index 04d7be51d..2bcc7baa0 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -361,11 +361,11 @@ function main() { var port = cmd.parseInt(); if (port) { - F.config['directory-temp'] = '~' + path.join(os.tmpdir(), 'totaljs' + dir.hash()); - F.config['directory-public'] = '~' + dir; - F.config['allow-compile-html'] = false; - F.config['allow-compile-script'] = false; - F.config['allow-compile-style'] = false; + CONF.directory_temp = '~' + path.join(os.tmpdir(), 'totaljs' + dir.hash()); + CONF.directory_public = '~' + dir; + CONF.allow_compile_html = false; + CONF.allow_compile_script = false; + CONF.allow_compile_style = false; F.accept('.less', 'text/less'); diff --git a/builders.js b/builders.js index 57d880cfa..217f1253d 100755 --- a/builders.js +++ b/builders.js @@ -3111,16 +3111,16 @@ SchemaInstance.prototype.$constant = function(name) { function ErrorBuilder(onResource) { this.items = []; - this.transformName = transforms['error_default']; + this.transformName = transforms.error_default; this.onResource = onResource; - this.resourceName = F.config['default-errorbuilder-resource-name']; - this.resourcePrefix = F.config['default-errorbuilder-resource-prefix'] || ''; + this.resourceName = CONF.default_errorbuilder_resource_name; + this.resourcePrefix = CONF.default_errorbuilder_resource_prefix || ''; this.isResourceCustom = false; this.count = 0; this.replacer = []; this.isPrepared = false; this.contentType = 'application/json'; - this.status = F.config['default-errorbuilder-status'] || 200; + this.status = CONF.default_errorbuilder_status || 200; // Hidden: when the .push() contains a classic Error instance // this.unexpected; diff --git a/bundles.js b/bundles.js index 7eaa944cc..0263dc3ec 100755 --- a/bundles.js +++ b/bundles.js @@ -30,10 +30,10 @@ exports.make = function(callback) { META.ignore = makeignore(Fs.readFileSync(Path.join(path, '.bundlesignore')).toString('utf8').split('\n')); } catch (e) {} - blacklist[F.config['directory-temp']] = 1; - blacklist[F.config['directory-bundles']] = 1; - blacklist[F.config['directory-src']] = 1; - blacklist[F.config['directory-logs']] = 1; + blacklist[CONF.directory_temp] = 1; + blacklist[CONF.directory_bundles] = 1; + blacklist[CONF.directory_src] = 1; + blacklist[CONF.directory_logs] = 1; blacklist['/node_modules/'] = 1; blacklist['/debug.pid'] = 1; blacklist['/package-lock.json'] = 1; @@ -47,11 +47,11 @@ exports.make = function(callback) { async.push(cleanFiles); async.push(function(next) { - var target = F.path.root(F.config['directory-src']); - U.ls(F.path.root(F.config['directory-bundles']), function(files) { + var target = F.path.root(CONF.directory_src); + U.ls(F.path.root(CONF.directory_bundles), function(files) { var dirs = {}; files.wait(function(filename, resume) { - var dbpath = F.config['directory-databases']; + var dbpath = CONF.directory_databases; F.restore(filename, target, resume, function(p, dir) { @@ -118,9 +118,9 @@ exports.make = function(callback) { var file = files[i].substring(Length); var type = 0; - if (file.startsWith(F.config['directory-databases']) || file.startsWith('/flow/') || file.startsWith('/dashboard/')) + if (file.startsWith(CONF.directory_databases) || file.startsWith('/flow/') || file.startsWith('/dashboard/')) type = 1; - else if (file.startsWith(F.config['directory-public'])) + else if (file.startsWith(CONF.directory_public)) type = 2; else if (REGAPPEND.test(file)) { file = file.replace(/\/--/g, '/'); @@ -144,7 +144,7 @@ exports.make = function(callback) { }); async.push(function(next) { - Fs.writeFileSync(Path.join(F.path.root(F.config['directory-src']), 'bundle.json'), JSON.stringify(META, null, '\t')); + Fs.writeFileSync(Path.join(F.path.root(CONF.directory_src), 'bundle.json'), JSON.stringify(META, null, '\t')); next(); }); @@ -196,13 +196,13 @@ function normalize(path) { function cleanFiles(callback) { - var path = F.path.root(F.config['directory-src']); + var path = F.path.root(CONF.directory_src); var length = path.length - 1; var blacklist = {}; - blacklist[F.config['directory-public']] = 1; - blacklist[F.config['directory-private']] = 1; - blacklist[F.config['directory-databases']] = 1; + blacklist[CONF.directory_public] = 1; + blacklist[CONF.directory_private] = 1; + blacklist[CONF.directory_databases] = 1; var meta; @@ -240,7 +240,7 @@ function cleanFiles(callback) { function createDirectories(dirs, callback) { - var path = F.path.root(F.config['directory-src']); + var path = F.path.root(CONF.directory_src); try { Fs.mkdirSync(path); @@ -259,7 +259,7 @@ function createDirectories(dirs, callback) { } function copyFiles(files, callback) { - var path = F.path.root(F.config['directory-src']); + var path = F.path.root(CONF.directory_src); files.wait(function(file, next) { if (!META.ignore(file.name)) @@ -289,7 +289,7 @@ function copyFiles(files, callback) { append = true; if (CONSOLE && exists) { - F.config['allow-debug'] && F.consoledebug(append ? 'EXT:' : 'REW:', p); + CONF.allow_debug && F.consoledebug(append ? 'EXT:' : 'REW:', p); } else F.consoledebug(append ? 'EXT:' : 'COP:', p); @@ -299,7 +299,7 @@ function copyFiles(files, callback) { copyFile(file.filename, filename, next); if (CONSOLE && exists) - F.config['allow-debug'] && F.consoledebug('REW:', p); + CONF.allow_debug && F.consoledebug('REW:', p); else F.consoledebug('COP:', p); diff --git a/debug.js b/debug.js index 1924a6728..dd83e6d97 100644 --- a/debug.js +++ b/debug.js @@ -107,25 +107,25 @@ function runwatching() { const fork = require('child_process').fork; const directories = [ - U.combine(F.config['directory-components']), - U.combine(F.config['directory-controllers']), - U.combine(F.config['directory-definitions']), - U.combine(F.config['directory-operations']), - U.combine(F.config['directory-isomorphic']), - U.combine(F.config['directory-modules']), - U.combine(F.config['directory-models']), - U.combine(F.config['directory-schemas']), - U.combine(F.config['directory-resources']), - U.combine(F.config['directory-source']), - U.combine(F.config['directory-workers']), - U.combine(F.config['directory-packages']), - U.combine(F.config['directory-themes']), - U.combine(F.config['directory-configs']), - U.combine(F.config['directory-bundles']), + U.combine(CONF.directory_components), + U.combine(CONF.directory_controllers), + U.combine(CONF.directory_definitions), + U.combine(CONF.directory_operations), + U.combine(CONF.directory_isomorphic), + U.combine(CONF.directory_modules), + U.combine(CONF.directory_models), + U.combine(CONF.directory_schemas), + U.combine(CONF.directory_resources), + U.combine(CONF.directory_source), + U.combine(CONF.directory_workers), + U.combine(CONF.directory_packages), + U.combine(CONF.directory_themes), + U.combine(CONF.directory_configs), + U.combine(CONF.directory_bundles), U.combine('/startup/') ]; - const SRC = U.combine(F.config['directory-src']); + const SRC = U.combine(CONF.directory_src); const prefix = '----------------> '; options.watch && options.watch.forEach(function(item) { @@ -173,13 +173,13 @@ function runwatching() { } try { - Fs.statSync(F.path.root(F.config['directory-bundles'])); + Fs.statSync(F.path.root(CONF.directory_bundles)); isBUNDLE = true; } catch(e) {} if (isBUNDLE || isRELOAD) { - directories.push(U.combine(F.config['directory-public'])); - directories.push(U.combine(F.config['directory-views'])); + directories.push(U.combine(CONF.directory_public)); + directories.push(U.combine(CONF.directory_views)); } function onFilter(path, isDirectory) { @@ -225,12 +225,12 @@ function runwatching() { var index = fn.indexOf('/', 1); var dir = fn.substring(0, index + 1); - if (dir === F.config['directory-themes']) { + if (dir === CONF.directory_themes) { index = fn.indexOf('/', index + 1); dir = fn.substring(index, fn.indexOf('/', index + 1) + 1); } - return F.config['directory-views'] === dir || F.config['directory-public'] === dir ? fn : ''; + return CONF.directory_views === dir || CONF.directory_public === dir ? fn : ''; } function refresh() { diff --git a/image.js b/image.js index 04bcd26bc..2b8edeb30 100755 --- a/image.js +++ b/image.js @@ -141,7 +141,7 @@ function Image(filename, useImageMagick, width, height) { this.builder = []; this.filename = type === 'string' ? filename : null; this.currentStream = type === 'object' ? filename : null; - this.isIM = useImageMagick == null ? F.config['default-image-converter'] === 'im' : useImageMagick; + this.isIM = useImageMagick == null ? CONF.default_image_converter === 'im' : useImageMagick; this.outputType = type === 'string' ? framework_utils.getExtension(filename) : 'jpg'; this.islimit = false; } @@ -338,7 +338,7 @@ Image.prototype.cmd = function(filenameFrom, filenameTo) { var cmd = ''; if (!self.islimit) { - var tmp = F.config['default-image-consumption']; + var tmp = CONF.default_image_consumption; self.limit('memory', (1500 / 100) * tmp); self.limit('map', (3000 / 100) * tmp); } @@ -365,7 +365,7 @@ Image.prototype.arg = function(first, last) { first && arr.push(first); if (!self.islimit) { - var tmp = F.config['default-image-consumption']; + var tmp = CONF.default_image_consumption; self.limit('memory', (1500 / 100) * tmp); self.limit('map', (3000 / 100) * tmp); } diff --git a/index.js b/index.js index 3c6747aa2..ea8bbeed2 100755 --- a/index.js +++ b/index.js @@ -66,6 +66,7 @@ const REG_WWW = /^www\./i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; +const REG_OLDCONF = /-/g; const REG_UTF8 = /[^\x20-\x7E]+/; const FLAGS_INSTALL = ['get']; const FLAGS_DOWNLOAD = ['get', 'dnscache']; @@ -291,7 +292,7 @@ global.NOSQLSTORAGE = (name) => F.nosql(name).storage; global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); global.CONFIG = function(name, val) { - return arguments.length === 1 ? F.config[name] : (F.config[name] = val); + return arguments.length === 1 ? CONF[name] : (CONF[name] = val); }; global.CACHE = function(name, value, expire, persistent) { @@ -655,121 +656,135 @@ function Framework() { debug: true, trace: true, - 'trace-console': true, + trace_console: true, name: 'Total.js', version: '1.0.0', author: '', secret: self.syshash, - 'secret-uid': self.syshash.substring(10), + secret_uid: self.syshash.substring(10), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', - 'etag-version': '', - 'directory-src': '/.src/', - 'directory-bundles': '/bundles/', - 'directory-controllers': '/controllers/', - 'directory-components': '/components/', - 'directory-views': '/views/', - 'directory-definitions': '/definitions/', - 'directory-temp': '/tmp/', - 'directory-models': '/models/', - 'directory-schemas': '/schemas/', - 'directory-operations': '/operations/', - 'directory-resources': '/resources/', - 'directory-public': '/public/', - 'directory-public-virtual': '/app/', - 'directory-modules': '/modules/', - 'directory-source': '/source/', - 'directory-logs': '/logs/', - 'directory-tests': '/tests/', - 'directory-databases': '/databases/', - 'directory-workers': '/workers/', - 'directory-packages': '/packages/', - 'directory-private': '/private/', - 'directory-isomorphic': '/isomorphic/', - 'directory-configs': '/configs/', - 'directory-services': '/services/', - 'directory-themes': '/themes/', + etag_version: '', + directory_src: '/.src/', + directory_bundles: '/bundles/', + directory_controllers: '/controllers/', + directory_components: '/components/', + directory_views: '/views/', + directory_definitions: '/definitions/', + directory_temp: '/tmp/', + directory_models: '/models/', + directory_schemas: '/schemas/', + directory_operations: '/operations/', + directory_resources: '/resources/', + directory_public: '/public/', + directory_public_virtual: '/app/', + directory_modules: '/modules/', + directory_source: '/source/', + directory_logs: '/logs/', + directory_tests: '/tests/', + directory_databases: '/databases/', + directory_workers: '/workers/', + directory_packages: '/packages/', + directory_private: '/private/', + directory_isomorphic: '/isomorphic/', + directory_configs: '/configs/', + directory_services: '/services/', + directory_themes: '/themes/', // all HTTP static request are routed to directory-public - 'static-url': '', - 'static-url-script': '/js/', - 'static-url-style': '/css/', - 'static-url-image': '/img/', - 'static-url-video': '/video/', - 'static-url-font': '/fonts/', - 'static-url-download': '/download/', - 'static-url-components': '/components.', - 'static-accepts': { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, js: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, md: true, m4v: true, jsx: true, heif: true, heic: true }, + static_url: '', + static_url_script: '/js/', + static_url_style: '/css/', + static_url_image: '/img/', + static_url_video: '/video/', + static_url_font: '/fonts/', + static_url_download: '/download/', + static_url_components: '/components.', + static_accepts: { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, js: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, md: true, m4v: true, jsx: true, heif: true, heic: true }, // 'static-accepts-custom': [], - 'default-xpoweredby': 'Total.js', - 'default-layout': 'layout', - 'default-theme': '', - 'default-proxy': '', - 'default-request-maxkeys': 33, + default_xpoweredby: 'Total.js', + default_layout: 'layout', + default_theme: '', + default_proxy: '', + default_request_maxkeys: 33, // default maximum request size / length // default 10 kB - 'default-request-maxlength': 10, - 'default-websocket-maxlength': 2, - 'default-websocket-encodedecode': true, - 'default-maxopenfiles': 0, - 'default-timezone': '', - 'default-root': '', - 'default-response-maxage': '11111111', - 'default-errorbuilder-status': 200, + default_request_maxlength: 10, + default_websocket_maxlength: 2, + default_websocket_encodedecode: true, + default_maxopenfiles: 0, + default_timezone: '', + default_root: '', + default_response_maxage: '11111111', + default_errorbuilder_status: 200, // Default originators - 'default-cors': null, + default_cors: null, // Seconds (2 minutes) - 'default-cors-maxage': 120, + default_cors_maxage: 120, // in milliseconds - 'default-request-timeout': 3000, - 'default-dependency-timeout': 1500, - 'default-restbuilder-timeout': 10000, + default_request_timeout: 3000, + default_dependency_timeout: 1500, + default_restbuilder_timeout: 10000, // otherwise is used ImageMagick (Heroku supports ImageMagick) // gm = graphicsmagick or im = imagemagick - 'default-image-converter': 'gm', - 'default-image-quality': 93, - 'default-image-consumption': 30, - - 'allow-static-files': true, - 'allow-gzip': true, - 'allow-websocket': true, - 'allow-websocket-compression': true, - 'allow-compile': true, - 'allow-compile-script': true, - 'allow-compile-style': true, - 'allow-compile-html': true, - 'allow-compile-build': true, - 'allow-performance': false, - 'allow-custom-titles': false, - 'allow-cache-snapshot': false, - 'allow-cache-cluster': false, - 'allow-debug': false, - 'allow-head': false, - 'allow-filter-errors': true, - 'allow-clear-temp': true, - 'allow-ssc-validation': false, - 'nosql-worker': false, - 'nosql-inmemory': null, // String Array - 'nosql-cleaner': 1440, - 'nosql-logger': true, + default_image_converter: 'gm', + default_image_quality: 93, + default_image_consumption: 30, + + allow_static_files: true, + allow_gzip: true, + allow_websocket: true, + allow_websocket_compression: true, + allow_compile: true, + allow_compile_script: true, + allow_compile_style: true, + allow_compile_html: true, + allow_performance: false, + allow_custom_titles: false, + allow_cache_snapshot: false, + allow_cache_cluster: false, + allow_debug: false, + allow_head: false, + allow_filter_errors: true, + allow_clear_temp: true, + allow_ssc_validation: false, + nosql_worker: false, + nosql_inmemory: null, // String Array + nosql_cleaner: 1440, + nosql_logger: true, logger: false, // Used in F.service() // All values are in minutes - 'default-interval-clear-resources': 20, - 'default-interval-clear-cache': 10, - 'default-interval-clear-dnscache': 120, - 'default-interval-precompile-views': 61, - 'default-interval-websocket-ping': 3, - 'default-interval-uptodate': 5 + default_interval_clear_resources: 20, + default_interval_clear_cache: 10, + default_interval_clear_dnscache: 120, + default_interval_precompile_views: 61, + default_interval_websocket_ping: 3, + default_interval_uptodate: 5, + + set ['mail-address-reply'] (val) { + CONF['mail_address_reply'] = val; + return null; + }, + + set ['mail-address-from'] (val) { + CONF['mail_address_from'] = val; + return null; + }, + + set ['mail-address-copy'] (val) { + CONF['mail_address_copy'] = val; + return null; + } }; global.G = self.global = {}; @@ -1202,9 +1217,9 @@ F.useSMTP = function(smtp, options, callback) { Mail.try(smtp, options, function(err) { if (!err) { - delete F.temporary['mail-settings']; - F.config['mail-smtp'] = smtp; - F.config['mail-smtp-options'] = options; + delete F.temporary.mail_settings; + CONF.mail_smtp = smtp; + CONF.mail_smtp_options = options; } if (callback) @@ -1340,11 +1355,6 @@ global.TABLE = function(name) { return db; }; -global.GRAPHDB = function(name) { - var key = 'gdb_' + name; - return F.databases[key] ? F.databases[key] : F.databases[key] = require('./graphdb').load(name, +(F.config['graphdb.' + name] || 0)); -}; - F.stop = F.kill = function(signal) { if (F.isKilled) @@ -1362,7 +1372,7 @@ F.stop = F.kill = function(signal) { framework_nosql.kill(signal); - F.emit('exit', signal); + EMIT('exit', signal); if (!F.isWorker && process.send) TRY(() => process.send('total:stop')); @@ -1727,8 +1737,8 @@ global.CORS = F.cors = function(url, flags, credentials) { if (!methods.length) methods = 'POST,PUT,GET,DELETE,PATCH,GET,HEAD'.split(','); - if (!origin.length && CONF['default-cors']) - origin = CONF['default-cors']; + if (!origin.length && CONF.default_cors) + origin = CONF.default_cors; route.isWILDCARD = url.lastIndexOf('*') !== -1; @@ -1752,7 +1762,7 @@ global.CORS = F.cors = function(url, flags, credentials) { route.methods = methods.length ? methods : null; route.headers = headers.length ? headers : null; route.credentials = credentials; - route.age = age || F.config['default-cors-maxage']; + route.age = age || CONF.default_cors_maxage; var e = F.routes.cors.findItem(function(item) { return item.hash === route.hash; @@ -2366,7 +2376,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu method += (method ? ',' : '') + 'get'; } - if (F.config['allow-head'] && flags.indexOf('get') !== -1) { + if (CONF.allow_head && flags.indexOf('get') !== -1) { flags.append('head'); method += (method ? ',' : '') + 'head'; } @@ -2430,9 +2440,9 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.flags2 = flags_to_object(flags); r.method = method; r.execute = funcExecute; - r.length = (length || F.config['default-request-maxlength']) * 1024; + r.length = (length || CONF.default_request_maxlength) * 1024; r.middleware = middleware; - r.timeout = timeout === undefined ? (isDELAY ? 0 : F.config['default-request-timeout']) : timeout; + r.timeout = timeout === undefined ? (isDELAY ? 0 : CONF.default_request_timeout) : timeout; r.isGET = flags.indexOf('get') !== -1; r.isMULTIPLE = isMULTIPLE; r.isJSON = isJSON; @@ -2496,7 +2506,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu if (isMOBILE) F._request_check_mobile = true; - F.emit('route', 'web', instance); + EMIT('route', 'web', instance); return instance; }; @@ -2642,7 +2652,7 @@ global.MAP = F.map = function(url, filename, filter) { isPackage = true; } else if (c === '=') { if (F.isWindows) - filename = U.combine(F.config['directory-themes'], filename.substring(1)); + filename = U.combine(CONF.directory_themes, filename.substring(1)); else filename = F.path.themes(filename.substring(1)); isPackage = true; @@ -3095,7 +3105,7 @@ global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { r.onInitialize = funcInitialize; r.protocols = protocols || EMPTYARRAY; r.allow = allow || []; - r.length = (length || F.config['default-websocket-maxlength']) * 1024; + r.length = (length || CONF.default_websocket_maxlength) * 1024; r.isWEBSOCKET = true; r.MEMBER = membertype; r.isJSON = isJSON; @@ -3115,13 +3125,13 @@ global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { r.type = 'websocket'; F.routes.websockets.push(r); F.initwebsocket && F.initwebsocket(); - F.emit('route', 'websocket', r); + EMIT('route', 'websocket', r); !_controller && F.$routesSort(2); return instance; }; F.initwebsocket = function() { - if (F.routes.websockets.length && F.config['allow-websocket'] && F.server) { + if (F.routes.websockets.length && CONF.allow_websocket && F.server) { F.server.on('upgrade', F.$upgrade); F.initwebsocket = null; } @@ -3238,7 +3248,7 @@ global.FILE = F.file = function(fnValidation, fnExecute, flags) { F.routes.files.push(r); F.routes.files.sort((a, b) => !a.url ? -1 : !b.url ? 1 : a.url.length > b.url.length ? -1 : 1); - F.emit('route', 'file', r); + EMIT('route', 'file', r); F._length_files++; return F; }; @@ -3411,7 +3421,7 @@ F.errorcallback = function(err) { * @return {Framework} */ F.problem = F.wtf = function(message, name, uri, ip) { - F.$events.problem && F.emit('problem', message, name, uri, ip); + F.$events.problem && EMIT('problem', message, name, uri, ip); if (message instanceof framework_builders.ErrorBuilder) message = message.plain(); @@ -3442,7 +3452,7 @@ global.PRINTLN = function(msg) { * @return {Framework} */ F.change = function(message, name, uri, ip) { - F.$events.change && F.emit('change', message, name, uri, ip); + F.$events.change && EMIT('change', message, name, uri, ip); if (message instanceof framework_builders.ErrorBuilder) message = message.plain(); @@ -3470,10 +3480,10 @@ F.change = function(message, name, uri, ip) { */ F.trace = function(message, name, uri, ip) { - if (!F.config.trace) + if (!CONF.trace) return F; - F.$events.trace && F.emit('trace', message, name, uri, ip); + F.$events.trace && EMIT('trace', message, name, uri, ip); if (message instanceof framework_builders.ErrorBuilder) message = message.plain(); @@ -3484,7 +3494,7 @@ F.trace = function(message, name, uri, ip) { var obj = { message: message, name: name, url: uri ? typeof(uri) === 'string' ? uri : Parser.format(uri) : undefined, ip: ip, date: NOW }; F.logger('traces', obj.message, 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); - F.config['trace-console'] && console.log(NOW.format('yyyy-MM-dd HH:mm:ss'), '[trace]', message, '|', 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); + CONF.trace_console && console.log(NOW.format('yyyy-MM-dd HH:mm:ss'), '[trace]', message, '|', 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); if (F.traces) { F.traces.push(obj); @@ -3509,7 +3519,7 @@ global.MODULE = F.module = function(name) { * @return {Framework} */ F.modify = function(fn) { - OBSOLETE('F.modify()', 'This method will be removed from in versions.'); + OBSOLETE('F.modify()', 'This method will be removed in new versions.'); if (!F.modificators) F.modificators = []; F.modificators.push(fn); @@ -3521,18 +3531,18 @@ F.$bundle = function(callback) { var makebundle = function() { require('./bundles').make(function() { - F.directory = HEADERS.workers.cwd = directory = F.path.root(F.config['directory-src']); + F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); callback(); }); }; try { - Fs.statSync(F.path.root(F.config['directory-bundles'])); + Fs.statSync(F.path.root(CONF.directory_bundles)); if (F.$bundling) { makebundle(); return; } else - F.directory = HEADERS.workers.cwd = directory = F.path.root(F.config['directory-src']); + F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); } catch(e) {} callback(); }; @@ -3599,7 +3609,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('modules') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/modules/' : F.config['directory-modules']); + dir = U.combine(targetdirectory, isPackage ? '/modules/' : CONF.directory_modules); arr = []; listing(dir, 0, arr, '.js'); arr.forEach((item) => dependencies.push(next => F.install('module', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3609,7 +3619,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('isomorphic') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/isomorphic/' : F.config['directory-isomorphic']); + dir = U.combine(targetdirectory, isPackage ? '/isomorphic/' : CONF.directory_isomorphic); arr = []; listing(dir, 0, arr, '.js'); arr.forEach((item) => dependencies.push(next => F.install('isomorphic', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3619,7 +3629,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('packages') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/packages/' : F.config['directory-packages']); + dir = U.combine(targetdirectory, isPackage ? '/packages/' : CONF.directory_packages); arr = []; listing(dir, 0, arr, '.package'); var dirtmp = U.$normalize(dir); @@ -3666,7 +3676,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('models') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/models/' : F.config['directory-models']); + dir = U.combine(targetdirectory, isPackage ? '/models/' : CONF.directory_models); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('model', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3676,7 +3686,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('schemas') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/schemas/' : F.config['directory-schemas']); + dir = U.combine(targetdirectory, isPackage ? '/schemas/' : CONF.directory_schemas); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('schema', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3686,7 +3696,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('operations') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/operations/' : F.config['directory-operations']); + dir = U.combine(targetdirectory, isPackage ? '/operations/' : CONF.directory_operations); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('operation', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3697,7 +3707,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('themes') !== -1) { operations.push(function(resume) { arr = []; - dir = U.combine(targetdirectory, isPackage ? '/themes/' : F.config['directory-themes']); + dir = U.combine(targetdirectory, isPackage ? '/themes/' : CONF.directory_themes); listing(dir, 0, arr, undefined, true); arr.forEach(function(item) { var themeName = item.name; @@ -3713,7 +3723,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('definitions') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/definitions/' : F.config['directory-definitions']); + dir = U.combine(targetdirectory, isPackage ? '/definitions/' : CONF.directory_definitions); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('definition', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3724,7 +3734,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('controllers') !== -1) { operations.push(function(resume) { arr = []; - dir = U.combine(targetdirectory, isPackage ? '/controllers/' : F.config['directory-controllers']); + dir = U.combine(targetdirectory, isPackage ? '/controllers/' : CONF.directory_controllers); listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('controller', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); resume(); @@ -3734,7 +3744,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('components') !== -1) { operations.push(function(resume) { arr = []; - dir = U.combine(targetdirectory, isPackage ? '/components/' : F.config['directory-components']); + dir = U.combine(targetdirectory, isPackage ? '/components/' : CONF.directory_components); listing(dir, 0, arr, '.html'); arr.forEach((item) => dependencies.push(next => F.install('component', item.name, item.filename, undefined, undefined, undefined, undefined, undefined, undefined, next, packageName))); resume(); @@ -3971,8 +3981,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.dependencies[key].count++; setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -3983,9 +3993,9 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (type === 'config' || type === 'configuration' || type === 'settings') { F.$configure_configs(declaration instanceof Array ? declaration : declaration.toString().split('\n'), true); setTimeout(function() { - delete F.temporary['mail-settings']; - F.emit(type + '#' + name, F.config); - F.emit('install', type, name); + delete F.temporary.mail_settings; + EMIT(type + '#' + name, CONF); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -3999,8 +4009,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_versions(declaration.toString().split('\n')); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4014,8 +4024,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_workflows(declaration.toString().split('\n')); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; F.consoledebug('install', type + '#' + name); }, 500); @@ -4029,8 +4039,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_sitemap(declaration.toString().split('\n')); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; F.consoledebug('install', type + '#' + name); }, 500); @@ -4053,8 +4063,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); if (F.$bundling) { - content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); - content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); } if (content.js) @@ -4076,7 +4086,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.components.has = true; - var link = F.config['static-url-components']; + var link = CONF.static_url_components; F.components.version = NOW.getTime(); F.components.links = (F.components.js ? ''.format(link, F.components.version) : '') + (F.components.css ? ''.format(link, F.components.version) : ''); @@ -4096,7 +4106,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe _controller = ''; obj.name = name; F.components.instances[name] = obj; - obj && typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + obj && typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); } catch(e) { F.error(e, 'F.install(\'component\', \'{0}\')'.format(name)); } @@ -4110,7 +4120,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe obj.$owner = _owner; _controller = ''; F.components.instances[name] = obj; - typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); (function(name) { setTimeout(function() { delete require.cache[name]; @@ -4131,12 +4141,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp = F.components.groups[obj.group] = {}; if (content.js) { - Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); tmp.js = true; } if (content.css) { - Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); tmp.css = true; } @@ -4145,8 +4155,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } !skipEmit && setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4159,7 +4169,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (type === 'package') { var id = Path.basename(declaration, '.' + U.getExtension(declaration)); - var dir = F.config['directory-temp'][0] === '~' ? Path.join(F.config['directory-temp'].substring(1), id + '.package') : Path.join(F.path.root(), F.config['directory-temp'], id + '.package'); + var dir = CONF.directory_temp[0] === '~' ? Path.join(CONF.directory_temp.substring(1), id + '.package') : Path.join(F.path.root(), CONF.directory_temp, id + '.package'); F.routes.packages[id] = dir; @@ -4171,12 +4181,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe return; } - F.install('module', id, filename, options || F.config['package#' + name], function(err) { + F.install('module', id, filename, options || CONF['package#' + name], function(err) { setTimeout(function() { - F.emit('module#' + name); - F.emit(type + '#' + name); - F.emit('install', 'module', name); - F.emit('install', type, name); + EMIT('module#' + name); + EMIT(type + '#' + name); + EMIT('install', 'module', name); + EMIT('install', type, name); F.temporary.ready['package#' + name] = NOW; F.temporary.ready['module#' + name] = NOW; }, 500); @@ -4201,11 +4211,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe obj.$owner = _owner; F.temporary.owners[_owner] = true; - typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); !skipEmit && setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4224,14 +4234,14 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (type === 'package2') { type = type.substring(0, type.length - 1); var id = U.getName(declaration, '.package'); - var dir = F.config['directory-temp'][0] === '~' ? Path.join(F.config['directory-temp'].substring(1), id) : Path.join(F.path.root(), F.config['directory-temp'], id); + var dir = CONF.directory_temp[0] === '~' ? Path.join(CONF.directory_temp.substring(1), id) : Path.join(F.path.root(), CONF.directory_temp, id); var filename = Path.join(dir, 'index.js'); - F.install('module', id.replace(/\.package$/i, ''), filename, options || F.config['package#' + name], function(err) { + F.install('module', id.replace(/\.package$/i, ''), filename, options || CONF['package#' + name], function(err) { setTimeout(function() { - F.emit('module#' + name); - F.emit(type + '#' + name); - F.emit('install', type, name); - F.emit('install', 'module', name); + EMIT('module#' + name); + EMIT(type + '#' + name); + EMIT('install', type, name); + EMIT('install', 'module', name); F.temporary.ready['package#' + name] = NOW; F.temporary.ready['module#' + name] = NOW; }, 500); @@ -4260,8 +4270,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe Fs.writeFileSync(item.filename, framework_internal.modificators(declaration, name)); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4310,8 +4320,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe callback && callback(null, name); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4380,8 +4390,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe callback && callback(null, name); setTimeout(function() { - F.emit(type + '#' + name, obj); - F.emit('install', type, name, obj); + EMIT(type + '#' + name, obj); + EMIT('install', type, name, obj); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4479,11 +4489,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe else F.sources[name] = obj; - typeof(obj.install) === 'function' && obj.install(options || F.config[type + '#' + name], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[type + '#' + name], name); !skipEmit && setTimeout(function() { - F.emit(type + '#' + name, obj); - F.emit('install', type, name, obj); + EMIT(type + '#' + name, obj); + EMIT('install', type, name, obj); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4571,7 +4581,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } else { F.$configure_configs('@' + name + '/config'); - if (F.config.debug) + if (CONF.debug) F.$configure_configs('@' + name + '/config-debug'); else F.$configure_configs('@' + name + '/config-release'); @@ -4688,7 +4698,7 @@ F.install_prepare = function(noRecursive) { if (keys.length) throw new Error('Dependency exception, missing dependencies for: ' + keys.join(', ').trim()); delete F.temporary.other.dependencies; - }, F.config['default-dependency-timeout']); + }, CONF.default_dependency_timeout); if (!keys.length || noRecursive) return F; @@ -4707,7 +4717,7 @@ F.install_make = function(key, name, obj, options, callback, skipEmit, type) { _controller = routeID; _owner = type + '#' + name.replace(/\.package$/gi, ''); - typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); me.processed = true; var id = (type === 'module' ? '#' : '') + name; @@ -4735,8 +4745,8 @@ F.install_make = function(key, name, obj, options, callback, skipEmit, type) { if (!skipEmit) { setTimeout(function() { - F.emit(type + '#' + name, obj); - F.emit('install', type, name, obj); + EMIT(type + '#' + name, obj); + EMIT('install', type, name, obj); F.temporary.ready[type + '#' + name] = NOW; }, 500); } @@ -5005,7 +5015,7 @@ F.uninstall = function(type, name, options, skipEmit, packageName) { F.consoledebug('uninstall', type + '#' + name); } - !skipEmit && F.emit('uninstall', type, name); + !skipEmit && EMIT('uninstall', type, name); return F; }; @@ -5092,7 +5102,7 @@ F.register = function(path) { path = F.path.package(path.substring(1)); else if (c === '=') { if (path[1] === '?') - F.path.themes(F.config['default-theme'] + path.substring(2)); + F.path.themes(CONF.default_theme + path.substring(2)); else path = F.path.themes(path.substring(1)); } @@ -5181,8 +5191,8 @@ F.onMapping = function(url, def, ispublic, encode) { url = '/' + url; var tmp = url; - if (F.config['default-root']) - tmp = tmp.substring(F.config['default-root'].length - 1); + if (CONF.default_root) + tmp = tmp.substring(CONF.default_root.length - 1); // component files if (tmp[1] === '~') { @@ -5429,16 +5439,16 @@ F.onMail = function(address, subject, body, callback, replyTo) { } else message.to(address); - message.from(F.config['mail-address-from'] || '', F.config.name); + message.from(CONF.mail_address_from || '', CONF.name); if (replyTo) message.reply(replyTo); else { - tmp = F.config['mail-address-reply']; + tmp = CONF.mail_address_reply; tmp && tmp.length > 3 && message.reply(tmp); } - tmp = F.config['mail-address-copy']; + tmp = CONF.mail_address_copy; tmp && tmp.length > 3 && message.bcc(tmp); message.$sending = setImmediate(cb => message.send2(cb), callback); @@ -5459,7 +5469,7 @@ F.onMeta = function() { switch (i) { case 0: - builder += '' + (arg + (F.url !== '/' && !F.config['allow-custom-titles'] ? ' - ' + F.config.name : '')) + ''; + builder += '' + (arg + (F.url !== '/' && !CONF.allow_custom_titles ? ' - ' + CONF.name : '')) + ''; break; case 1: builder += ''; @@ -5499,7 +5509,7 @@ global.LOG = F.log = function() { } F.path.verify('logs'); - U.queue('F.log', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], filename + '.log'), time + ' | ' + str + '\n', next)); + U.queue('F.log', 5, (next) => Fs.appendFile(U.combine(CONF.directory_logs, filename + '.log'), time + ' | ' + str + '\n', next)); return F; }; @@ -5521,7 +5531,7 @@ global.LOGGER = F.logger = function() { } F.path.verify('logs'); - U.queue('F.logger', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], arguments[0] + '.log'), dt + ' | ' + str + '\n', next)); + U.queue('F.logger', 5, (next) => Fs.appendFile(U.combine(CONF.directory_logs, arguments[0] + '.log'), dt + ' | ' + str + '\n', next)); return F; }; @@ -5537,7 +5547,7 @@ global.LOGMAIL = F.logmail = function(address, subject, body, callback) { } if (!subject) - subject = F.config.name + ' v' + F.config.version; + subject = CONF.name + ' v' + CONF.version; var body = '' + subject + '
' + (typeof(body) === 'object' ? JSON.stringify(body).escape() : body) + '
'; return F.onMail(address, subject, body, callback); @@ -5574,13 +5584,13 @@ F.usage = function(detailed) { memoryTotal: (memory.heapTotal / 1024 / 1024).floor(2), memoryUsage: (memory.heapUsed / 1024 / 1024).floor(2), memoryRss: (memory.rss / 1024 / 1024).floor(2), - mode: F.config.debug ? 'debug' : 'release', + mode: CONF.debug ? 'debug' : 'release', port: F.port, ip: F.ip, directory: process.cwd() }; - if (F.config['nosql-worker']) + if (CONF.nosql_worker) output.framework.pidnosql = framework_nosql.pid(); var keys = Object.keys(U.queuecache); @@ -5730,7 +5740,7 @@ function compile_merge(res) { var merge = F.routes.merge[uri.pathname]; var filename = merge.filename; - if (!F.config.debug && existsSync(filename)) { + if (!CONF.debug && existsSync(filename)) { var stats = Fs.statSync(filename); var tmp = F.temporary.path[req.$key] = [filename, stats.size, stats.mtime.toUTCString()]; compile_gzip(tmp, function() { @@ -5867,7 +5877,7 @@ function component_debug(filename, value, extension) { F.compile_virtual = function(res) { var req = res.req; - var tmpname = res.options.filename.replace(F.config['directory-public'], F.config['directory-public-virtual']); + var tmpname = res.options.filename.replace(CONF.directory_public, CONF.directory_public_virtual); if (tmpname === res.options.filename) { F.temporary.notfound[req.$key] = true; @@ -5885,13 +5895,13 @@ F.compile_virtual = function(res) { return; } - if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) { + if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && CONF.allow_compile && !REG_NOCOMPRESS.test(res.options.filename)) { res.options.filename = tmpname; return compile_file(res); } var tmp = F.temporary.path[req.$key] = [tmpname, size, stats.mtime.toUTCString()]; - if (F.config['allow-gzip'] && COMPRESSION[U.getContentType(req.extension)]) { + if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { compile_gzip(tmp, function() { delete F.temporary.processing[req.$key]; res.$file(); @@ -5919,11 +5929,11 @@ function compile_check(res) { if (e) { - if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) + if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && CONF.allow_compile && !REG_NOCOMPRESS.test(res.options.filename)) return compile_file(res); var tmp = F.temporary.path[req.$key] = [res.options.filename, size, stats.mtime.toUTCString()]; - if (F.config['allow-gzip'] && COMPRESSION[U.getContentType(req.extension)]) { + if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { compile_gzip(tmp, function() { res.$file(); delete F.temporary.processing[req.$key]; @@ -5971,10 +5981,10 @@ function compile_content(extension, content, filename) { switch (extension) { case 'js': - return F.config['allow-compile-script'] ? framework_internal.compile_javascript(content, filename) : content; + return CONF.allow_compile_script ? framework_internal.compile_javascript(content, filename) : content; case 'css': - content = F.config['allow-compile-style'] ? framework_internal.compile_css(content, filename) : content; + content = CONF.allow_compile_style ? framework_internal.compile_css(content, filename) : content; var matches = content.match(REG_COMPILECSS); if (matches) { for (var i = 0, length = matches.length; i < length; i++) { @@ -6280,7 +6290,7 @@ F.exists = function(req, res, max, callback) { var filename = F.path.temp(name); var httpcachevalid = false; - RELEASE && (req.headers['if-none-match'] === ETAG + F.config['etag-version']) && (httpcachevalid = true); + RELEASE && (req.headers['if-none-match'] === ETAG + CONF.etag_version) && (httpcachevalid = true); if (F.isProcessed(name) || httpcachevalid) { res.options.filename = filename; @@ -6336,7 +6346,7 @@ F.isProcessing = function(filename) { if (index !== -1) name = name.substring(0, index); - filename = U.combine(F.config['directory-public'], $decodeURIComponent(name)); + filename = U.combine(CONF.directory_public, $decodeURIComponent(name)); return !!F.temporary.processing[filename]; }; @@ -6434,7 +6444,7 @@ global.LOAD = F.load = function(debug, types, pwd) { } F.isWorker = true; - F.config.debug = debug; + CONF.debug = debug; F.isDebug = debug; global.DEBUG = debug; @@ -6459,7 +6469,7 @@ global.LOAD = F.load = function(debug, types, pwd) { F.consoledebug('init'); F.cache.init(); - F.emit('init'); + EMIT('init'); F.$load(types, directory, function() { @@ -6469,8 +6479,8 @@ global.LOAD = F.load = function(debug, types, pwd) { setTimeout(function() { try { - F.emit('load', F); - F.emit('ready', F); + EMIT('load', F); + EMIT('ready', F); } catch (err) { F.error(err, 'F.on("load/ready")'); } @@ -6491,7 +6501,7 @@ global.LOAD = F.load = function(debug, types, pwd) { }, 500); - if (F.config['allow-debug']) { + if (CONF.allow_debug) { F.consoledebug('done'); F.usagesnapshot(); } @@ -6518,10 +6528,10 @@ F.initialize = function(http, debug, options) { var ip = options.ip; var listenpath = options.listenpath; - options.config && U.extend_headers2(F.config, options.config); + options.config && U.extend_headers2(CONF, options.config); - if (options.debug || options['allow-debug']) - F.config['allow-debug'] = true; + if (options.debug || options['allow-debug'] || options.allow_debug) + CONF.allow_debug = true; F.isHTTPS = http.STATUS_CODES === undefined; @@ -6531,7 +6541,7 @@ F.initialize = function(http, debug, options) { if (options.id) F.id = options.id; - F.config.debug = debug; + CONF.debug = debug; F.isDebug = debug; if (options.bundling != null) @@ -6556,21 +6566,21 @@ F.initialize = function(http, debug, options) { F.isTest && F.$configure_configs('config-test', true); F.cache.init(); F.consoledebug('init'); - F.emit('init'); + EMIT('init'); if (!port) { - if (F.config['default-port'] === 'auto') { + if (CONF.default_port === 'auto') { var envPort = +(process.env.PORT || ''); if (!isNaN(envPort)) port = envPort; } else - port = F.config['default-port']; + port = CONF.default_port; } F.port = port || 8000; if (ip !== null) { - F.ip = ip || F.config['default-ip'] || '0.0.0.0'; + F.ip = ip || CONF.default_ip || '0.0.0.0'; if (F.ip === 'null' || F.ip === 'undefined' || F.ip === 'auto') F.ip = null; } else @@ -6579,7 +6589,7 @@ F.initialize = function(http, debug, options) { if (F.ip == null) F.ip = '0.0.0.0'; - !listenpath && (listenpath = F.config['default-listenpath']); + !listenpath && (listenpath = CONF.default_listenpath); F.listenpath = listenpath; if (F.server) { @@ -6602,7 +6612,7 @@ F.initialize = function(http, debug, options) { else F.server = http.createServer(F.listener); - F.config['allow-performance'] && F.server.on('connection', connection_tunning); + CONF.allow_performance && F.server.on('connection', connection_tunning); F.initwebsocket && F.initwebsocket(); F.consoledebug('HTTP listening'); @@ -6626,7 +6636,7 @@ F.initialize = function(http, debug, options) { else listen(); - if (F.config['allow-debug']) { + if (CONF.allow_debug) { F.consoledebug('done'); F.usagesnapshot(); } @@ -6637,8 +6647,8 @@ F.initialize = function(http, debug, options) { setTimeout(function() { try { - F.emit('load', F); - F.emit('ready', F); + EMIT('load', F); + EMIT('ready', F); } catch (err) { F.error(err, 'F.on("load/ready")'); } @@ -6744,8 +6754,8 @@ F.mode = function(http, name, options) { debug = true; break; } - F.config.debug = debug; - F.config.trace = debug; + CONF.debug = debug; + CONF.trace = debug; F.isDebug = debug; global.DEBUG = debug; global.RELEASE = !debug; @@ -6784,7 +6794,7 @@ F.mode = function(http, name, options) { break; } - F.config.trace = debug; + CONF.trace = debug; F.consoledebug('startup'); F.$startup(function() { F.consoledebug('startup (done)'); @@ -6833,7 +6843,7 @@ F.custom = function(mode, http, request, response, options) { break; } - F.config.trace = debug; + CONF.trace = debug; F.consoledebug('startup'); F.$startup(function() { F.consoledebug('startup (done)'); @@ -6850,14 +6860,14 @@ F.console = function() { console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); - F.config['nosql-worker'] && console.log('NoSQL PID : ' + framework_nosql.pid()); + CONF.nosql_worker && console.log('NoSQL PID : ' + framework_nosql.pid()); console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); console.log('===================================================='); - console.log('Name : ' + F.config.name); - console.log('Version : ' + F.config.version); - console.log('Author : ' + F.config.author); + console.log('Name : ' + CONF.name); + console.log('Version : ' + CONF.version); + console.log('Author : ' + CONF.author); console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); - console.log('Mode : ' + (F.config.debug ? 'debug' : 'release')); + console.log('Mode : ' + (CONF.debug ? 'debug' : 'release')); console.log('===================================================='); console.log('Directory : ' + process.cwd()); console.log('node_modules : ' + PATHMODULES); @@ -6875,7 +6885,7 @@ F.usagesnapshot = function(filename) { F.consoledebug = function() { - if (!F.config['allow-debug']) + if (!CONF.allow_debug) return F; var arr = [new Date().format('yyyy-MM-dd HH:mm:ss'), '--------->']; @@ -6890,10 +6900,10 @@ F.consoledebug = function() { * @return {Framework} */ F.reconnect = function() { - if (F.config['default-port'] !== undefined) - F.port = F.config['default-port']; - if (F.config['default-ip'] !== undefined) - F.ip = F.config['default-ip']; + if (CONF.default_port !== undefined) + F.port = CONF.default_port; + if (CONF.default_ip !== undefined) + F.ip = CONF.default_ip; F.server.close(() => F.server.listen(F.port, F.ip)); return F; }; @@ -6937,8 +6947,8 @@ F.service = function(count) { F.temporary.notfound = {}; // every 7 minutes (default) service clears static cache - if (count % F.config['default-interval-clear-cache'] === 0) { - F.$events.clear && F.emit('clear', 'temporary', F.temporary); + if (count % CONF.default_interval_clear_cache === 0) { + F.$events.clear && EMIT('clear', 'temporary', F.temporary); F.temporary.path = {}; F.temporary.range = {}; F.temporary.views = {}; @@ -6953,24 +6963,24 @@ F.service = function(count) { F.databases[key] && F.databases[key].inmemorylastusage < dt && F.databases[key].release(); releasegc = true; - F.config['allow-debug'] && F.consoledebug('clear temporary cache'); + CONF.allow_debug && F.consoledebug('clear temporary cache'); } // every 61 minutes (default) services precompile all (installed) views - if (count % F.config['default-interval-precompile-views'] === 0) { + if (count % CONF.default_interval_precompile_views === 0) { for (var key in F.routes.views) { var item = F.routes.views[key]; F.install('view', key, item.url, null); } } - if (count % F.config['default-interval-clear-dnscache'] === 0) { - F.$events.clear && F.emit('clear', 'dns'); + if (count % CONF.default_interval_clear_dnscache === 0) { + F.$events.clear && EMIT('clear', 'dns'); U.clearDNS(); - F.config['allow-debug'] && F.consoledebug('clear DNS cache'); + CONF.allow_debug && F.consoledebug('clear DNS cache'); } - var ping = F.config['default-interval-websocket-ping']; + var ping = CONF.default_interval_websocket_ping; if (ping > 0 && count % ping === 0) { var has = false; for (var item in F.connections) { @@ -6981,10 +6991,10 @@ F.service = function(count) { has = true; } } - has && F.config['allow-debug'] && F.consoledebug('ping websocket connections'); + has && CONF.allow_debug && F.consoledebug('ping websocket connections'); } - if (F.uptodates && (count % F.config['default-interval-uptodate'] === 0) && F.uptodates.length) { + if (F.uptodates && (count % CONF.default_interval_uptodate === 0) && F.uptodates.length) { var hasUpdate = false; F.uptodates.wait(function(item, next) { @@ -6995,10 +7005,10 @@ F.service = function(count) { item.count++; setTimeout(function() { - F.config['allow-debug'] && F.consoledebug('uptodate', item.type + '#' + item.url); + CONF.allow_debug && F.consoledebug('uptodate', item.type + '#' + item.url); F.install(item.type, item.url, item.options, function(err, name, skip) { - F.config['allow-debug'] && F.consoledebug('uptodate', item.type + '#' + item.url + ' (done)'); + CONF.allow_debug && F.consoledebug('uptodate', item.type + '#' + item.url + ' (done)'); if (skip) return next(); @@ -7009,7 +7019,7 @@ F.service = function(count) { } else { hasUpdate = true; item.name = name; - F.$events.uptodate && F.emit('uptodate', item.type, name); + F.$events.uptodate && EMIT('uptodate', item.type, name); } item.callback && item.callback(err, name); @@ -7031,17 +7041,17 @@ F.service = function(count) { } // every 20 minutes (default) service clears resources - if (count % F.config['default-interval-clear-resources'] === 0) { - F.$events.clear && F.emit('clear', 'resources'); + if (count % CONF.default_interval_clear_resources === 0) { + F.$events.clear && EMIT('clear', 'resources'); F.resources = {}; releasegc = true; - F.config['allow-debug'] && F.consoledebug('clear resources'); + CONF.allow_debug && F.consoledebug('clear resources'); } // Update expires date count % 1000 === 0 && (DATE_EXPIRES = NOW.add('y', 1).toUTCString()); - if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { + if (count % CONF.nosql_cleaner === 0 && CONF.nosql_cleaner) { keys = Object.keys(F.databasescleaner); keys.wait(function(item, next) { if (item[0] === '$') @@ -7051,16 +7061,16 @@ F.service = function(count) { }); } - F.$events.service && F.emit('service', count); + F.$events.service && EMIT('service', count); - if (F.config['allow-debug']) { + if (CONF.allow_debug) { F.consoledebug('service ({0}x)'.format(count)); F.usagesnapshot(); } releasegc && global.gc && setTimeout(function() { global.gc(); - F.config['allow-debug'] && F.consoledebug('gc()'); + CONF.allow_debug && F.consoledebug('gc()'); }, 1000); // Run schedules @@ -7084,7 +7094,7 @@ F.service = function(count) { else F.schedules.splice(index, 1); - F.config['allow-debug'] && F.consoledebug('schedule', schedule.id); + CONF.allow_debug && F.consoledebug('schedule', schedule.id); schedule.fn.call(F); } @@ -7114,7 +7124,7 @@ F.listener = function(req, res) { req.uri = framework_internal.parseURI(req); F.stats.request.request++; - F.$events.request && F.emit('request', req, res); + F.$events.request && EMIT('request', req, res); if (F._request_check_redirect) { var redirect = F.routes.redirects[req.$protocol + '://' + req.host]; @@ -7133,7 +7143,7 @@ F.listener = function(req, res) { req.xhr = headers['x-requested-with'] === 'XMLHttpRequest'; res.success = false; req.user = req.session = null; - req.isStaticFile = F.config['allow-static-files'] && U.isStaticFile(req.uri.pathname); + req.isStaticFile = CONF.allow_static_files && U.isStaticFile(req.uri.pathname); if (req.isStaticFile) req.extension = U.getExtension(req.uri.pathname); @@ -7254,7 +7264,7 @@ F.$requestcontinue = function(req, res, headers) { flags.push('sse'); } - if (F.config.debug) { + if (CONF.debug) { req.$flags += 'h'; flags.push('debug'); } @@ -7277,7 +7287,7 @@ F.$requestcontinue = function(req, res, headers) { } req.flags = flags; - F.$events['request-begin'] && F.emit('request-begin', req, res); + F.$events['request-begin'] && EMIT('request-begin', req, res); var isCORS = (F._length_cors || F.routes.corsall) && req.headers.origin != null; @@ -7427,13 +7437,13 @@ F.$cors = function(req, res, fn, arg) { F.temporary.other[key] = stop ? 2 : 1; } - } else if (CONF['default-cors']) { + } else if (CONF.default_cors) { key = headers.origin; if (F.temporary.other[key]) { stop = F.temporary.other[key] === 2; } else { origin = key.toLowerCase().substring(key.indexOf('/') + 2); - stop = origin !== headers.host && CONF['default-cors'].indexOf(origin) === -1; + stop = origin !== headers.host && CONF.default_cors.indexOf(origin) === -1; F.temporary.other[key] = stop ? 2 : 1; } } @@ -7467,7 +7477,7 @@ F.$cors = function(req, res, fn, arg) { if (stop) { fn = null; - F.$events['request-end'] && F.emit('request-end', req, res); + F.$events['request-end'] && EMIT('request-end', req, res); F.reqstats(false, false); F.stats.request.blocked++; res.writeHead(404); @@ -7479,7 +7489,7 @@ F.$cors = function(req, res, fn, arg) { return fn(req, res, arg); fn = null; - F.$events['request-end'] && F.emit('request-end', req, res); + F.$events['request-end'] && EMIT('request-end', req, res); F.reqstats(false, false); res.writeHead(200); res.end(); @@ -7506,7 +7516,7 @@ F.$upgrade = function(req, socket, head) { req.uri = framework_internal.parseURI(req); - F.$events.websocket && F.emit('websocket', req, socket, head); + F.$events.websocket && EMIT('websocket', req, socket, head); F.stats.request.websocket++; req.session = null; @@ -7628,7 +7638,7 @@ global.MODEL = F.model = function(name) { var obj = F.models[name]; if (obj || obj === null) return obj; - var filename = U.combine(F.config['directory-models'], name + '.js'); + var filename = U.combine(CONF.directory_models, name + '.js'); existsSync(filename) && F.install('model', name, filename, undefined, undefined, undefined, true); return F.models[name] || null; }; @@ -7643,7 +7653,7 @@ global.INCLUDE = global.SOURCE = F.source = function(name, options, callback) { var obj = F.sources[name]; if (obj || obj === null) return obj; - var filename = U.combine(F.config['directory-source'], name + '.js'); + var filename = U.combine(CONF.directory_source, name + '.js'); existsSync(filename) && F.install('source', name, filename, options, callback, undefined, true); return F.sources[name] || null; }; @@ -7725,8 +7735,8 @@ global.VIEW = F.view = function(name, model, layout, repository, language) { if (theme) { controller.themeName = theme; name = prepare_viewname(name); - } else if (this.onTheme) - controller.themeName = this.onTheme(controller); + } else if (F.onTheme) + controller.themeName = F.onTheme(controller); else controller.themeName = undefined; @@ -7787,7 +7797,7 @@ F.clear = function(callback, isInit) { var plus = F.id ? 'i-' + F.id + '_' : ''; if (isInit) { - if (!F.config['allow-clear-temp']) { + if (!CONF.allow_clear_temp) { if (F.$bundling) { // clears only JS and CSS files U.ls(dir, function(files) { @@ -7934,7 +7944,7 @@ F.encrypt = function(value, key, isUnique) { else if (type === 'object') value = JSON.stringify(value); - return value.encrypt(F.config.secret + '=' + key, isUnique); + return value.encrypt(CONF.secret + '=' + key, isUnique); }; /** @@ -7955,7 +7965,7 @@ F.decrypt = function(value, key, jsonConvert) { if (typeof(jsonConvert) !== 'boolean') jsonConvert = true; - var response = (value || '').decrypt(F.config.secret + '=' + key); + var response = (value || '').decrypt(CONF.secret + '=' + key); return response ? (jsonConvert ? (response.isJSON() ? response.parseJSON(true) : null) : response) : null; }; @@ -7973,7 +7983,7 @@ F.hash = function(type, value, salt) { if (typeof(salt) === 'string') plus = salt; else if (salt !== false) - plus = (F.config.secret || ''); + plus = (CONF.secret || ''); hash.update(value.toString() + plus, ENCODING); return hash.digest('hex'); @@ -8010,7 +8020,7 @@ F.resource = function(name, key) { } } - var filename = U.combine(F.config['directory-resources'], name + '.resource'); + var filename = U.combine(CONF.directory_resources, name + '.resource'); var empty = false; if (existsSync(filename)) body += (body ? '\n' : '') + Fs.readFileSync(filename).toString(ENCODING); @@ -8595,7 +8605,7 @@ F.$configure_configs = function(arr, rewrite) { if (!arr) { var filenameA = U.combine('/', 'config'); - var filenameB = U.combine('/', 'config-' + (F.config.debug ? 'debug' : 'release')); + var filenameB = U.combine('/', 'config-' + (CONF.debug ? 'debug' : 'release')); arr = []; @@ -8626,8 +8636,8 @@ F.$configure_configs = function(arr, rewrite) { } var done = function() { - process.title = 'total: ' + F.config.name.removeDiacritics().toLowerCase().replace(REG_EMPTY, '-').substring(0, 8); - F.isVirtualDirectory = existsSync(U.combine(F.config['directory-public-virtual'])); + process.title = 'total: ' + CONF.name.removeDiacritics().toLowerCase().replace(REG_EMPTY, '-').substring(0, 8); + F.isVirtualDirectory = existsSync(U.combine(CONF.directory_public_virtual)); }; if (!(arr instanceof Array) || !arr.length) { @@ -8671,51 +8681,80 @@ F.$configure_configs = function(arr, rewrite) { switch (name) { case 'secret': case 'secret-uid': + case 'secret_uid': + name = name.replace(REG_OLDCONF, '_'); obj[name] = value; break; case 'default-request-length': - OBSOLETE(name, 'You need to use "default-request-maxlength"'); - obj['default-request-maxlength'] = U.parseInt(value); + OBSOLETE(name, 'You need to use "default_request_maxlength"'); + obj.default_request_maxlength = U.parseInt(value); break; case 'default-websocket-request-length': - OBSOLETE(name, 'You need to use "default-websocket-maxlength"'); - obj['default-websocket-maxlength'] = U.parseInt(value); + OBSOLETE(name, 'You need to use "default_websocket_maxlength"'); + obj.default_websocket_maxlength = U.parseInt(value); break; case 'default-maximum-file-descriptors': - OBSOLETE(name, 'You need to use "default-maxopenfiles"'); - obj['default-maxopenfiles'] = U.parseInt(value); + OBSOLETE(name, 'You need to use "default_maxopenfiles"'); + obj.default_maxopenfiles = U.parseInt(value); break; - case 'default-cors-maxage': - case 'default-request-timeout': - case 'default-request-maxlength': - case 'default-request-maxkeys': - case 'default-websocket-maxlength': - case 'default-interval-clear-cache': - case 'default-interval-clear-resources': - case 'default-interval-precompile-views': - case 'default-interval-uptodate': - case 'default-interval-websocket-ping': - case 'default-interval-clear-dnscache': - case 'default-dependency-timeout': - case 'default-restbuilder-timeout': - case 'nosql-cleaner': + case 'default-cors-maxage': // old + case 'default-request-timeout': // old + case 'default-request-maxlength': // old + case 'default-request-maxkeys': // old + case 'default-websocket-maxlength': // old + case 'default-interval-clear-cache': // old + case 'default-interval-clear-resources': // old + case 'default-interval-precompile-views': // old + case 'default-interval-uptodate': // old + case 'default-interval-websocket-ping': // old + case 'default-interval-clear-dnscache': // old + case 'default-dependency-timeout': // old + case 'default-restbuilder-timeout': // old + case 'nosql-cleaner': // old + case 'default-errorbuilder-status': // old + case 'default-maxopenfiles': // old + case 'default_maxopenfiles': + case 'default_errorbuilder_status': + case 'default_cors_maxage': + case 'default_request_timeout': + case 'default_request_maxlength': + case 'default_request_maxkeys': + case 'default_websocket_maxlength': + case 'default_interval_clear_cache': + case 'default_interval_clear_resources': + case 'default_interval_precompile_views': + case 'default_interval_uptodate': + case 'default_interval_websocket_ping': + case 'default_interval_clear_dnscache': + case 'default_dependency_timeout': + case 'default_restbuilder_timeout': + case 'nosql_cleaner': + name = obsolete_config(name); obj[name] = U.parseInt(value); break; - case 'default-image-consumption': - case 'default-image-quality': + case 'default-image-consumption': // old + case 'default-image-quality': // old + case 'default_image_consumption': + case 'default_image_quality': + name = obsolete_config(name); obj[name] = U.parseInt(value.replace(/%|\s/g, '')); break; - case 'static-accepts-custom': + case 'static-accepts-custom': // old + case 'static_accepts_custom': accepts = value.replace(REG_ACCEPTCLEANER, '').split(','); break; - case 'default-root': + case 'default-root': // old + case 'default_root': + name = obsolete_config(name); if (value) obj[name] = U.path(value); break; - case 'static-accepts': + case 'static-accepts': // old + case 'static_accepts': + name = obsolete_config(name); obj[name] = {}; tmp = value.replace(REG_ACCEPTCLEANER, '').split(','); for (var j = 0; j < tmp.length; j++) @@ -8730,7 +8769,7 @@ F.$configure_configs = function(arr, rewrite) { case 'mail.address.reply': if (name === 'mail.address.bcc') - tmp = 'mail-address-copy'; + tmp = 'mail_address_copy'; else tmp = name.replace(/\./g, '-'); @@ -8738,7 +8777,9 @@ F.$configure_configs = function(arr, rewrite) { obj[tmp] = value; break; - case 'default-cors': + case 'default-cors': // old + case 'default_cors': + name = obsolete_config(name); value = value.replace(/,/g, ' ').split(' '); tmp = []; for (var j = 0; j < value.length; j++) { @@ -8755,43 +8796,73 @@ F.$configure_configs = function(arr, rewrite) { break; case 'allow-handle-static-files': - OBSOLETE('config["allow-handle-static-files"]', 'The key has been renamed to "allow-static-files"'); - obj['allow-static-files'] = true; + OBSOLETE('config["allow-handle-static-files"]', 'The key has been renamed to "allow_static_files"'); + obj.allow_static_files = true; break; case 'disable-clear-temporary-directory': - OBSOLETE('disable-clear-temporary-directory', 'You need to use "allow-clear-temp : true|false"'); - obj['allow-clear-temp'] = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); + OBSOLETE('disable-clear-temporary-directory', 'You need to use "allow_clear_temp : true|false"'); + obj.allow_clear_temp = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); break; case 'disable-strict-server-certificate-validation': - OBSOLETE('disable-strict-server-certificate-validation', 'You need to use "allow-ssc-validation : true|false"'); - obj['allow-ssc-validation'] = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); + OBSOLETE('disable-strict-server-certificate-validation', 'You need to use "allow_ssc_validation : true|false"'); + obj.allow_ssc_validation = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); break; - case 'allow-compile-html': - case 'allow-compile-script': - case 'allow-compile-style': - case 'allow-ssc-validation': - case 'allow-debug': - case 'allow-gzip': - case 'allow-performance': - case 'allow-static-files': - case 'allow-websocket': - case 'allow-clear-temp': - case 'allow-cache-snapshot': + case 'allow-compile': // old + case 'allow-compile-html': // old + case 'allow-compile-script': // old + case 'allow-compile-style': // old + case 'allow-ssc-validation': // old + case 'allow-debug': // old + case 'allow-gzip': // old + case 'allow-head': // old + case 'allow-performance': // old + case 'allow-static-files': // old + case 'allow-websocket': // old + case 'allow-websocket-compression': // old + case 'allow-clear-temp': // old + case 'allow-cache-snapshot': // old + case 'allow-cache-cluster': // old + case 'allow-custom-titles': // old + case 'nosql-worker': // old + case 'nosql-logger': // old + case 'allow-filter-errors': // old + case 'default-websocket-encodedecode': // old + case 'allow_compile': + case 'allow_compile_html': + case 'allow_compile_script': + case 'allow_compile_style': + case 'allow_ssc_validation': + case 'allow_debug': + case 'allow_gzip': + case 'allow_head': + case 'allow_performance': + case 'allow_static_files': + case 'allow_websocket': + case 'allow_websocket_compression': + case 'allow_clear_temp': + case 'allow_cache_snapshot': + case 'allow_cache_cluster': + case 'allow_filter_errors': + case 'allow_custom_titles': case 'trace': - case 'nosql-worker': - case 'nosql-logger': + case 'nosql_worker': + case 'nosql_logger': + case 'default_websocket_encodedecode': + name = obsolete_config(name); obj[name] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; - case 'nosql-inmemory': + case 'nosql-inmemory': // old + case 'nosql_inmemory': + name = obsolete_config(name); obj[name] = typeof(value) === 'string' ? value.split(',').trim() : value instanceof Array ? value : null; break; case 'allow-compress-html': - obj['allow-compile-html'] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; + obj.allow_compile_html = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; case 'version': @@ -8802,6 +8873,34 @@ F.$configure_configs = function(arr, rewrite) { obj[name] = value ? value.split(',').trim().join('\n') : ''; break; + // backward compatibility + case 'mail-smtp': // old + case 'mail-smtp-options': // old + case 'mail-address-from': // old + case 'mail-address-copy': // old + case 'mail-address-bcc': // old + case 'mail-address-reply': // old + case 'default-image-converter': // old + case 'static-url': // old + case 'static-url-script': // old + case 'static-url-style': // old + case 'static-url-image': // old + case 'static-url-video': // old + case 'static-url-font': // old + case 'static-url-download': // old + case 'static-url-components': // old + case 'default-xpoweredby': // old + case 'default-layout': // old + case 'default-theme': // old + case 'default-proxy': // old + case 'default-timezone': // old + case 'default-response-maxage': // old + case 'default-errorbuilder-resource-name': // old + case 'default-errorbuilder-resource-prefix': // old + name = obsolete_config(name); + obj[name] = value; + break; + default: if (subtype === 'string') @@ -8834,44 +8933,44 @@ F.$configure_configs = function(arr, rewrite) { } } - U.extend(F.config, obj, rewrite); + U.extend(CONF, obj, rewrite); - if (!F.config['secret-uid']) - F.config['secret-uid'] = (F.config.name).crc32(true).toString(); + if (!CONF.secret_uid) + CONF.secret_uid = (CONF.name).crc32(true).toString(); - var tmp = F.config['mail-smtp-options']; + var tmp = CONF.mail_smtp_options; if (typeof(tmp) === 'string' && tmp) { tmp = new Function('return ' + tmp)(); - F.config['mail-smtp-options'] = tmp; + CONF.mail_smtp_options = tmp; } - if (!F.config['directory-temp']) - F.config['directory-temp'] = '~' + U.path(Path.join(Os.tmpdir(), 'totaljs' + F.directory.hash())); + if (!CONF.directory_temp) + CONF.directory_temp = '~' + U.path(Path.join(Os.tmpdir(), 'totaljs' + F.directory.hash())); - if (!F.config['etag-version']) - F.config['etag-version'] = F.config.version.replace(/\.|\s/g, ''); + if (!CONF.etag_version) + CONF.etag_version = CONF.version.replace(/\.|\s/g, ''); - if (F.config['default-timezone']) - process.env.TZ = F.config['default-timezone']; + if (CONF.default_timezone) + process.env.TZ = CONF.default_timezone; - F.config['nosql-worker'] && framework_nosql.worker(); - F.config['nosql-inmemory'] && F.config['nosql-inmemory'].forEach(n => framework_nosql.inmemory(n)); - accepts && accepts.length && accepts.forEach(accept => F.config['static-accepts'][accept] = true); + CONF.nosql_worker && framework_nosql.worker(); + CONF.nosql_inmemory && CONF.nosql_inmemory.forEach(n => framework_nosql.inmemory(n)); + accepts && accepts.length && accepts.forEach(accept => CONF.static_accepts[accept] = true); - if (F.config['allow-ssc-validation'] === false) + if (CONF.allow_ssc_validation === false) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; - if (F.config['allow-performance']) + if (CONF.allow_performance) http.globalAgent.maxSockets = 9999; - QUERYPARSEROPTIONS.maxKeys = F.config['default-request-maxkeys'] || 33; + QUERYPARSEROPTIONS.maxKeys = CONF.default_request_maxkeys || 33; - var xpowered = F.config['default-xpoweredby']; + var xpowered = CONF.default_xpoweredby; Object.keys(HEADERS).forEach(function(key) { Object.keys(HEADERS[key]).forEach(function(subkey) { if (RELEASE && subkey === 'Cache-Control') - HEADERS[key][subkey] = HEADERS[key][subkey].replace(/max-age=\d+/, 'max-age=' + F.config['default-response-maxage']); + HEADERS[key][subkey] = HEADERS[key][subkey].replace(/max-age=\d+/, 'max-age=' + CONF.default_response_maxage); if (subkey === 'X-Powered-By') { if (xpowered) HEADERS[key][subkey] = xpowered; @@ -8881,19 +8980,27 @@ F.$configure_configs = function(arr, rewrite) { }); }); - IMAGEMAGICK = F.config['default-image-converter'] === 'im'; + IMAGEMAGICK = CONF.default_image_converter === 'im'; done(); - F.emit('configure', F.config); + EMIT('configure', CONF); return F; }; +function obsolete_config(name) { + if (name.indexOf('-') === -1) + return name; + var n = name.replace(REG_OLDCONF, '_'); + OBSOLETE('config[\'' + name + '\']', 'Replace key "{0}" to "{1}" in your config file'.format(name, n)); + return n; +} + /** * Create URL: JavaScript (according to config['static-url-script']) * @param {String} name * @return {String} */ F.routeScript = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-script'], theme); + return F.$routeStatic(name, CONF.static_url_script, theme); }; /** @@ -8902,27 +9009,27 @@ F.routeScript = function(name, theme) { * @return {String} */ F.routeStyle = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-style'], theme); + return F.$routeStatic(name, CONF.static_url_style, theme); }; F.routeImage = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-image'], theme); + return F.$routeStatic(name, CONF.static_url_image, theme); }; F.routeVideo = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-video'], theme); + return F.$routeStatic(name, CONF.static_url_video, theme); }; F.routeFont = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-font'], theme); + return F.$routeStatic(name, CONF.static_url_font, theme); }; F.routeDownload = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-download'], theme); + return F.$routeStatic(name, CONF.static_url_download, theme); }; F.routeStatic = function(name, theme) { - return F.$routeStatic(name, F.config['static-url'], theme); + return F.$routeStatic(name, CONF.static_url, theme); }; F.$routeStatic = function(name, directory, theme) { @@ -8941,7 +9048,7 @@ F.$routeStatic = function(name, directory, theme) { if (index !== -1) { theme = name.substring(1, index); if (theme === '?') { - theme = F.config['default-theme']; + theme = CONF.default_theme; name = name.substring(index); } else name = name.substring(index + 1); @@ -9155,7 +9262,7 @@ F.lookup_websocket = function(req, url, membertype) { F.accept = function(extension, contentType) { if (extension[0] === '.') extension = extension.substring(1); - F.config['static-accepts'][extension] = true; + CONF.static_accepts[extension] = true; contentType && U.setContentType(extension, contentType); return F; }; @@ -9196,7 +9303,7 @@ F.worker = function(name, id, timeout, args) { if (fork) return fork; - var filename = name[0] === '@' ? F.path.package(name.substring(1)) : U.combine(F.config['directory-workers'], name); + var filename = name[0] === '@' ? F.path.package(name.substring(1)) : U.combine(CONF.directory_workers, name); if (!args) args = []; @@ -9380,7 +9487,7 @@ FrameworkPath.prototype.verify = function(name) { var prop = '$directory-' + name; if (F.temporary.path[prop]) return F; - var directory = F.config['directory-' + name] || name; + var directory = CONF['directory_' + name] || name; var dir = U.combine(directory); !existsSync(dir) && Fs.mkdirSync(dir); F.temporary.path[prop] = true; @@ -9435,43 +9542,43 @@ FrameworkPath.prototype.exists = function(path, callback) { }; FrameworkPath.prototype.public = function(filename) { - return U.combine(F.config['directory-public'], filename); + return U.combine(CONF.directory_public, filename); }; FrameworkPath.prototype.public_cache = function(filename) { var key = 'public_' + filename; var item = F.temporary.other[key]; - return item ? item : F.temporary.other[key] = U.combine(F.config['directory-public'], filename); + return item ? item : F.temporary.other[key] = U.combine(CONF.directory_public, filename); }; FrameworkPath.prototype.private = function(filename) { - return U.combine(F.config['directory-private'], filename); + return U.combine(CONF.directory_private, filename); }; FrameworkPath.prototype.isomorphic = function(filename) { - return U.combine(F.config['directory-isomorphic'], filename); + return U.combine(CONF.directory_isomorphic, filename); }; FrameworkPath.prototype.configs = function(filename) { - return U.combine(F.config['directory-configs'], filename); + return U.combine(CONF.directory_configs, filename); }; FrameworkPath.prototype.virtual = function(filename) { - return U.combine(F.config['directory-public-virtual'], filename); + return U.combine(CONF.directory_public_virtual, filename); }; FrameworkPath.prototype.logs = function(filename) { this.verify('logs'); - return U.combine(F.config['directory-logs'], filename); + return U.combine(CONF.directory_logs, filename); }; FrameworkPath.prototype.models = function(filename) { - return U.combine(F.config['directory-models'], filename); + return U.combine(CONF.directory_models, filename); }; FrameworkPath.prototype.temp = function(filename) { this.verify('temp'); - return U.combine(F.config['directory-temp'], filename); + return U.combine(CONF.directory_temp, filename); }; FrameworkPath.prototype.temporary = function(filename) { @@ -9479,52 +9586,52 @@ FrameworkPath.prototype.temporary = function(filename) { }; FrameworkPath.prototype.views = function(filename) { - return U.combine(F.config['directory-views'], filename); + return U.combine(CONF.directory_views, filename); }; FrameworkPath.prototype.workers = function(filename) { - return U.combine(F.config['directory-workers'], filename); + return U.combine(CONF.directory_workers, filename); }; FrameworkPath.prototype.databases = function(filename) { this.verify('databases'); - return U.combine(F.config['directory-databases'], filename); + return U.combine(CONF.directory_databases, filename); }; FrameworkPath.prototype.modules = function(filename) { - return U.combine(F.config['directory-modules'], filename); + return U.combine(CONF.directory_modules, filename); }; FrameworkPath.prototype.controllers = function(filename) { - return U.combine(F.config['directory-controllers'], filename); + return U.combine(CONF.directory_controllers, filename); }; FrameworkPath.prototype.definitions = function(filename) { - return U.combine(F.config['directory-definitions'], filename); + return U.combine(CONF.directory_definitions, filename); }; FrameworkPath.prototype.tests = function(filename) { - return U.combine(F.config['directory-tests'], filename); + return U.combine(CONF.directory_tests, filename); }; FrameworkPath.prototype.resources = function(filename) { - return U.combine(F.config['directory-resources'], filename); + return U.combine(CONF.directory_resources, filename); }; FrameworkPath.prototype.services = function(filename) { - return U.combine(F.config['directory-services'], filename); + return U.combine(CONF.directory_services, filename); }; FrameworkPath.prototype.packages = function(filename) { - return U.combine(F.config['directory-packages'], filename); + return U.combine(CONF.directory_packages, filename); }; FrameworkPath.prototype.themes = function(filename) { - return U.combine(F.config['directory-themes'], filename); + return U.combine(CONF.directory_themes, filename); }; FrameworkPath.prototype.components = function(filename) { - return U.combine(F.config['directory-components'], filename); + return U.combine(CONF.directory_components, filename); }; FrameworkPath.prototype.root = function(filename) { @@ -9542,7 +9649,7 @@ FrameworkPath.prototype.package = function(name, filename) { } } - var tmp = F.config['directory-temp']; + var tmp = CONF.directory_temp; var p = tmp[0] === '~' ? Path.join(tmp.substring(1), name + '.package', filename || '') : Path.join(directory, tmp, name + '.package', filename || ''); return F.isWindows ? p.replace(REG_WINDOWSPATH, '/') : p; }; @@ -9561,7 +9668,7 @@ FrameworkCache.prototype.init = function() { var self = this; clearInterval(self.interval); self.interval = setInterval(() => F.cache.recycle(), 1000 * 60); - if (F.config['allow-cache-snapshot']) + if (CONF.allow_cache_snapshot) self.load(() => self.loadpersistent()); else self.loadpersistent(); @@ -9631,7 +9738,7 @@ FrameworkCache.prototype.stop = function() { FrameworkCache.prototype.clear = function(sync) { this.items = {}; - F.isCluster && sync !== false && F.config['allow-cache-cluster'] && process.send(CLUSTER_CACHE_CLEAR); + F.isCluster && sync !== false && CONF.allow_cache_cluster && process.send(CLUSTER_CACHE_CLEAR); this.savePersist(); return this; }; @@ -9652,13 +9759,13 @@ FrameworkCache.prototype.recycle = function() { else if (value.expire < NOW) { if (value.persist) isPersist = true; - F.emit('cache-expire', o, value.value); + EMIT('cache-expire', o, value.value); delete items[o]; } } isPersist && this.savePersist(); - F.config['allow-cache-snapshot'] && this.save(); + CONF.allow_cache_snapshot && this.save(); F.service(this.count); return this; }; @@ -9670,7 +9777,7 @@ FrameworkCache.prototype.set2 = function(name, value, expire, sync) { FrameworkCache.prototype.set = FrameworkCache.prototype.add = function(name, value, expire, sync, persist) { var type = typeof(expire); - if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { + if (F.isCluster && sync !== false && CONF.allow_cache_cluster) { CLUSTER_CACHE_SET.key = name; CLUSTER_CACHE_SET.value = value; CLUSTER_CACHE_SET.expire = expire; @@ -9694,7 +9801,7 @@ FrameworkCache.prototype.set = FrameworkCache.prototype.add = function(name, val } this.items[name] = obj; - F.$events['cache-set'] && F.emit('cache-set', name, value, expire, sync !== false); + F.$events['cache-set'] && EMIT('cache-set', name, value, expire, sync !== false); return value; }; @@ -9708,7 +9815,7 @@ FrameworkCache.prototype.read = FrameworkCache.prototype.get = function(key, def if (value.expire < NOW) { this.items[key] = undefined; - F.$events['cache-expire'] && F.emit('cache-expire', key, value.value); + F.$events['cache-expire'] && EMIT('cache-expire', key, value.value); return def; } @@ -9723,7 +9830,7 @@ FrameworkCache.prototype.read2 = FrameworkCache.prototype.get2 = function(key, d if (value.expire < NOW) { this.items[key] = undefined; - F.$events['cache-expire'] && F.emit('cache-expire', key, value.value); + F.$events['cache-expire'] && EMIT('cache-expire', key, value.value); return def; } @@ -9745,7 +9852,7 @@ FrameworkCache.prototype.remove = function(name, sync) { this.items[name] = undefined; } - if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { + if (F.isCluster && sync !== false && CONF.allow_cache_cluster) { CLUSTER_CACHE_REMOVE.key = name; process.send(CLUSTER_CACHE_REMOVE); } @@ -9771,7 +9878,7 @@ FrameworkCache.prototype.removeAll = function(search, sync) { count++; } - if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { + if (F.isCluster && sync !== false && CONF.allow_cache_cluster) { CLUSTER_CACHE_REMOVEALL.key = search; process.send(CLUSTER_CACHE_REMOVEALL); } @@ -9837,8 +9944,8 @@ function Controller(name, req, res, currentView) { // controller.type === 1 - server sent events // this.type = 0; - // this.layoutName = F.config['default-layout']; - // this.themeName = F.config['default-theme']; + // this.layoutName =CONF.default_layout; + // this.themeName =CONF.default_theme; // this.status = 200; // this.isLayout = false; @@ -9941,7 +10048,7 @@ Controller.prototype = { }, get config() { - return F.config; + return CONF; }, get controllers() { @@ -9949,7 +10056,7 @@ Controller.prototype = { }, get isDebug() { - return F.config.debug; + return CONF.debug; }, get isTest() { @@ -10299,7 +10406,7 @@ Controller.prototype.component = function(name, settings, model) { for (var i = 0; i < generator.components.length; i++) self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; } - return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, self.config, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); + return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, CONF, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); } } return ''; @@ -10689,7 +10796,7 @@ Controller.prototype.$meta = function() { return ''; } - F.$events['controller-render-meta'] && F.emit('controller-render-meta', self); + F.$events['controller-render-meta'] && EMIT('controller-render-meta', self); var repository = self.repository; return F.onMeta.call(self, repository[REPOSITORY_META_TITLE], repository[REPOSITORY_META_DESCRIPTION], repository[REPOSITORY_META_KEYWORDS], repository[REPOSITORY_META_IMAGE]); }; @@ -11406,7 +11513,7 @@ Controller.prototype.head = function() { var self = this; if (!arguments.length) { - var author = self.repository[REPOSITORY_META_AUTHOR] || self.config.author; + var author = self.repository[REPOSITORY_META_AUTHOR] || CONF.author; var plus = ''; var components = self.repository[REPOSITORY_COMPONENTS]; if (components) { @@ -12587,7 +12694,7 @@ Controller.prototype.close = function(end) { self.isConnected = false; self.res.success = true; F.reqstats(false, false); - F.$events['request-end'] && F.emit('request-end', self.req, self.res); + F.$events['request-end'] && EMIT('request-end', self.req, self.res); self.type = 0; end && self.res.end(); self.req.clear(true); @@ -12601,7 +12708,7 @@ Controller.prototype.close = function(end) { self.res.success = true; F.reqstats(false, false); - F.$events['request-end'] && F.emit('request-end', self.req, self.res); + F.$events['request-end'] && EMIT('request-end', self.req, self.res); end && self.res.end(); self.req.clear(true); return self; @@ -12710,9 +12817,9 @@ Controller.prototype.view = function(name, model, headers, partial, noasync, cac return self; if (self.layoutName === undefined) - self.layoutName = F.config['default-layout']; + self.layoutName = CONF.default_layout; if (self.themeName === undefined) - self.themeName = F.config['default-theme']; + self.themeName = CONF.default_theme; // theme root `~some_view` // views root `~~some_view` @@ -12863,7 +12970,7 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; } - value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, CONF, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); } catch (ex) { @@ -13202,7 +13309,7 @@ WebSocket.prototype = { }, get config() { - return F.config; + return CONF; }, get cache() { @@ -13210,7 +13317,7 @@ WebSocket.prototype = { }, get isDebug() { - return F.config.debug; + return CONF.debug; }, get path() { @@ -13530,7 +13637,7 @@ WebSocket.prototype.destroy = function(problem) { return self; self.close(); - self.$events.destroy && self.emit('destroy'); + self.$events.destroy && selEMIT('destroy'); setTimeout(function() { @@ -13778,7 +13885,7 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { } } - var compress = (F.config['allow-websocket-compression'] && self.req.headers['sec-websocket-extensions'] || '').indexOf('permessage-deflate') !== -1; + var compress = (CONF.allow_websocket_compression && self.req.headers['sec-websocket-extensions'] || '').indexOf('permessage-deflate') !== -1; var header = protocols.length ? (compress ? SOCKET_RESPONSE_PROTOCOL_COMPRESS : SOCKET_RESPONSE_PROTOCOL).format(self.$websocket_key(self.req), protocols.join(', ')) : (compress ? SOCKET_RESPONSE_COMPRESS : SOCKET_RESPONSE).format(self.$websocket_key(self.req)); self.socket.write(U.createBuffer(header, 'binary')); @@ -13841,7 +13948,7 @@ WebSocketClient.prototype.upgrade = function(container) { self.socket.on('end', websocket_close); self.container.$add(self); self.container.$refresh(); - F.$events['websocket-begin'] && F.emit('websocket-begin', self.container, self); + F.$events['websocket-begin'] && EMIT('websocket-begin', self.container, self); self.container.$events.open && self.container.emit('open', self); return self; }; @@ -13924,7 +14031,7 @@ WebSocketClient.prototype.$ondata = function(data) { this.closemessage = current.buffer.slice(4).toString('utf8'); this.closecode = current.buffer[2] << 8 | current.buffer[3]; - if (this.closemessage && F.config['default-websocket-encodedecode']) + if (this.closemessage && CONF.default_websocket_encodedecode) this.closemessage = $decodeURIComponent(this.closemessage); this.close(); @@ -14080,7 +14187,7 @@ WebSocketClient.prototype.$decode = function() { case 3: // JSON if (data instanceof Buffer) data = data.toString(ENCODING); - F.config['default-websocket-encodedecode'] === true && (data = $decodeURIComponent(data)); + CONF.default_websocket_encodedecode === true && (data = $decodeURIComponent(data)); if (data.isJSON()) { var tmp = F.onParseJSON(data, this.req); if (tmp !== undefined) @@ -14091,7 +14198,7 @@ WebSocketClient.prototype.$decode = function() { default: // TEXT if (data instanceof Buffer) data = data.toString(ENCODING); - this.container.emit('message', this, F.config['default-websocket-encodedecode'] === true ? $decodeURIComponent(data) : data); + this.container.emit('message', this, CONF.default_websocket_encodedecode === true ? $decodeURIComponent(data) : data); break; } @@ -14174,7 +14281,7 @@ WebSocketClient.prototype.$onclose = function() { this.container.$refresh(); this.container.$events.close && this.container.emit('close', this, this.closecode, this.closemessage); this.socket.removeAllListeners(); - F.$events['websocket-end'] && F.emit('websocket-end', this.container, this); + F.$events['websocket-end'] && EMIT('websocket-end', this.container, this); }; /** @@ -14190,7 +14297,7 @@ WebSocketClient.prototype.send = function(message, raw, replacer) { if (this.type !== 1) { var data = this.type === 3 ? (raw ? message : JSON.stringify(message, replacer)) : (message || '').toString(); - if (F.config['default-websocket-encodedecode'] === true && data) + if (CONF.default_websocket_encodedecode === true && data) data = encodeURIComponent(data); if (this.deflate) { this.deflatepending.push(U.createBuffer(data)); @@ -14256,7 +14363,7 @@ WebSocketClient.prototype.close = function(message, code) { if (!self.isClosed) { self.isClosed = true; if (self.ready) - self.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + self.socket.end(U.getWebSocketFrame(code || 1000, message ? (CONF.default_websocket_encodedecode ? encodeURIComponent(message) : message) : '', 0x08)); else self.socket.end(); self.req.connection.destroy(); @@ -14542,7 +14649,7 @@ function extend_request(PROTO) { this.$total_header = header; if (this.$total_route) { F.path.verify('temp'); - framework_internal.parseMULTIPART(this, header, this.$total_route, F.config['directory-temp']); + framework_internal.parseMULTIPART(this, header, this.$total_route, CONF.directory_temp); } else this.$total_status(404); }; @@ -14568,7 +14675,7 @@ function extend_request(PROTO) { F.reqstats(false, false); this.res.writeHead(status); this.res.end(U.httpStatus(status)); - F.$events['request-end'] && F.emit('request-end', this, this.res); + F.$events['request-end'] && EMIT('request-end', this, this.res); this.clear(true); }; @@ -14590,7 +14697,7 @@ function extend_request(PROTO) { if (isError || !route) { F.stats.response['error' + status]++; - status !== 500 && F.$events['error'] && F.emit('error' + status, this, res, this.$total_exception); + status !== 500 && F.$events['error'] && EMIT('error' + status, this, res, this.$total_exception); } if (!route) { @@ -14659,8 +14766,8 @@ function extend_request(PROTO) { return; var ctrlname = '@' + name; - F.$events.controller && F.emit('controller', controller, name, this.$total_route.options); - F.$events[ctrlname] && F.emit(ctrlname, controller, name, this.$total_route.options); + F.$events.controller && EMIT('controller', controller, name, this.$total_route.options); + F.$events[ctrlname] && EMIT(ctrlname, controller, name, this.$total_route.options); if (controller.isCanceled) return; @@ -15144,7 +15251,7 @@ function extend_response(PROTO) { if (!accept && isGZIP(req)) accept = 'gzip'; - var compress = F.config['allow-gzip'] && accept.indexOf('gzip') !== -1; + var compress = CONF.allow_gzip && accept.indexOf('gzip') !== -1; if (isHEAD) { compress && (headers['Content-Encoding'] = 'gzip'); res.writeHead(200, headers); @@ -15351,7 +15458,7 @@ function extend_response(PROTO) { var options = { protocol: uri.protocol, auth: uri.auth, method: 'GET', hostname: uri.hostname, port: uri.port, path: uri.path, agent: false, headers: headers }; var connection = options.protocol === 'https:' ? require('https') : http; - var gzip = F.config['allow-gzip'] && (res.req.headers['accept-encoding'] || '').lastIndexOf('gzip') !== -1; + var gzip = CONF.allow_gzip && (res.req.headers['accept-encoding'] || '').lastIndexOf('gzip') !== -1; var client = connection.get(options, function(response) { @@ -15457,13 +15564,13 @@ function extend_response(PROTO) { if (res.success || res.headersSent) return res; - if (!F.config['static-accepts'][req.extension]) { + if (!CONF.static_accepts[req.extension]) { res.throw404(); return res; } - if (SECURITYTXT[req.url] && F.config['security.txt']) { - res.send(200, F.config['security.txt'], 'text/plain'); + if (SECURITYTXT[req.url] && CONF['security.txt']) { + res.send(200, CONF['security.txt'], 'text/plain'); return; } @@ -15501,7 +15608,7 @@ function extend_response(PROTO) { if (!canresize) { - if (F.components.has && F.components[req.extension] && req.uri.pathname === F.config['static-url-components'] + req.extension) { + if (F.components.has && F.components[req.extension] && req.uri.pathname === CONF.static_url_components + req.extension) { res.noCompress = true; res.options.components = true; var g = req.query.group ? req.query.group.substring(0, req.query.group.length - 6) : ''; @@ -15611,7 +15718,7 @@ function extend_response(PROTO) { if (name === undefined) { if (F.temporary.processing[req.$key]) { - if (req.processing > F.config['default-request-timeout']) { + if (req.processing > CONF.default_request_timeout) { res.throw408(); } else { req.processing += 500; @@ -15632,7 +15739,7 @@ function extend_response(PROTO) { !accept && isGZIP(req) && (accept = 'gzip'); - var compress = F.config['allow-gzip'] && COMPRESSION[contentType] && accept.indexOf('gzip') !== -1 && name.length > 2; + var compress = CONF.allow_gzip && COMPRESSION[contentType] && accept.indexOf('gzip') !== -1 && name.length > 2; var range = req.headers.range; var canCache = !res.$nocache && RELEASE && contentType !== 'text/cache-manifest'; @@ -15676,7 +15783,7 @@ function extend_response(PROTO) { else if (!res.options.lastmodified) headers['Last-Modified'] = name[2]; - headers.Etag = ETAG + F.config['etag-version']; + headers.Etag = ETAG + CONF.etag_version; if (range) { $file_range(name[0], range, headers, res); @@ -15756,7 +15863,7 @@ function extend_response(PROTO) { var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); - var compress = F.config['allow-gzip'] && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; + var compress = CONF.allow_gzip && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; var headers = compress ? HEADERS.binary_compress : HEADERS.binary; headers['Vary'] = 'Accept-Encoding' + (req.$mobile ? ', User-Agent' : ''); @@ -15815,7 +15922,7 @@ function extend_response(PROTO) { var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); - var compress = (options.compress === undefined || options.compress) && F.config['allow-gzip'] && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; + var compress = (options.compress === undefined || options.compress) && CONF.allow_gzip && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; var headers; if (RELEASE) { @@ -15916,7 +16023,7 @@ function extend_response(PROTO) { } if (F.temporary.processing[req.$key]) { - if (req.processing > F.config['default-request-timeout']) { + if (req.processing > CONF.default_request_timeout) { res.throw408(); } else { req.processing += 500; @@ -15964,7 +16071,7 @@ function extend_response(PROTO) { var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); - var gzip = F.config['allow-gzip'] && (options.compress === undefined || options.compress) ? accept.indexOf('gzip') !== -1 : false; + var gzip = CONF.allow_gzip && (options.compress === undefined || options.compress) ? accept.indexOf('gzip') !== -1 : false; var headers; if (req.$mobile) @@ -16079,7 +16186,7 @@ function extend_response(PROTO) { req.$total_execute(res.options.code, true); } - F.$events[key] && F.emit(key, req, res, res.options.problem); + F.$events[key] && EMIT(key, req, res, res.options.problem); return res; }; } @@ -16109,7 +16216,7 @@ function $file_notmodified(res, name) { if (res.getHeader('ETag')) delete headers.Etag; else - headers.Etag = ETAG + F.config['etag-version']; + headers.Etag = ETAG + CONF.etag_version; headers[HEADER_TYPE] = U.getContentType(req.extension); res.writeHead(304, headers); @@ -16313,7 +16420,7 @@ function $image_filename(exists, size, isFile, stats, res) { function response_end(res) { F.reqstats(false, res.req.isStaticFile); res.success = true; - !res.req.isStaticFile && F.$events['request-end'] && F.emit('request-end', res.req, res); + !res.req.isStaticFile && F.$events['request-end'] && EMIT('request-end', res.req, res); res.req.clear(true); res.controller && res.req.$total_success(); @@ -16356,14 +16463,14 @@ process.on('uncaughtException', function(e) { console.log('\nThe IP address and the PORT is already in use.\nYou must change the PORT\'s number or IP address.\n'); process.exit('SIGTERM'); return; - } else if (F.config['allow-filter-errors'] && REG_SKIPERROR.test(err)) + } else if (CONF.allow_filter_errors && REG_SKIPERROR.test(err)) return; F.error(e, '', null); }); function fsFileRead(filename, callback, a, b, c) { - U.queue('F.files', F.config['default-maxopenfiles'], function(next) { + U.queue('F.files', CONF.default_maxopenfiles, function(next) { Fs.readFile(filename, function(err, result) { next(); callback(err, result, a, b, c); @@ -16372,7 +16479,7 @@ function fsFileRead(filename, callback, a, b, c) { } function fsFileExists(filename, callback, a, b, c) { - U.queue('F.files', F.config['default-maxopenfiles'], function(next) { + U.queue('F.files', CONF.default_maxopenfiles, function(next) { Fs.lstat(filename, function(err, stats) { next(); callback(!err && stats.isFile(), stats ? stats.size : 0, stats ? stats.isFile() : false, stats, a, b, c); @@ -16401,7 +16508,7 @@ function fsStreamRead(filename, options, callback, res) { } else opt = HEADERS.fsStreamRead; - U.queue('F.files', F.config['default-maxopenfiles'], function(next) { + U.queue('F.files', CONF.default_maxopenfiles, function(next) { var stream = Fs.createReadStream(filename, opt); stream.on('error', NOOP); callback(stream, next, res); @@ -16491,11 +16598,11 @@ process.on('message', function(msg, h) { msg.TYPE === 'cache-clear' && F.cache.clear(false); msg.TYPE === 'req' && F.cluster.req(msg); msg.TYPE === 'res' && msg.target === F.id && F.cluster.res(msg); - msg.TYPE === 'emit' && F.$events[msg.name] && F.emit(msg.name, msg.data); + msg.TYPE === 'emit' && F.$events[msg.name] && EMIT(msg.name, msg.data); msg.TYPE === 'nosql-meta' && NOSQL(msg.name).meta(msg.key, msg.value, true); msg.TYPE === 'table-meta' && TABLE(msg.name).meta(msg.key, msg.value, true); } - F.$events.message && F.emit('message', msg, h); + F.$events.message && EMIT('message', msg, h); }); function prepare_error(e) { @@ -16508,7 +16615,7 @@ function prepare_error(e) { } function prepare_filename(name) { - return name[0] === '@' ? (F.isWindows ? U.combine(F.config['directory-temp'], name.substring(1)) : F.path.package(name.substring(1))) : U.combine('/', name); + return name[0] === '@' ? (F.isWindows ? U.combine(CONF.directory_temp, name.substring(1)) : F.path.package(name.substring(1))) : U.combine('/', name); } function prepare_staticurl(url, isDirectory) { @@ -16878,7 +16985,7 @@ function parseSchema(name) { function ilogger(body) { F.path.verify('logs'); - U.queue('F.ilogger', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], 'logger.log'), body, next)); + U.queue('F.ilogger', 5, (next) => Fs.appendFile(U.combine(CONF.directory_logs, 'logger.log'), body, next)); } F.ilogger = function(name, req, ts) { diff --git a/internal.js b/internal.js index 47f88241c..c9b54755e 100755 --- a/internal.js +++ b/internal.js @@ -771,7 +771,7 @@ HFP.isAudio = function() { HFP.image = function(im) { if (im === undefined) - im = F.config['default-image-converter'] === 'im'; + im = CONF.default_image_converter === 'im'; return framework_image.init(this.path, im, this.width, this.height); }; @@ -1813,7 +1813,7 @@ function view_parse(content, minify, filename, controller) { if (cmd[0] === '\'' || cmd[0] === '"') { if (cmd[1] === '%') { - var t = F.config[cmd.substring(2, cmd.length - 1)]; + var t = CONF[cmd.substring(2, cmd.length - 1)]; if (t != null) builder += '+' + DELIMITER + t + DELIMITER; } else @@ -1922,7 +1922,7 @@ function view_parse(content, minify, filename, controller) { if (can && !counter) { try { - var r = (new Function('self', 'config', 'return ' + tmp))(controller, F.config).replace(REG_7, '\\\\').replace(REG_8, '\\\''); + var r = (new Function('self', 'config', 'return ' + tmp))(controller, CONF).replace(REG_7, '\\\\').replace(REG_8, '\\\''); if (r) { txtindex = $VIEWCACHE.indexOf(r); if (txtindex === -1) { @@ -2044,7 +2044,7 @@ function view_prepare(command, dynamicCommand, functions, controller, components return '$STRING(' + command + ')'; case 'root': - var r = F.config['default-root']; + var r = CONF.default_root; return '\'' + (r ? r.substring(0, r.length - 1) : r) + '\''; case 'M': @@ -2503,7 +2503,7 @@ function compressView(html, minify) { */ function compressJS(html, index, filename, nomarkup) { - if (!F.config['allow-compile-script']) + if (!CONF.allow_compile_script) return html; var strFrom = '', 'framework.view()'); - }, 100); + exports.installed = true; + ROUTE('/inline-view-route/'); + setTimeout(function() { + assert.ok(VIEW('view') === '
Total.js
', 'VIEW()'); + }, 100); }; \ No newline at end of file diff --git a/test/modules/test.js b/test/modules/test.js index 88825c6f6..0699116ed 100755 --- a/test/modules/test.js +++ b/test/modules/test.js @@ -5,9 +5,9 @@ exports.install = function() { app = framework; assert.ok(typeof(framework.modules) === 'object', 'module install'); - setTimeout(function() { - assert.ok(MODULE('inline-view').installed, 'module install dependencies'); - }, 3000); + setTimeout(function() { + assert.ok(MODULE('inline-view').installed, 'module install dependencies'); + }, 3000); }; exports.message = function() { diff --git a/test/test-tmp.js b/test/test-tmp.js index b6aceaf82..97fa07942 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -7,7 +7,7 @@ require('../index'); // NOSQL('test').on('modify', console.log); // NOSQL('test').update({ name: GUID(5) }).between('index', 1, 3).callback(console.log); -F.config['table.test'] = 'index:number | name:string'; +CONF['table.test'] = 'index:number | name:string'; TABLE('test').find().take(10).skip(10).callback(console.log); diff --git a/utils.js b/utils.js index ef49b8004..e1c2d0ea9 100755 --- a/utils.js +++ b/utils.js @@ -485,7 +485,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (callback === NOOP) callback = null; - var options = { length: 0, timeout: timeout || CONF['default-restbuilder-timeout'], evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; + var options = { length: 0, timeout: timeout || CONF.default_restbuilder_timeout, evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; var method; var type = 0; var isCookies = false; @@ -641,8 +641,8 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (options.resolve && (uri.hostname === 'localhost' || uri.hostname.charCodeAt(0) < 64)) options.resolve = null; - if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) - proxy = parseProxy(F.config['default-proxy']); + if (CONF.default_proxy && !proxy && !PROXYBLACKLIST[uri.hostname]) + proxy = parseProxy(CONF.default_proxy); if (proxy && (uri.hostname === 'localhost' || uri.hostname === '127.0.0.1')) proxy = null; @@ -1180,8 +1180,8 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi headers['Content-Length'] = options.data.length; } - if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) - proxy = parseProxy(F.config['default-proxy']); + if (CONF.default_proxy && !proxy && !PROXYBLACKLIST[uri.hostname]) + proxy = parseProxy(CONF.default_proxy); options.proxy = proxy; @@ -3023,7 +3023,7 @@ SP.ROOT = function(noremap) { return ''; }).replace(REG_ROOT, $urlmaker); - if (!noremap && F.config['default-root']) + if (!noremap && CONF.default_root) str = str.replace(REG_REMAP, $urlremap); return str; @@ -3031,12 +3031,12 @@ SP.ROOT = function(noremap) { function $urlremap(text) { var pos = text[0] === 'h' ? 6 : 5; - return REG_URLEXT.test(text) ? text : ((text[0] === 'h' ? 'href' : 'src') + '="' + F.config['default-root'] + (text[pos] === '/' ? text.substring(pos + 1) : text)); + return REG_URLEXT.test(text) ? text : ((text[0] === 'h' ? 'href' : 'src') + '="' + CONF.default_root + (text[pos] === '/' ? text.substring(pos + 1) : text)); } function $urlmaker(text) { var c = text[4]; - return F.config['default-root'] ? F.config['default-root'] : (c || ''); + return CONF.default_root ? CONF.default_root : (c || ''); } if (!SP.trim) { @@ -3492,7 +3492,7 @@ SP.parseConfig = function(def, onerr) { obj[name] = (/true|on|1|enabled/i).test(value); break; case 'config': - obj[name] = F.config[value]; + obj[name] = CONF[value]; break; case 'eval': case 'object': @@ -3857,7 +3857,7 @@ SP.encrypt = function(key, isUnique, secret) { for (var i = 0; i < str.length; i++) sum += str.charCodeAt(i); - return (sum + checksum((secret || F.config.secret) + key)) + '-' + str; + return (sum + checksum((secret || CONF.secret) + key)) + '-' + str; }; SP.decrypt = function(key, secret) { @@ -3871,7 +3871,7 @@ SP.decrypt = function(key, secret) { return null; var hash = this.substring(index + 1); - var sum = checksum((secret || F.config.secret) + key); + var sum = checksum((secret || CONF.secret) + key); for (var i = 0; i < hash.length; i++) sum += hash.charCodeAt(i); @@ -3910,7 +3910,7 @@ exports.encryptUID = function(val, key) { var sum = 0; if (!key) - key = F.config.secret; + key = CONF.secret; val = val.toString(); @@ -3920,7 +3920,7 @@ exports.encryptUID = function(val, key) { for (var i = 0; i < key.length; i++) sum += key.charCodeAt(i); - return (num ? 'n' : 'x') + (F.config['secret-uid'] + val + sum + key).crc32(true).toString(16) + 'x' + val; + return (num ? 'n' : 'x') + (CONF.secret_uid + val + sum + key).crc32(true).toString(16) + 'x' + val; }; exports.decryptUID = function(val, key) { @@ -5792,7 +5792,7 @@ exports.parseTheme = function(value) { if (index === -1) return ''; value = value.substring(1, index); - return value === '?' ? F.config['default-theme'] : value; + return value === '?' ? CONF.default_theme : value; }; exports.set = function(obj, path, value) { From a46441d52e3abf3d6bc5bad52aae3d53a3201987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Oct 2018 19:00:41 +0200 Subject: [PATCH 0710/1669] Added config encryption. --- changes.txt | 1 + index.js | 5 +++++ test/config-debug | 5 ++++- test/config-release | 5 ++++- test/test-framework-debug.js | 2 ++ test/test-framework-release.js | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 97e65cd37..752c7f1cf 100755 --- a/changes.txt +++ b/changes.txt @@ -19,6 +19,7 @@ - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` +- added: encryption of config values - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index ea8bbeed2..f1c8b0c4f 100755 --- a/index.js +++ b/index.js @@ -8672,6 +8672,11 @@ F.$configure_configs = function(arr, rewrite) { value = str.substring(index + 1).trim(); index = name.indexOf('('); + if (value.substring(0, 7) === 'base64 ' && value.length > 8) + value = U.createBuffer(value.substring(7).trim(), 'base64').toString('utf8'); + else if (value.substring(0, 4) === 'hex ' && value.length > 6) + value = U.createBuffer(value.substring(4).trim(), 'hex').toString('utf8'); + if (index !== -1) { subtype = name.substring(index + 1, name.indexOf(')')).trim().toLowerCase(); name = name.substring(0, index).trim(); diff --git a/test/config-debug b/test/config-debug index ef433010f..7a700c7d6 100755 --- a/test/config-debug +++ b/test/config-debug @@ -1,3 +1,6 @@ etag-version : 1 secret : total.js test -array (Array) : [1, 2, 3, 4] \ No newline at end of file +array (Array) : [1, 2, 3, 4] + +testbase : base64 MTIzNDU2 +testhex : hex 313233343536 \ No newline at end of file diff --git a/test/config-release b/test/config-release index ef433010f..7a700c7d6 100755 --- a/test/config-release +++ b/test/config-release @@ -1,3 +1,6 @@ etag-version : 1 secret : total.js test -array (Array) : [1, 2, 3, 4] \ No newline at end of file +array (Array) : [1, 2, 3, 4] + +testbase : base64 MTIzNDU2 +testhex : hex 313233343536 \ No newline at end of file diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 5f723711b..7b814a460 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -29,6 +29,8 @@ framework.on('ready', function() { }); t.on('exit', () => assert.ok(a === true, 'F.load() in worker')); assert.ok(F.config.array.length === 4, 'Problem with config sub types.'); + assert.ok(CONF.testhex === 123456, 'config: hex encode'); + assert.ok(CONF.testbase === 123456, 'config: base encode'); }); framework.onAuthorize = function(req, res, flags, cb) { diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 3cfccb3ac..e5a2efb7d 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -29,6 +29,8 @@ framework.on('ready', function() { }); t.on('exit', () => assert.ok(a === true, 'F.load() in worker')); assert.ok(F.config.array.length === 4, 'Problem with config sub types.'); + assert.ok(CONF.testhex === 123456, 'config: hex encode'); + assert.ok(CONF.testbase === 123456, 'config: base encode'); }); framework.onAuthorize = function(req, res, flags, cb) { From b74c04beb80ac449c4fc24a3f3930e6bed249d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Oct 2018 20:29:33 +0200 Subject: [PATCH 0711/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bdff2dd63..ba55e40af 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-19", + "version": "3.0.1-20", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From d8bc00e616eccc3126ed386c877659e21d5e66fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 22 Oct 2018 11:04:01 +0200 Subject: [PATCH 0712/1669] Fixed config values. --- index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/index.js b/index.js index f1c8b0c4f..5fffc760f 100755 --- a/index.js +++ b/index.js @@ -771,6 +771,16 @@ function Framework() { default_interval_websocket_ping: 3, default_interval_uptodate: 5, + set ['mail-smtp'] (val) { + CONF['mail_smtp'] = val; + return null; + }, + + set ['mail-smtp-options'] (val) { + CONF['mail_smtp_options'] = val; + return null; + }, + set ['mail-address-reply'] (val) { CONF['mail_address_reply'] = val; return null; From e081cc478027ba9c7787d5ad85d7e0b1d1571714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 22 Oct 2018 11:12:57 +0200 Subject: [PATCH 0713/1669] Updated TABLE reference to `CONF.table_{name}`. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 6e1964058..ffbb0a7ba 100755 --- a/nosql.js +++ b/nosql.js @@ -567,7 +567,7 @@ function Table(name, filename) { t.counter = new Counter(t); t.$meta(); - var schema = CONF['table.' + name]; + var schema = CONF['table_' + name] || CONF['table.' + name]; Fs.createReadStream(t.filename, { end: 1200 }).once('data', function(chunk) { From efc113c58ff2ca6d9f0fed70d6932a2004775a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 23 Oct 2018 20:01:52 +0200 Subject: [PATCH 0714/1669] Fixed `repeat` mode in `SCHEDULE()`. --- changes.txt | 1 + index.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 752c7f1cf..a07cb7468 100755 --- a/changes.txt +++ b/changes.txt @@ -48,6 +48,7 @@ - fixed: compressing CSS with `\t` tabs - fixed: `controller.autoclear()` - fixed: `controller.proxy()` +- fixed: `repeat` mode in `SCHEDULE()` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/index.js b/index.js index 5fffc760f..3776b2f9d 100755 --- a/index.js +++ b/index.js @@ -1476,7 +1476,7 @@ F.schedule = function(date, repeat, fn) { if (type === 'string') { date = date.parseDate(); - repeat && date < NOW && (date = NOW.add(repeat)); + repeat && date < NOW && (date = date.add(repeat)); } else if (type === 'number') date = new Date(date); @@ -1492,6 +1492,7 @@ F.clearSchedule = function(id) { return F; }; + /** * Auto resize picture according the path * @param {String} url Relative path. From ad273be75a3c5a42a5714d99421a3d3e4398361a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 23 Oct 2018 20:29:50 +0200 Subject: [PATCH 0715/1669] Fixed filtering in NoSQL. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index ffbb0a7ba..f0ad598d0 100755 --- a/nosql.js +++ b/nosql.js @@ -6573,6 +6573,7 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { } else { var item = {}; item.scalarcount = 0; + item.all = 0; item.count = 0; item.counter = 0; item.builder = builder; @@ -6586,7 +6587,6 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { NoSQLReader.prototype.compare2 = function(docs, custom, done) { var self = this; - for (var i = 0; i < docs.length; i++) { var doc = docs[i]; @@ -6605,7 +6605,7 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { if (item.canceled) continue; - var output = item.compare(doc, item.filter, j); + var output = item.compare(doc, item.filter, item.all++); if (!output) continue; @@ -6647,7 +6647,7 @@ NoSQLReader.prototype.compare = function(docs) { if (item.canceled) continue; - var output = item.compare(doc, item.filter, j); + var output = item.compare(doc, item.filter, item.all++); if (!output) continue; From c25093e3a4fad7b7fdb52c5d75db7c551ea05147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 23 Oct 2018 20:40:34 +0200 Subject: [PATCH 0716/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ba55e40af..b47557352 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-20", + "version": "3.0.1-21", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 560f7d3f0f298f9a3f5f526d27c47fe068b144e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 09:44:39 +0200 Subject: [PATCH 0717/1669] Fixed `SCHEDULE()` and `UTC()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 3776b2f9d..2f97992a2 100755 --- a/index.js +++ b/index.js @@ -1475,7 +1475,7 @@ F.schedule = function(date, repeat, fn) { var type = typeof(date); if (type === 'string') { - date = date.parseDate(); + date = date.parseDate().toUTC(); repeat && date < NOW && (date = date.add(repeat)); } else if (type === 'number') date = new Date(date); From 03fdcd9482ee2d9ef10cd64088beeec8de0c5a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 15:18:36 +0200 Subject: [PATCH 0718/1669] Fixed saving data of `NoSQL.Counter`. --- nosql.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nosql.js b/nosql.js index f0ad598d0..fe5a562bd 100755 --- a/nosql.js +++ b/nosql.js @@ -3061,6 +3061,13 @@ function Counter(db) { t.key = (db instanceof Table ? 'table' : 'nosql') + db.name.hash(); t.type = 0; // 1 === saving, 2 === reading t.$events = {}; + t.$cb_save = function() { + t.tid = undefined; + if (F.isCluster) + clusterlock(t, '$save'); + else + t.$save(); + }; } const CP = Counter.prototype; @@ -3161,7 +3168,7 @@ CP.min = function(id, count) { arr[1] = count; } - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); + self.save(); this.$events.min && self.emit('min', id, count || 1); return self; }; @@ -3187,8 +3194,8 @@ CP.max = function(id, count) { arr[1] = count; } - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); - this.$events.max && self.emit('max', id, count || 1); + self.save(); + self.$events.max && self.emit('max', id, count || 1); return self; }; @@ -3208,7 +3215,7 @@ CP.inc = CP.hit = function(id, count) { else self.cache[key] += count || 1; - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); + self.save(); this.$events.sum && self.emit('sum', id, count || 1); this.$events.hits && self.emit('hit', id, count || 1); return self; @@ -3224,7 +3231,7 @@ CP.remove = function(id) { else self.cache[id] = null; - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); + self.save(); self.emit('remove', id); return self; }; @@ -3987,16 +3994,15 @@ function counter_parse_days_all(output, value, year, opt) { CP.save = function() { var self = this; - if (F.isCluster) - clusterlock(self, '$save'); - else - self.$save(); + !self.tid && (self.tid = setTimeout(self.$cb_save, self.TIMEOUT)); return self; }; CP.$save = function() { var self = this; + + self.tid && clearTimeout(self.tid); self.db.readonly && self.db.throwReadonly(); if (self.type) { From 03b8bf12c0f83713fe329d0fc238932c122385a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 15:18:53 +0200 Subject: [PATCH 0719/1669] Fixed `setTimeout2()` with a limit. --- index.js | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index 2f97992a2..cc37797ff 100755 --- a/index.js +++ b/index.js @@ -6975,6 +6975,11 @@ F.service = function(count) { releasegc = true; CONF.allow_debug && F.consoledebug('clear temporary cache'); + + keys = Object.keys(F.temporary.internal); + for (var i = 0; i < keys.length; i++) + if (!F.temporary.internal[keys[i]]) + delete F.temporary.internal[keys[i]]; } // every 61 minutes (default) services precompile all (installed) views @@ -16752,20 +16757,35 @@ function async_middleware(index, req, res, middleware, callback, options, contro global.setTimeout2 = function(name, fn, timeout, limit, param) { var key = ':' + name; + var internal = F.temporary.internal; + if (limit > 0) { - var key2 = key + ':limit'; - if (F.temporary.internal[key2] >= limit) + + var key2 = key + '_limit'; + var key3 = key + '_fn'; + + if (internal[key2] >= limit) { + internal[key] && clearTimeout(internal[key]); + internal[key] = internal[key2] = internal[key3] = undefined; + fn(); return; - F.temporary.internal[key2] = (F.temporary.internal[key2] || 0) + 1; - F.temporary.internal[key] && clearTimeout(F.temporary.internal[key]); - return F.temporary.internal[key] = setTimeout(function(param) { - F.temporary.internal[key2] = undefined; + } + + internal[key] && clearTimeout(internal[key]); + internal[key2] = (internal[key2] || 0) + 1; + + return internal[key] = setTimeout(function(param, key) { + F.temporary.internal[key] = F.temporary.internal[key + '_limit'] = F.temporary.internal[key + '_fn'] = undefined; fn && fn(param); - }, timeout, param); + }, timeout, param, key); + } + + if (internal[key]) { + clearTimeout(internal[key]); + internal[key] = undefined; } - F.temporary.internal[key] && clearTimeout(F.temporary.internal[key]); - return F.temporary.internal[key] = setTimeout(fn, timeout, param); + return internal[key] = setTimeout(fn, timeout, param); }; global.clearTimeout2 = function(name) { From 32f0299a1d3bb1a809fb7aecbea10fe483f4b90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 15:51:41 +0200 Subject: [PATCH 0720/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b47557352..b9cace217 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-21", + "version": "3.0.1-22", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4b64e02fb699c161f1d1ff251111089006fd6d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 20:34:04 +0200 Subject: [PATCH 0721/1669] Added new fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index a07cb7468..65d129162 100755 --- a/changes.txt +++ b/changes.txt @@ -29,6 +29,7 @@ - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator +- fixed: a critical bug with NoSQL counter and freezing app - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories From b9cda59ec18234be37fd2fe2425334df21b4e235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 20:46:29 +0200 Subject: [PATCH 0722/1669] Added new fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 65d129162..fd543d663 100755 --- a/changes.txt +++ b/changes.txt @@ -30,6 +30,7 @@ - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator - fixed: a critical bug with NoSQL counter and freezing app +- fixed: a critical bug with rendering of multiple async components - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories From 1d700fdf8985842a5b7f27ab91ca1e4b2b35b8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Oct 2018 08:51:49 +0100 Subject: [PATCH 0723/1669] Improved `String.toUnicode()` and `String.fromUnicode()`. --- utils.js | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/utils.js b/utils.js index e1c2d0ea9..b6b7a9d3a 100755 --- a/utils.js +++ b/utils.js @@ -3763,22 +3763,28 @@ SP.capitalize = function(first) { return builder; }; - -SP.toUnicode = function() { - var result = ''; - var self = this; - var length = self.length; - for(var i = 0; i < length; ++i){ - if(self.charCodeAt(i) > 126 || self.charCodeAt(i) < 32) - result += '\\u' + self.charCodeAt(i).hex(4); +String.prototype.toUnicode = function() { + var output = ''; + for (var i = 0; i < this.length; i++) { + var c = this[i].charCodeAt(0); + if(c > 126 || c < 32) + output += '\\u' + ('000' + c.toString(16)).substr(-4); else - result += self[i]; + output += this[i]; } - return result; + return output; }; -SP.fromUnicode = function() { - return unescape(this.replace(regexpUNICODE, (match, v) => String.fromCharCode(parseInt(v, 16)))); +String.prototype.fromUnicode = function() { + var output = ''; + for (var i = 0; i < this.length; i++) { + if (this[i] === '\\' && this[i + 1] === 'u') { + output += String.fromCharCode(parseInt(this[i + 2] + this[i + 3] + this[i + 4] + this[i + 5], 16)); + i += 5; + } else + output += this[i]; + } + return output; }; SP.sha1 = function(salt) { From b1a2b76f83da8b570a4747af9b9edfc8ef908b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Oct 2018 17:28:06 +0100 Subject: [PATCH 0724/1669] Improved `TABLE()`. --- nosql.js | 221 ++++++++++++++++++++++++------- nosqlstream.js | 343 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 425 insertions(+), 139 deletions(-) diff --git a/nosql.js b/nosql.js index fe5a562bd..bf414d818 100755 --- a/nosql.js +++ b/nosql.js @@ -577,17 +577,22 @@ function Table(name, filename) { } t.parseSchema(chunk.toString('utf8').split('\n', 1)[0].split('|')); - t.ready = true; - t.next(0); - if (schema && t.stringifySchema() !== schema) + if (schema && t.stringifySchema() !== schema) { + t.$header = Buffer.byteLength(t.stringifySchema()) + 1; t.extend(schema); + } else + t.$header = Buffer.byteLength(schema ? schema : t.stringifySchema()) + 1; + + t.next(0); }).on('error', function(e) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); - Fs.writeFileSync(t.filename, t.stringifySchema() + NEWLINE, 'utf8'); + var bschema = t.stringifySchema(); + t.$header = Buffer.byteLength(bschema) + 1; + Fs.writeFileSync(t.filename, bschema + NEWLINE, 'utf8'); t.ready = true; t.next(0); } else { @@ -1146,8 +1151,11 @@ DP.find2 = function(builder) { var self = this; if (builder instanceof DatabaseBuilder) builder.db = self; - else + else { builder = new DatabaseBuilder(self); + builder.$options.notall = true; + } + if (self.readonly) return self.find(builder); self.pending_reader2.push(builder); @@ -5535,8 +5543,11 @@ TP.find2 = function(builder) { var self = this; if (builder) builder.db = self; - else + else { builder = new DatabaseBuilder(self); + builder.$options.notall = true; + } + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; @@ -5563,13 +5574,14 @@ TP.extend = function(schema, callback) { var olds = self.$schema; var oldk = self.$keys; + var oldl = self.$size; + var oldh = Buffer.byteLength(self.stringifySchema() + NEWLINE); self.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); var meta = self.stringifySchema() + NEWLINE; var news = self.$schema; var newk = self.$keys; - self.$schema = olds; self.$keys = oldk; @@ -5594,8 +5606,14 @@ TP.extend = function(schema, callback) { }); data.keys = self.$keys; + fs.start = oldh; fs.divider = '\n'; + if (oldl) + self.linesize = oldl; + + var size = self.$size; + fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); @@ -5603,8 +5621,9 @@ TP.extend = function(schema, callback) { self.$schema = olds; self.$keys = oldk; + self.$size = oldl; - for (var a = count ? 0 : 1; a < lines.length; a++) { + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = count++; var doc = self.parseData(data); @@ -5614,6 +5633,7 @@ TP.extend = function(schema, callback) { self.$schema = news; self.$keys = newk; + self.$size = size; var buffer = ''; for (var i = 0; i < items.length; i++) buffer += self.stringify(items[i], true) + NEWLINE; @@ -5623,6 +5643,7 @@ TP.extend = function(schema, callback) { fs.$callback = function() { self.$schema = news; self.$keys = newk; + self.$header = Buffer.byteLength(meta); writer.end(); fs = null; }; @@ -5816,7 +5837,11 @@ TP.$reader = function() { var data = {}; var indexer = 0; + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; + data.keys = self.$keys; if (self.buffersize) @@ -5827,16 +5852,16 @@ TP.$reader = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; - for (var j = indexer ? 0 : 1; j < lines.length; j++) { + for (var j = 0; j < lines.length; j++) { data.line = lines[j].split('|'); data.index = indexer++; arr.push(self.parseData(data)); } - return filters.compare(arr, jsonparser); + return filters.compare(arr); }; fs.$callback = function() { @@ -5868,6 +5893,9 @@ TP.$reader3 = function() { var data = {}; var indexer = 0; + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; data.keys = self.$keys; @@ -5879,18 +5907,18 @@ TP.$reader3 = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; for (var j = 0; j < lines.length; j++) { data.line = lines[j].split('|'); - if (!TABLERECORD[data.line[0]]) - continue; - data.index = indexer++; - arr.push(self.parseData(data)); + if (TABLERECORD[data.line[0]]) { + data.index = indexer++; + arr.push(self.parseData(data)); + } } - return filters.compare(arr, jsonparser); + return filters.compare(arr); }; fs.$callback = function() { @@ -5926,6 +5954,9 @@ TP.$update = function() { for (var i = 0; i < filter.length; i++) filters.add(filter[i].builder, true); + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; if (self.buffersize) @@ -5937,7 +5968,6 @@ TP.$update = function() { var update = function(docs, doc, dindex, f, findex) { var rec = fs.docsbuffer[dindex]; - var fil = filter[findex]; var e = fil.keys ? 'modify' : 'update'; var old = self.$events[e] ? CLONE(doc) : 0; @@ -5987,6 +6017,7 @@ TP.$update = function() { var rec = fs.docsbuffer[dindex]; var upd = self.stringify(doc, null, rec.length); + if (upd === rec.doc) return; @@ -6004,13 +6035,10 @@ TP.$update = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; - if (!indexer) - arr.push(EMPTYOBJECT); - - for (var a = indexer ? 0 : 1; a < lines.length; a++) { + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.length = lines[a].length; data.index = indexer++; @@ -6069,6 +6097,9 @@ TP.$remove = function() { var change = false; var indexer = 0; + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; if (self.buffersize) @@ -6094,13 +6125,13 @@ TP.$remove = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; if (!indexer) arr.push(EMPTYOBJECT); - for (var a = indexer ? 0 : 1; a < lines.length; a++) { + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = indexer++; arr.push(self.parseData(data)); @@ -6143,6 +6174,8 @@ TP.$clean = function() { var fs = new NoSQLStream(self.filename); var writer = Fs.createWriteStream(self.filename + '-tmp'); + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = NEWLINE; if (self.buffersize) @@ -6232,6 +6265,9 @@ TP.$streamer = function() { var data = {}; data.keys = self.$keys; + + fs.array = true; + fs.start = self.$header; fs.divider = '\n'; if (self.buffersize) @@ -6241,8 +6277,8 @@ TP.$streamer = function() { fs.buffercount = self.buffercount; fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); - for (var a = count ? 0 : 1; a < lines.length; a++) { + var lines = fs.docs; + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = count++; var doc = self.parseData(data); @@ -6271,37 +6307,62 @@ TP.allocations = function(enable) { TP.parseSchema = function() { var self = this; var arr = arguments[0] instanceof Array ? arguments[0] : arguments; + var sized = true; self.$schema = {}; self.$keys = []; + self.$size = 2; for (var i = 0; i < arr.length; i++) { var arg = arr[i].split(':'); var type = 0; - switch ((arg[1] || '').toLowerCase().trim()) { + var T = (arg[1] || '').toLowerCase().trim(); + var size = 0; + + var index = T.indexOf('('); + if (index != -1) { + size = +T.substring(index + 1, T.lastIndexOf(')')); + T = T.substring(0, index); + } + + switch (T) { case 'number': type = 2; + !size && (size = 16); break; case 'boolean': case 'bool': type = 3; + size = 1; break; case 'date': type = 4; + size = 13; break; case 'object': type = 5; + size = 0; + sized = false; break; case 'string': default: type = 1; + if (!size) + sized = false; break; } var name = arg[0].trim(); - self.$schema[name] = { type: type, pos: i }; + self.$schema[name] = { type: type, pos: i, size: size }; self.$keys.push(name); + self.$size += size + 1; } + if (sized) { + self.$allocations = false; + self.$size++; // newline + } else + self.$size = 0; + return self; }; @@ -6318,8 +6379,15 @@ TP.stringifySchema = function() { switch (meta.type) { case 2: + type = 'number'; + + // string + if (self.$size && meta.size !== 16) + type += '(' + (meta.size) + ')'; + break; + case 3: type = 'boolean'; break; @@ -6329,6 +6397,11 @@ TP.stringifySchema = function() { case 5: type = 'object'; break; + default: + // string + if (meta.size) + type += '(' + (meta.size) + ')'; + break; } data.push(key + ':' + type); @@ -6362,29 +6435,43 @@ TP.parseData = function(data, cache) { continue; var pos = meta.pos + 1; + var line = data.line[pos]; + + if (self.$size) { + for (var j = line.length - 1; j > -1; j--) { + if (line[j] !== ' ') { + line = line.substring(0, j + 1); + break; + } + } + } switch (meta.type) { case 1: // String - obj[key] = data.line[pos]; + obj[key] = line; if (esc && obj[key]) obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); + if (self.$size && obj[key].indexOf('\\u') !== -1) + obj[key] = obj[key].fromUnicode(); break; case 2: // Number - val = +data.line[pos]; + val = +line; obj[key] = val < 0 || val > 0 ? val : 0; break; case 3: // Boolean - val = data.line[pos]; + val = line; obj[key] = BOOLEAN[val] == 1; break; case 4: // Date - val = data.line[pos]; + val = line; obj[key] = val ? new Date(val[10] === 'T' ? val : +val) : null; break; case 5: // Object - val = data.line[pos]; + val = line; if (esc && val) val = val.replace(REGTUNESCAPE, regtescapereverse); + if (self.$size && obj[key].indexOf('\\u') !== -1) + obj[key] = obj[key].fromUnicode(); obj[key] = val ? val.parseJSON(true) : null; break; } @@ -6408,21 +6495,67 @@ TP.stringify = function(doc, insert, byteslen) { switch (meta.type) { case 1: // String - val = val ? val : ''; - size += 4; + + if (self.$size) { + switch (typeof(val)) { + case 'number': + val = val + ''; + break; + case 'boolean': + val = val ? '1' : '0'; + break; + case 'object': + var is = !!val; + val = JSON.stringify(val); + if (!is) + val = val.toUnicode(); + break; + case 'string': + val = val.toUnicode(); + break; + } + + if (val.length > meta.size) + val = val.substring(0, meta.size); + else + val = val.padRight(meta.size, ' '); + + // bytes + var diff = meta.size - Buffer.byteLength(val); + if (diff > 0) { + for (var j = 0; j < diff; j++) + val += ' '; + } + + } else { + val = val ? val : ''; + if (meta.size && val.length > meta.sized) + val = val.substring(0, meta.size); + size += 4; + } + break; case 2: // Number - val = (val || 0); - size += 2; + val = (val || 0) + ''; + if (self.$size) { + if (val.length < meta.size) + val = val.padRight(meta.size, ' '); + } else + size += 2; break; + case 3: // Boolean val = (val == true ? '1' : '0'); break; + case 4: // Date - // val = val ? val.toISOString() : ''; val = val ? val instanceof Date ? val.getTime() : val : ''; - !val && (size += 13); + if (self.$size) + val = (val + '').padRight(meta.size, ' '); + else if (!val) + size += 10; break; + case 5: // Object val = val ? JSON.stringify(val) : ''; size += 4; @@ -6440,7 +6573,9 @@ TP.stringify = function(doc, insert, byteslen) { output += '|' + val; } - if (doc.$$alloc) { + if (self.$size && (insert || byteslen)) { + output += '|'; + } else if (doc.$$alloc) { var l = output.length; var a = doc.$$alloc; if (l <= a.length) { @@ -6665,7 +6800,7 @@ NoSQLReader.prototype.compare = function(docs) { item.counter++; - if (!b.$inlinesort && !item.done) + if (b.$options.notall && !b.$inlinesort && !item.done) item.done = b.$options.take && b.$options.take <= item.counter; if (b.$options.readertype) @@ -6729,7 +6864,7 @@ NoSQLReader.prototype.compare = function(docs) { break; } - if (item.first) { + if (item.first || item.done) { item.canceled = true; self.canceled++; } diff --git a/nosqlstream.js b/nosqlstream.js index 6665e067e..4c39f619e 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -46,6 +46,8 @@ function NoSQLStream(filename) { this.remchar = '-'; this.buffercount = BUFFERDOCS; this.buffersize = BUFFERSIZE; + this.linesize = 0; + this.start = 0; // this.canceled = false; // this.docs = ''; // this.docscount = 0; @@ -81,44 +83,84 @@ NoSQLStream.prototype.readhelpers = function() { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); - while (index !== -1) { + if (self.linesize) { - var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] === self.remchar) { - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; - continue; + while (self.buffer.length >= self.linesize) { + + var tmp = self.buffer.toString('utf8', 0, self.linesize - 1); + if (tmp[0] === self.remchar) { + self.buffer = self.buffer.slice(self.linesize); + if (self.buffer.length > self.linesize) + continue; + else + break; + } + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; + + self.docscount++; + self.indexer++; + + if (self.docscount >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = self.array ? [] : ''; + self.docscount = 0; + if (self.canceled) { + self.read(self.$callback, self.$noclose); + return; + } + } + + self.buffer = self.buffer.slice(self.linesize); } - self.docs += (self.docs ? self.divider : '') + tmp; - self.docscount++; - self.indexer++; + } else { + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index); + if (tmp[0] === self.remchar) { + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + continue; + } - if (self.docscount >= self.buffercount) { + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - if (self.ondocuments() === false) - self.canceled = true; + self.docscount++; + self.indexer++; - self.docs = ''; - self.docscount = 0; + if (self.docscount >= self.buffercount) { - if (self.canceled) { - self.read(self.$callback, self.$noclose); - return; + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = self.array ? [] : ''; + self.docscount = 0; + + if (self.canceled) { + self.read(self.$callback, self.$noclose); + return; + } } - } - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } } self.ticks++; - if (self.ticks % 5 === 0) setImmediate(self.cb_readticks); else @@ -144,38 +186,78 @@ NoSQLStream.prototype.readhelpers = function() { } else self.buffer = chunk; - var index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + if (self.linesize) { - while (index !== -1) { + while (self.buffer.length >= self.linesize) { - var tmp = self.buffer.toString('utf8', index); - if (tmp[1] === self.remchar) { - self.buffer = self.buffer.slice(0, index); - index = self.buffer.lastIndexOf(NEWLINEBUFFER); - if (index === -1) - break; - continue; - } + var tmp = self.buffer.toString('utf8', self.buffer.length - self.linesize, self.buffer.length - 1); + if (tmp[0] === self.remchar) { + self.buffer = self.buffer.slice(0, self.buffer.length - self.linesize); + if (self.buffer.length > self.linesize) + continue; + else + break; + } - self.docs += (self.docs ? self.divider : '') + tmp.trim(); - self.docscount++; + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - if (self.docscount >= self.buffercount) { - if (self.ondocuments() === false) - self.canceled = true; - self.docs = ''; - self.docscount = 0; - } + self.docscount++; + self.indexer++; - if (self.canceled) { - self.readreverse2(); - return; + if (self.docscount >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = self.array ? [] : ''; + self.docscount = 0; + if (self.canceled) { + self.read(self.$callback, self.$noclose); + return; + } + } + + self.buffer = self.buffer.slice(0, self.buffer.length - self.linesize); } - self.buffer = self.buffer.slice(0, index); - index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); - if (index === -1) - break; + } else { + var index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', index); + if (tmp[1] === self.remchar) { + self.buffer = self.buffer.slice(0, index); + index = self.buffer.lastIndexOf(NEWLINEBUFFER); + if (index === -1) + break; + continue; + } + + if (self.array) + self.docs.push(tmp.trim()); + else + self.docs += (self.docs ? self.divider : '') + tmp.trim(); + + self.docscount++; + + if (self.docscount >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = self.array ? [] : ''; + self.docscount = 0; + } + + if (self.canceled) { + self.readreverse2(); + return; + } + + self.buffer = self.buffer.slice(0, index); + index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + if (index === -1) + break; + } } self.ticks++; @@ -199,7 +281,6 @@ NoSQLStream.prototype.readhelpers = function() { self.cache[0] = self.buffer; self.cache[1] = chunk; self.buffer = Buffer.concat(self.cache); - beg = self.cache[0].length - 1; if (beg < 0) @@ -208,34 +289,68 @@ NoSQLStream.prototype.readhelpers = function() { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); - while (index !== -1) { + if (self.linesize) { + while (self.buffer.length >= self.linesize) { + var tmp = self.buffer.toString('utf8', 0, self.linesize - 1); - var tmp = self.buffer.toString('utf8', 0, index).trim(); + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - self.docs += (self.docs ? self.divider : '') + tmp; - self.docscount++; - self.indexer++; + self.docscount++; + self.indexer++; - if (self.docscount >= self.buffercount) { + if (self.docscount >= self.buffercount) { - if (self.ondocuments() === false) - self.canceled = true; + if (self.ondocuments() === false) + self.canceled = true; - self.docs = ''; - self.docscount = 0; + self.docs = self.array ? [] : ''; + self.docscount = 0; - if (self.canceled) { - self.stream.destroy && self.stream.destroy(); - return; + if (self.canceled) { + self.stream.destroy && self.stream.destroy(); + return; + } } + self.buffer = self.buffer.slice(self.linesize); } + } else { + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index).trim(); + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; + + + self.docscount++; + self.indexer++; + + if (self.docscount >= self.buffercount) { - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = self.array ? [] : ''; + self.docscount = 0; + + if (self.canceled) { + self.stream.destroy && self.stream.destroy(); + return; + } + } + + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } } }; @@ -305,6 +420,7 @@ NoSQLStream.prototype.writehelpers = function() { self.position += size; var beg = 0; + var index; if (self.buffer) { self.cache[0] = self.buffer; @@ -319,32 +435,66 @@ NoSQLStream.prototype.writehelpers = function() { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + if (self.linesize) { + while (self.buffer.length >= self.linesize) { + + var tmp = self.buffer.toString('utf8', 0, self.linesize - 1); + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - while (index !== -1) { - var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] !== self.remchar) { - self.docs += (self.docs ? self.divider : '') + tmp; - self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); + self.docsbuffer.push({ length: self.linesize - 1, doc: tmp, position: self.positionupdate }); self.docscount++; + if (self.docsbuffer.length >= self.buffercount) { if (self.ondocuments() === false) self.canceled = true; self.docsbuffer = []; - self.docs = ''; + self.docs = self.array ? [] : ''; if (self.canceled) break; } + + self.positionupdate += self.linesize; + self.buffer = self.buffer.slice(self.linesize); } - self.positionupdate += Buffer.byteLength(tmp, 'utf8') + 1; - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; + } else { + + index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + var tmp = self.buffer.toString('utf8', 0, index); + + if (tmp[0] !== self.remchar) { + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; + + self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); + self.docscount++; + if (self.docsbuffer.length >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docsbuffer = []; + self.docs = self.array ? [] : ''; + if (self.canceled) + break; + } + } + + self.positionupdate += Buffer.byteLength(tmp, 'utf8') + 1; + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } } if (self.bufferstack.length || self.bufferstacknew.length) @@ -364,7 +514,7 @@ NoSQLStream.prototype.writehelpers = function() { NoSQLStream.prototype.openread = function() { var self = this; self.type = 'r'; - self.position = 0; + self.position = self.start; self.open(); return self; }; @@ -372,7 +522,7 @@ NoSQLStream.prototype.openread = function() { NoSQLStream.prototype.openreadreverse = function() { var self = this; self.type = 'r'; - self.position = 0; + self.position = self.start; self.$reverse = true; self.open(); return self; @@ -390,7 +540,7 @@ NoSQLStream.prototype.openupdate = function() { Fs.fstat(fd, function(err, stats) { - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; if (err) { @@ -399,13 +549,13 @@ NoSQLStream.prototype.openupdate = function() { return; } - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; self.fd = fd; self.stats = stats; - self.position = 0; + self.position = self.start; self.positionappend = self.stats.size; - self.positionupdate = 0; + self.positionupdate = self.start; self.bufferstack = []; self.bufferstacknew = []; self.docsbuffer = []; @@ -426,13 +576,13 @@ NoSQLStream.prototype.openstream = function(stream) { if (self.docscount) { self.ondocuments(); self.docscount = 0; - self.docs = ''; + self.docs = self.array ? [] : ''; } self.$callback && self.$callback(); self.$callback = null; }; - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; self.readhelpers(); self.stream = stream; @@ -452,11 +602,11 @@ NoSQLStream.prototype.open = function() { } Fs.fstat(fd, function(err, stats) { - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; self.fd = fd; self.stats = stats; - self.position = 0; + self.position = self.start; if (err) { Fs.close(fd, NOOP); @@ -553,6 +703,7 @@ NoSQLStream.prototype.read = function() { var self = this; var size = self.stats.size - self.position; + if (!self.fd || size <= 0 || self.canceled) { if (!self.canceled && self.buffer && self.buffer.length) { @@ -564,7 +715,7 @@ NoSQLStream.prototype.read = function() { if (self.docscount) { self.ondocuments(); self.docscount = 0; - self.docs = ''; + self.docs = self.array ? [] : ''; } self.close(); @@ -586,7 +737,7 @@ NoSQLStream.prototype.readreverse = function() { NoSQLStream.prototype.readreverse2 = function() { var self = this; - if (!self.fd || self.position <= 0 || self.canceled) { + if (!self.fd || self.position <= self.start || self.canceled) { if (!self.canceled && self.buffer && self.buffer.length) { self.cb_readreversebuffer(null, 1, NEWLINEBUFFER); @@ -596,7 +747,7 @@ NoSQLStream.prototype.readreverse2 = function() { if (self.docscount) { self.ondocuments(); - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; } @@ -628,7 +779,7 @@ NoSQLStream.prototype.readupdate = function() { if (self.docsbuffer.length) { self.ondocuments(); self.docsbuffer = []; - self.docs = ''; + self.docs = self.array ? [] : ''; } self.flush(); From ebed53f34a52b5407077780c50ecc5330e37a7c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Oct 2018 23:42:02 +0100 Subject: [PATCH 0725/1669] Added `F.refresh()`. --- changes.txt | 1 + index.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/changes.txt b/changes.txt index fd543d663..f3e4ad3e5 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` - added: encryption of config values +- added: `F.refresh()` for refreshing of internal cache - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index cc37797ff..daa680dfe 100755 --- a/index.js +++ b/index.js @@ -1023,6 +1023,38 @@ F.dir = function(path) { directory = path; }; +F.refresh = function() { + + NOW = new Date(); + + F.$events.clear && EMIT('clear', 'temporary', F.temporary); + F.temporary.path = {}; + F.temporary.range = {}; + F.temporary.views = {}; + F.temporary.other = {}; + global.$VIEWCACHE && global.$VIEWCACHE.length && (global.$VIEWCACHE = []); + + // Clears command cache + Image.clear(); + + CONF.allow_debug && F.consoledebug('clear temporary cache'); + + var keys = Object.keys(F.temporary.internal); + for (var i = 0; i < keys.length; i++) + if (!F.temporary.internal[keys[i]]) + delete F.temporary.internal[keys[i]]; + + F.$events.clear && EMIT('clear', 'resources'); + F.resources = {}; + CONF.allow_debug && F.consoledebug('clear resources'); + + F.$events.clear && EMIT('clear', 'dns'); + U.clearDNS(); + CONF.allow_debug && F.consoledebug('clear DNS cache'); + + return F; +}; + F.prototypes = function(fn) { var proto = {}; proto.Chunker = framework_utils.Chunker.prototype; From ae4cbc1d94b5a319009748dd45062dbd367010c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 29 Oct 2018 09:37:26 +0100 Subject: [PATCH 0726/1669] Improved NoSQLReader. --- nosql.js | 119 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/nosql.js b/nosql.js index bf414d818..021875a50 100755 --- a/nosql.js +++ b/nosql.js @@ -2206,6 +2206,24 @@ function DatabaseBuilder(db) { DatabaseBuilder.prototype.promise = promise; +DatabaseBuilder.prototype.reset = function() { + var self = this; + var reader = self.$nosqlreader; + if (reader) { + for (var i = 0; i < reader.builders.length; i++) { + var item = reader.builders[i]; + if (item.builder === self) { + item.response = null; + item.scalar = null; + item.counter = 0; + item.count = 0; + item.scalarcount = 0; + } + } + } + return self; +}; + DatabaseBuilder.prototype.makefilter = function() { return { repository: this.$repository, options: this.$options, arg: this.$args, fn: this.$functions }; }; @@ -6721,6 +6739,7 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { item.compare = builder.compile(noTrimmer); item.filter = builder.makefilter(); item.first = builder.$options.first && !builder.$options.sort; + builder.$nosqlreader = self; self.builders.push(item); } return self; @@ -6872,55 +6891,81 @@ NoSQLReader.prototype.compare = function(docs) { } }; -NoSQLReader.prototype.done = function() { - +NoSQLReader.prototype.reset = function() { var self = this; for (var i = 0; i < self.builders.length; i++) { - var item = self.builders[i]; - var builder = item.builder; - var output; - var opt = builder.$options; - - if (opt.scalar || !opt.sort) { - if (opt.scalar) - output = opt.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (opt.first) - output = item.response ? item.response[0] : undefined; - else if (opt.listing) - output = listing(builder, item); - else - output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); - continue; - } + item.canceled = false; + item.response = null; + item.scalar = null; + item.counter = 0; + item.count = 0; + item.scalarcount = 0; + } + self.canceled = 0; + return self; +}; - if (item.count) { - if (opt.sort.name) { - if (!builder.$inlinesort || opt.take !== item.response.length) - item.response.quicksort(opt.sort.name, opt.sort.asc); - } else if (opt.sort === null) - item.response.random(); - else - item.response.sort(opt.sort); - - if (opt.skip && opt.take) - item.response = item.response.splice(opt.skip, opt.take); - else if (opt.skip) - item.response = item.response.splice(opt.skip); - else if (!builder.$inlinesort && opt.take) - item.response = item.response.splice(0, opt.take); - } +NoSQLReader.prototype.callback = function(item) { + + var self = this; + var builder = item.builder; + var output; + var opt = builder.$options; - if (opt.first) + if (item.canceled) { + item.canceled = false; + if (self.canceled) + self.canceled--; + } + + if (opt.scalar || !opt.sort) { + if (opt.scalar) + output = opt.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (opt.first) output = item.response ? item.response[0] : undefined; else if (opt.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + return self; + } + + if (item.count) { + if (opt.sort.name) { + if (!builder.$inlinesort || opt.take !== item.response.length) + item.response.quicksort(opt.sort.name, opt.sort.asc); + } else if (opt.sort === null) + item.response.random(); + else + item.response.sort(opt.sort); + + if (opt.skip && opt.take) + item.response = item.response.splice(opt.skip, opt.take); + else if (opt.skip) + item.response = item.response.splice(opt.skip); + else if (!builder.$inlinesort && opt.take) + item.response = item.response.splice(0, opt.take); } + + if (opt.first) + output = item.response ? item.response[0] : undefined; + else if (opt.listing) + output = listing(builder, item); + else + output = item.response || []; + + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + return self; +}; + +NoSQLReader.prototype.done = function() { + var self = this; + for (var i = 0; i < self.builders.length; i++) + self.callback(self.builders[i]); + self.canceled = 0; + return self; }; exports.NoSQLReader = NoSQLReader; \ No newline at end of file From f32ab1a71cb112be9af6b513e4b73aacd30847be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 29 Oct 2018 09:43:18 +0100 Subject: [PATCH 0727/1669] Improved DataReader. --- utils.js | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/utils.js b/utils.js index b6b7a9d3a..a2131098f 100755 --- a/utils.js +++ b/utils.js @@ -6166,19 +6166,15 @@ exports.Callback = function(count, callback) { function Reader() {} const RP = Reader.prototype; -RP.clear = function() { - +RP.done = function() { var self = this; - var builders = self.reader.builders; - - for (var i = 0; i < builders.length; i++) { - var builder = builders[i]; - builder.scalarcount = 0; - builder.count = 0; - builder.counter = 0; - builder.response = null; - } + self.reader.done(); + return self; +}; +RP.reset = function() { + var self = this; + self.reader.reset(); return self; }; From eaf8961776db1563c2d26050946354cf70a884ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 08:53:56 +0100 Subject: [PATCH 0728/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b9cace217..2cbd58c99 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-22", + "version": "3.0.1-23", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 71ef13ad510cdb61aacc306252ad7085176f87e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 16:31:41 +0100 Subject: [PATCH 0729/1669] Added `DatabaseBuilder.each()`. --- nosql.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 021875a50..aee23a65b 100755 --- a/nosql.js +++ b/nosql.js @@ -3075,7 +3075,23 @@ DatabaseBuilder.prototype.prepare = function(fn) { !self.$scope && self.$code.push('if(!$is)return;'); } - return this; + return self; +}; + +DatabaseBuilder.prototype.each = function(fn) { + var self = this; + + if (!self.$functions) + self.$functions = []; + + var index = self.$functions.push(fn) - 1; + + if (!self.$iscache) { + var code = '$tmp=fn[{0}].call($F,doc,index,repository);'.format(index); + self.$code.push(code); + } + + return self; }; function Counter(db) { From a7dad6900517c331f680e06f4ba4ad67ba6c4ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 16:31:52 +0100 Subject: [PATCH 0730/1669] Added `CONF.bundling`. --- bundles.js | 15 ++++++++++++++- changes.txt | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/bundles.js b/bundles.js index 0263dc3ec..c900a0431 100755 --- a/bundles.js +++ b/bundles.js @@ -3,16 +3,17 @@ require('./index'); const Fs = require('fs'); const Path = require('path'); const CONSOLE = process.argv.indexOf('restart') === -1; -const META = {}; const INTERNAL = { '/sitemap': 1, '/versions': 1, '/workflows': 1, '/dependencies': 1, '/config': 1, '/config-release': 1, '/config-debug': 1 }; const isWindows = require('os').platform().substring(0, 3).toLowerCase() === 'win'; const REGAPPEND = /\/--[a-z0-9]+/i; +const META = {}; META.version = 1; META.created = new Date(); META.total = 'v' + F.version_header; META.node = F.version_node; META.files = []; +META.skip = false; META.directories = []; META.ignore = () => true; @@ -46,6 +47,11 @@ exports.make = function(callback) { async.push(cleanFiles); + async.push(function(next) { + META.skip && (async.length = 0); + next(); + }); + async.push(function(next) { var target = F.path.root(CONF.directory_src); U.ls(F.path.root(CONF.directory_bundles), function(files) { @@ -208,6 +214,13 @@ function cleanFiles(callback) { try { meta = U.parseJSON(Fs.readFileSync(Path.join(path, 'bundle.json')).toString('utf8'), true) || {}; + + if (CONF.bundling === 'shallow') { + META.skip = true; + callback(); + return; + } + } catch (e) { meta = {}; } diff --git a/changes.txt b/changes.txt index f3e4ad3e5..8492563e7 100755 --- a/changes.txt +++ b/changes.txt @@ -17,10 +17,12 @@ - added: a new config item `default-cors : https://www.totaljs.com, https://www.componentator.com` which allows originators for `CORS()` method - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations +- added: a new config item `bundling : shallow` which enables shallow bundling (if `bundle.json` exists then the bundles won't be extracted) - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` - added: encryption of config values - added: `F.refresh()` for refreshing of internal cache +- added: `DatabaseBuilder.each(fn)` for browsing of evaluated records - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) From aab692b84c8e55f07490e4eee5dfad463d02a63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 17:41:41 +0100 Subject: [PATCH 0731/1669] Fixed NoSQL `repository` in the `callback`. --- nosql.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index aee23a65b..c817b858f 100755 --- a/nosql.js +++ b/nosql.js @@ -2940,6 +2940,7 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { if (cache) { self.$mappers = cache.mitems; self.$mappersexec = cache.mexec; + self.$each = cache.each; return cache.filter; } } @@ -2956,10 +2957,14 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$mappersexec = new Function('doc', 'item', tmp); } + if (opt.each) + self.$each = new Function('item', 'doc', 'repository', 'R', opt.each.join('')); + var cache = {}; cache.filter = new Function('doc', '$F', 'index', code); cache.mexec = self.$mappersexec; cache.mitems = self.$mappers; + cache.each = self.$each; CACHE[key] = cache; return cache.filter; }; @@ -3087,8 +3092,11 @@ DatabaseBuilder.prototype.each = function(fn) { var index = self.$functions.push(fn) - 1; if (!self.$iscache) { - var code = '$tmp=fn[{0}].call($F,doc,index,repository);'.format(index); - self.$code.push(code); + var code = 'item.filter.fn[{0}].call(item,doc,item.filter.repository,item.filter.repository);'.format(index); + if (self.$options.each) + self.$options.each.push(code); + else + self.$options.each = [code]; } return self; @@ -6841,6 +6849,7 @@ NoSQLReader.prototype.compare = function(docs) { if (b.$options.readertype) continue; + b.$each && b.$each(item, output); b.$mappersexec && b.$mappersexec(output, item); var val; @@ -6944,7 +6953,7 @@ NoSQLReader.prototype.callback = function(item) { output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); return self; } @@ -6972,7 +6981,7 @@ NoSQLReader.prototype.callback = function(item) { else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); return self; }; From 6dee6487809381fbb8af7ac0880b7797299c5ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 20:31:04 +0100 Subject: [PATCH 0732/1669] Fixed bugs in `repository`. --- nosql.js | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/nosql.js b/nosql.js index c817b858f..028283f0b 100755 --- a/nosql.js +++ b/nosql.js @@ -1585,15 +1585,15 @@ DP.$update = function() { for (var i = 0; i < filters.builders.length; i++) { var item = filters.builders[i]; var fil = filter[i]; - if (fil.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + if (fil.insert && !item.counter) { + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.filter.repository || EMPTYOBJECT); var tmp = self.insert(fil.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository); } } @@ -1688,15 +1688,15 @@ DP.$update_inmemory = function() { for (var i = 0; i < filter.length; i++) { var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); + if (item.insert && !item.counter) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.filter.repository || EMPTYOBJECT); var tmp = self.insert(item.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository); } } @@ -6100,14 +6100,14 @@ TP.$update = function() { var item = filters.builders[i]; var fil = filter[i]; if (fil.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.filter.repository); var tmp = self.insert(fil.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository); } } @@ -6170,9 +6170,6 @@ TP.$remove = function() { var lines = fs.docs; var arr = []; - if (!indexer) - arr.push(EMPTYOBJECT); - for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = indexer++; @@ -6771,6 +6768,7 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { NoSQLReader.prototype.compare2 = function(docs, custom, done) { var self = this; + for (var i = 0; i < docs.length; i++) { var doc = docs[i]; @@ -6793,9 +6791,19 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { if (!output) continue; - item.is = false; - !is && (is = true); + // WTF? + // item.is = false; + item.count++; + + if ((item.builder.$options.skip && item.builder.$options.skip >= item.count) || (item.builder.$options.take && item.builder.$options.take <= item.counter)) + continue; + + !is && (is = true); + + item.counter++; + item.builder.$each && item.builder.$each(item, output); + var canceled = item.canceled; var c = custom(docs, output, i, item, j); @@ -6953,7 +6961,7 @@ NoSQLReader.prototype.callback = function(item) { output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.counter : output, item.count, item.filter.repository); return self; } @@ -6981,7 +6989,7 @@ NoSQLReader.prototype.callback = function(item) { else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.counter : output, item.count, item.filter.repository); return self; }; From 09005b567f455a9dade65ece60355f3e4bba0696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 21:20:04 +0100 Subject: [PATCH 0733/1669] Fixed `TABLE` cleaning. --- nosql.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 028283f0b..1aa23d547 100755 --- a/nosql.js +++ b/nosql.js @@ -6213,6 +6213,8 @@ TP.$clean = function() { var fs = new NoSQLStream(self.filename); var writer = Fs.createWriteStream(self.filename + '-tmp'); + writer.write(self.stringifySchema() + NEWLINE); + fs.start = self.$header; fs.linesize = self.$size; fs.divider = NEWLINE; @@ -6456,10 +6458,8 @@ TP.parseData = function(data, cache) { var esc = data.line[0] === '*'; var val, alloc; - if (cache && data.keys.length === data.line.length - 2) { - // alloc = data.line[data.line.length - 1].length - 1; + if (cache && !self.$size && data.keys.length === data.line.length - 2) alloc = data.line[data.line.length - 1].length; - } for (var i = 0; i < data.keys.length; i++) { var key = data.keys[i]; @@ -6802,7 +6802,7 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { !is && (is = true); item.counter++; - item.builder.$each && item.builder.$each(item, output); + item.builder.$each && item.builder.$each(item, doc); var canceled = item.canceled; var c = custom(docs, output, i, item, j); From f44d4e67d2f7ce7b8fbbe3be9623901df5358b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 21:20:52 +0100 Subject: [PATCH 0734/1669] Updated beta. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2cbd58c99..237b9a13e 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-23", + "version": "3.0.1-24", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 89cdc9e9f456446cd6492d7b07164e9440f935a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 15:45:00 +0100 Subject: [PATCH 0735/1669] Fixed `versions` with `auto` value. --- index.js | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index daa680dfe..3117672ec 100755 --- a/index.js +++ b/index.js @@ -2587,6 +2587,8 @@ F.routing = function(name) { */ global.MERGE = F.merge = function(url) { + F.temporary.other['merge_' + url] = 1; + if (url[0] === '#') url = sitemapurl(url.substring(1)); @@ -5244,6 +5246,9 @@ F.onMapping = function(url, def, ispublic, encode) { return F.components.files[name] && F.components.files[name][tmp.substring(index + 1)] ? (F.path.temp() + tmp.substring(1)) : null; } + if (F.routes.mapping[url]) + return F.routes.mapping[url]; + if (F._length_themes) { var index = tmp.indexOf('/', 2); if (index !== -1) { @@ -5253,9 +5258,6 @@ F.onMapping = function(url, def, ispublic, encode) { } } - if (F.routes.mapping[url]) - return F.routes.mapping[url]; - def = framework_internal.preparePath(def, true); if (encode) @@ -8591,7 +8593,22 @@ F.$configure_versions = function(arr, clean) { if (hash) { var index = key.lastIndexOf('.'); filename = key.substring(0, index) + '-' + hash + key.substring(index); + F.versions[key] = filename; + + if (!F.routes.merge[key] && !F.temporary.other['merge_' + key]) { + var index = key.indexOf('/', 1); + var theme = index === -1 ? null : key.substring(1, index); + if (theme) { + if (F.themes[theme]) + key = F.themes[theme] + 'public' + key.substring(index); + else + key = F.path.public(key); + } else + key = F.path.public(key); + F.map(filename, key); + } + F.temporary.views = {}; F.temporary.other = {}; global.$VIEWCACHE = []; From 8b9a4ae49d50b062305b9863426708730cb5ef16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 15:47:14 +0100 Subject: [PATCH 0736/1669] Updated beta version. --- changes.txt | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 8492563e7..191ade48d 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,7 @@ - fixed: `LOCALIZE()` for nested directories - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) - fixed: `versions` and `auto` feature with enabled `F.wait()` +- fixed: `versions` and `auto` feature with direct link to file - fixed: `LOAD('release')` a release mode - fixed: `SchemaInstance.$clean()` for nested schemas - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) diff --git a/package.json b/package.json index 237b9a13e..f06817e1c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-24", + "version": "3.0.1-25", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 59dbcc4b71db1d8be1dac42194f250a0f426d39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 18:14:47 +0100 Subject: [PATCH 0737/1669] Added bundle downloading. --- bundles.js | 4 ++++ changes.txt | 1 + index.js | 47 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/bundles.js b/bundles.js index c900a0431..f78443399 100755 --- a/bundles.js +++ b/bundles.js @@ -57,6 +57,10 @@ exports.make = function(callback) { U.ls(F.path.root(CONF.directory_bundles), function(files) { var dirs = {}; files.wait(function(filename, resume) { + + if (!filename.endsWith('.bundle')) + return resume(); + var dbpath = CONF.directory_databases; F.restore(filename, target, resume, function(p, dir) { diff --git a/changes.txt b/changes.txt index 191ade48d..0aa5a3116 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - added: encryption of config values - added: `F.refresh()` for refreshing of internal cache - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records +- added: Bundles can be downloaded from URL address - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index 3117672ec..d1ac27c1f 100755 --- a/index.js +++ b/index.js @@ -3574,15 +3574,54 @@ F.modify = function(fn) { F.$bundle = function(callback) { + var bundledir = F.path.root(CONF.directory_bundles); + var makebundle = function() { - require('./bundles').make(function() { - F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); - callback(); + + var arr = Fs.readdirSync(bundledir); + var url = []; + + for (var i = 0; i < arr.length; i++) { + if (arr[i].endsWith('.url')) + url.push(arr[i]); + } + + url.wait(function(item, next) { + + var filename = F.path.root(CONF.directory_bundles) + item.replace('.url', '.bundle'); + var link = Fs.readFileSync(F.path.root(CONF.directory_bundles) + item).toString('utf8'); + + F.consoledebug('Download bundle: ' + link); + + U.download(link, FLAGS_INSTALL, function(err, response) { + + if (err) { + F.error(err, 'Bundle: ' + link); + next(); + return; + } + + var stream = Fs.createWriteStream(filename); + + response.pipe(stream); + response.on('error', function(err) { + F.error(err, 'Bundle: ' + link); + next(); + }); + + CLEANUP(stream, next); + }); + + }, function() { + require('./bundles').make(function() { + F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); + callback(); + }); }); }; try { - Fs.statSync(F.path.root(CONF.directory_bundles)); + Fs.statSync(bundledir); if (F.$bundling) { makebundle(); return; From 0607bcb821fc81a38f425133656a244d02d3ca84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 20:59:09 +0100 Subject: [PATCH 0738/1669] Fix grammar. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 0aa5a3116..f1e84a94d 100755 --- a/changes.txt +++ b/changes.txt @@ -23,7 +23,7 @@ - added: encryption of config values - added: `F.refresh()` for refreshing of internal cache - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records -- added: Bundles can be downloaded from URL address +- added: Bundles can be downloaded from URL addresses - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) From aa232192549d1a6f2ce0efe3d6282687885cf7b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 09:08:40 +0100 Subject: [PATCH 0739/1669] Fixed NoSQL in-memory. --- nosql.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 1aa23d547..598191604 100755 --- a/nosql.js +++ b/nosql.js @@ -1686,8 +1686,8 @@ DP.$update_inmemory = function() { change && self.$save(); - for (var i = 0; i < filter.length; i++) { - var item = filter[i]; + for (var i = 0; i < filters.builders.length; i++) { + var item = filters.builders[i]; if (item.insert && !item.counter) { item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.filter.repository || EMPTYOBJECT); var tmp = self.insert(item.insert); @@ -6961,6 +6961,7 @@ NoSQLReader.prototype.callback = function(item) { output = listing(builder, item); else output = item.response || []; + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.counter : output, item.count, item.filter.repository); return self; } From 657e5c3fa52e656162f516d57abcaa9187f52ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 12:37:57 +0100 Subject: [PATCH 0740/1669] Fixed bundles. --- debug.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debug.js b/debug.js index dd83e6d97..12b8da8e9 100644 --- a/debug.js +++ b/debug.js @@ -258,6 +258,14 @@ function runwatching() { var ticks = stat.mtime.getTime(); if (files[filename] != null && files[filename] !== ticks) { + + if (filename.endsWith('.bundle') && files[filename.replace(/\.bundle$/, '.url')]) { + // Bundle from URL address + files[filename] = ticks; + next(); + return; + } + var log = stamp.replace('#', files[filename] === 0 ? 'ADD' : 'UPD') + prefix + normalize(filename.replace(directory, '')); if (files[filename]) { var tmp = isViewPublic(filename); From 9df8c5b475c3ab5ebb33ed530b71c03918777ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 13:12:13 +0100 Subject: [PATCH 0741/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f06817e1c..5b7b9a641 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-25", + "version": "3.0.1-26", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From ea694da798cedef9d0e9de000093cc4d0c2e6efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 20:12:59 +0100 Subject: [PATCH 0742/1669] Updated joins. --- changes.txt | 3 ++- nosql.js | 44 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index f1e84a94d..da63a84b1 100755 --- a/changes.txt +++ b/changes.txt @@ -29,7 +29,8 @@ - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` -- updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*`. +- updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` +- updated: NoSQL joins support array values - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/nosql.js b/nosql.js index 598191604..aacfcd67a 100755 --- a/nosql.js +++ b/nosql.js @@ -2262,8 +2262,15 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { for (var i = 0; i < response.length; i++) { var item = response[i]; var val = item[join.b]; - if (val !== undefined && unique.indexOf(val) === -1) - unique.push(val); + if (val !== undefined) { + if (val instanceof Array) { + for (var j = 0; j < val.length; j++) { + if (unique.indexOf(val[j]) === -1) + unique.push(val[j]); + } + } else if (unique.indexOf(val) === -1) + unique.push(val); + } } } else if (response) { var val = response[join.b]; @@ -2341,8 +2348,8 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository self.$callbackjoin(function() { var keys = Object.keys(self.$join); - var jl = keys.length; + if (response instanceof Array) { for (var i = 0, length = response.length; i < length; i++) { var item = response[i]; @@ -2368,23 +2375,44 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository function findItem(items, field, value) { for (var i = 0, length = items.length; i < length; i++) { - if (items[i][field] === value) + if (value instanceof Array) { + for (var j = 0; j < value.length; j++) { + if (items[i][field] === value[j]) + return items[i]; + } + } else if (items[i][field] === value) return items[i]; } } function findScalar(items, value) { + var sum = null; for (var i = 0, length = items.length; i < length; i++) { - if (items[i].id === value) - return items[i].response || null; + var item = items[i]; + if (value instanceof Array) { + for (var j = 0; j < value.length; j++) { + if (item.id === value[j]) { + sum = sum == null ? item.response : (sum + item.response); + break; + } + } + } else if (item.id === value) + sum = sum == null ? item.response : (sum + item.response); } - return null; + return sum; } function findItems(items, field, value) { var arr = []; for (var i = 0, length = items.length; i < length; i++) { - if (items[i][field] === value) + if (value instanceof Array) { + for (var j = 0; j < value.length; j++) { + if (items[i][field] === value[j]) { + arr.push(items[i]); + break; + } + } + } else if (items[i][field] === value) arr.push(items[i]); } return arr; From 694a4ee8d16b3228af87515dd983fe185f2ed13b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Nov 2018 15:59:45 +0100 Subject: [PATCH 0743/1669] Added `ONCE()`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index da63a84b1..595390fcc 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,7 @@ - added: `F.refresh()` for refreshing of internal cache - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records - added: Bundles can be downloaded from URL addresses +- added: `ONCE()` alias to `F.once()` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index d1ac27c1f..366894b36 100755 --- a/index.js +++ b/index.js @@ -1148,7 +1148,7 @@ global.EMIT = F.emit = function(name, a, b, c, d, e, f, g) { return F; }; -F.once = function(name, fn) { +global.ONCE = F.once = function(name, fn) { fn.$once = true; return F.on(name, fn); }; From 156dfd0c656232fb304be517cbc324a3eb9e5ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Nov 2018 18:19:18 +0100 Subject: [PATCH 0744/1669] Fixed DateTime in NoSQL backups. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index aacfcd67a..72eaa611e 100755 --- a/nosql.js +++ b/nosql.js @@ -2167,7 +2167,7 @@ DatabaseBuilder2.prototype.log = function(msg, user) { var self = this; if (msg) { NOW = new Date(); - self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.toUTC().format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; @@ -2242,7 +2242,7 @@ DatabaseBuilder.prototype.log = function(msg, user) { var self = this; if (msg) { NOW = new Date(); - self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.toUTC().format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; @@ -2570,7 +2570,7 @@ DatabaseBuilder.prototype.backup = function(user) { }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, NOW.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, NOW.toUTC().format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); return this; }; From 7e6ee9cdd811b4ff9af2effa35cfbd0de2d4220e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 6 Nov 2018 20:03:48 +0100 Subject: [PATCH 0745/1669] Improved `ROUTING()` method. --- changes.txt | 1 + index.js | 59 +++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/changes.txt b/changes.txt index 595390fcc..60b8d9c3e 100755 --- a/changes.txt +++ b/changes.txt @@ -32,6 +32,7 @@ - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values +- updated: `ROUTING(id:|search, [flags])` method - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 366894b36..3a5313efb 100755 --- a/index.js +++ b/index.js @@ -311,7 +311,6 @@ global.CREATE = (group, name) => framework_builders.getschema(group, name).defau global.SCRIPT = (body, value, callback, param) => F.script(body, value, callback, param); global.SINGLETON = (name, def) => SINGLETONS[name] || (SINGLETONS[name] = (new Function('return ' + (def || '{}')))()); global.FUNCTION = (name) => F.functions[name] || NOOP; -global.ROUTING = (name) => F.routing(name); global.SCHEDULE = (date, each, fn, param) => F.schedule(date, each, fn, param); global.FINISHED = framework_internal.onFinished; global.DESTROY = framework_internal.destroyStream; @@ -1896,7 +1895,6 @@ global.GROUP = F.group = function() { * @param {Number} timeout Response timeout. * @return {Framework} */ - global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, language) { var name; @@ -1922,6 +1920,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu flags = tmp; } + var search = (typeof(url) === 'string' ? url.toLowerCase().replace(/\s{2,}/g, ' ') : '') + (flags ? (' ' + flags.where(n => typeof(n) === 'string' && n.substring(0, 2) !== '//' && n[2] !== ':').join(' ')).toLowerCase() : ''); var method = ''; var CUSTOM = typeof(url) === 'function' ? url : null; if (CUSTOM) @@ -2463,6 +2462,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var instance = new FrameworkRoute(); var r = instance.route; r.hash = hash; + r.search = search.split(' '); r.id = id; r.name = name.trim(); r.groups = flags_to_object(groups); @@ -2516,6 +2516,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.regexp = reg; r.regexpIndexer = regIndex; r.type = 'web'; + r.remove = remove_route_web; if (r.isUPLOAD) PERF.upload = true; @@ -2559,20 +2560,62 @@ function flags_to_object(flags) { return obj; } +function remove_route_web() { + + if (this.isSYSTEM) { + var keys = Object.keys(F.routes.system); + for (var i = 0; i < keys.length; i++) { + if (F.routes.system[keys[i]] === this) { + delete F.routes.system[keys]; + F.temporary.other = {}; + return; + } + } + } + + var index = F.routes.web.indexOf(this); + if (index !== -1) { + F.routes.web.splice(index, 1); + F.$routesSort(); + F.temporary.other = {}; + } +} + /** * Get routing by name * @param {String} name * @return {Object} */ -F.routing = function(name) { +global.ROUTING = F.routing = function(name, flags) { + + var id = name.substring(0, 3) === 'id:' ? name.substring(3) : null; + if (id) + name = null; + + var search = id ? null : (name.toLowerCase().replace(/\s{2,}/g, ' ') + (flags ? (' ' + flags.where(n => typeof(n) === 'string' && n.substring(0, 2) !== '//' && n[2] !== ':').join(' ')).toLowerCase() : '')).split(' '); + for (var i = 0, length = F.routes.web.length; i < length; i++) { var route = F.routes.web[i]; - if (route.name === name) { - var url = U.path(route.url.join('/')); - if (url[0] !== '/') - url = '/' + url; - return { controller: route.controller, url: url, id: route.id, flags: route.flags, middleware: route.middleware, execute: route.execute, timeout: route.timeout, options: route.options, length: route.length }; + var is = true; + if (id && route.id !== id) + is = false; + else if (search) { + for (var j = 0; j < search.length; j++) { + if (route.search.indexOf(search[j]) === -1) { + is = false; + break; + } + } } + + if (!is) + continue; + + var url = U.path(route.url.join('/')); + if (url[0] !== '/') + url = '/' + url; + + return route; } }; From b82e83b1af792443fdd5176d715b1ad6f2b276d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 18:13:44 +0100 Subject: [PATCH 0746/1669] Improved code. --- internal.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal.js b/internal.js index c9b54755e..234126582 100755 --- a/internal.js +++ b/internal.js @@ -171,7 +171,6 @@ exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { } header = parse_multipart_header(header); - tmp.$step = 1; tmp.$is = header[1] !== null; tmp.name = header[0]; @@ -1413,11 +1412,14 @@ MultipartParser.prototype.write = function(buffer) { for (i = 0; i < len; i++) { c = buffer[i]; switch (state) { + case S.PARSER_UNINITIALIZED: return i; + case S.START: index = 0; state = S.START_BOUNDARY; + case S.START_BOUNDARY: if (index == boundary.length - 2) { if (c === HYPHEN) @@ -1445,10 +1447,12 @@ MultipartParser.prototype.write = function(buffer) { if (c === boundary[index + 2]) index++; break; + case S.HEADER_FIELD_START: state = S.HEADER_FIELD; mark('headerField'); index = 0; + case S.HEADER_FIELD: if (c === CR) { clear('headerField'); @@ -1472,12 +1476,15 @@ MultipartParser.prototype.write = function(buffer) { cl = lower(c); if (cl < A || cl > Z) return i; + break; + case S.HEADER_VALUE_START: if (c === SPACE) break; mark('headerValue'); state = S.HEADER_VALUE; + case S.HEADER_VALUE: if (c === CR) { dataCallback('headerValue', true); @@ -1485,23 +1492,26 @@ MultipartParser.prototype.write = function(buffer) { state = S.HEADER_VALUE_ALMOST_DONE; } break; + case S.HEADER_VALUE_ALMOST_DONE: if (c !== LF) return i; state = S.HEADER_FIELD_START; break; + case S.HEADERS_ALMOST_DONE: if (c !== LF) return i; callback('headersEnd'); state = S.PART_DATA_START; break; + case S.PART_DATA_START: state = S.PART_DATA; mark('partData'); + case S.PART_DATA: prevIndex = index; - if (!index) { // boyer-moore derrived algorithm to safely skip non-boundary data i += boundaryEnd; @@ -1566,8 +1576,10 @@ MultipartParser.prototype.write = function(buffer) { i--; } break; + case S.END: break; + default: return i; } From cebce48e809490489ffd01023631e4cde0d0274c Mon Sep 17 00:00:00 2001 From: Tema Smirnov Date: Wed, 7 Nov 2018 21:22:00 +0300 Subject: [PATCH 0747/1669] Fix workers crash when debugging main process with --inspect or --debug --- index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/index.js b/index.js index 3a5313efb..04619f4cd 100755 --- a/index.js +++ b/index.js @@ -868,6 +868,16 @@ function Framework() { self.isLE = Os.endianness ? Os.endianness() === 'LE' : true; self.isHTTPS = false; + // Fix for workers crash (port in use) when debugging main process with --inspect or --debug + // See: https://github.com/nodejs/node/issues/14325 and https://github.com/nodejs/node/issues/9435 + process.execArgv.forEach((val, i, arr) => { + if (val.indexOf('inspect') != -1 || val.indexOf('debug') != -1) { + // Setting inspect/debug port to random unused + arr[i] = '--inspect=0'; + } + }); + HEADERS.workers.execArgv = process.execArgv; + // It's hidden // self.waits = {}; From 8753b903b1a68576cafb3572d9cb35f11919cefa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 19:47:56 +0100 Subject: [PATCH 0748/1669] Improved code. --- index.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 04619f4cd..a035ec2a0 100755 --- a/index.js +++ b/index.js @@ -870,12 +870,14 @@ function Framework() { // Fix for workers crash (port in use) when debugging main process with --inspect or --debug // See: https://github.com/nodejs/node/issues/14325 and https://github.com/nodejs/node/issues/9435 - process.execArgv.forEach((val, i, arr) => { - if (val.indexOf('inspect') != -1 || val.indexOf('debug') != -1) { - // Setting inspect/debug port to random unused - arr[i] = '--inspect=0'; + for (var i = 0; i < process.execArgv.length; i++) { + // Setting inspect/debug port to random unused + if ((/inspect|debug/).test(process.execArgv[i])) { + process.execArgv[i] = '--inspect=0'; + break; } - }); + } + HEADERS.workers.execArgv = process.execArgv; // It's hidden From 37a306601ba26269ce94cbbaa5aa7eecd792a03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 19:49:35 +0100 Subject: [PATCH 0749/1669] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 60b8d9c3e..5e828e991 100755 --- a/changes.txt +++ b/changes.txt @@ -59,6 +59,7 @@ - fixed: `controller.autoclear()` - fixed: `controller.proxy()` - fixed: `repeat` mode in `SCHEDULE()` +- fixed: `--inspect` argument for Workers - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute From adeaf108fc311ba29c5cfb2e12fe8ca9e6a71803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 19:50:10 +0100 Subject: [PATCH 0750/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5b7b9a641..fd1f69d69 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-26", + "version": "3.0.1-27", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 32195fde9b4667ecaee16941c23e323c08dc6b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Nov 2018 21:58:15 +0100 Subject: [PATCH 0751/1669] Fixed `ROUTE()`. --- index.js | 14 ++++++++------ package.json | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index a035ec2a0..818258b84 100755 --- a/index.js +++ b/index.js @@ -1939,18 +1939,20 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = '/'; if (url) { + url = url.replace(/\t/g, ' '); - var index = url.indexOf(' '); - if (index !== -1) { - method = url.substring(0, index).toLowerCase().trim(); - url = url.substring(index + 1).trim(); - } url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; - }); + }).trim(); + + var index = url.indexOf(' '); + if (index !== -1) { + method = url.substring(0, index).toLowerCase().trim(); + url = url.substring(index + 1).trim(); + } if (method.indexOf(',') !== -1) { !flags && (flags = []); diff --git a/package.json b/package.json index fd1f69d69..40259b217 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-27", + "version": "3.0.1-28", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From f365c08162c814427c767f2d9d4adefc026ccde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 9 Nov 2018 20:28:47 +0100 Subject: [PATCH 0752/1669] Added `image.define()`. --- changes.txt | 1 + image.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 5e828e991..1aae99a03 100755 --- a/changes.txt +++ b/changes.txt @@ -25,6 +25,7 @@ - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records - added: Bundles can be downloaded from URL addresses - added: `ONCE()` alias to `F.once()` +- added: `image.define(value)` performs `convert -define 'value'` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/image.js b/image.js index 2b8edeb30..022dd8dfa 100755 --- a/image.js +++ b/image.js @@ -21,7 +21,7 @@ /** * @module FrameworkImage - * @version 3.0.0 + * @version 3.1.0 */ 'use strict'; @@ -667,6 +667,10 @@ Image.prototype.flop = function() { return this.push('-flop', null, 10); }; +Image.prototype.define = function(value) { + return this.push('-define', value, 10, true); +}; + Image.prototype.minify = function() { return this.push('+profile', '*', null, 10, true); }; From 21bbaa15601cd9a05fa41d95a97c18efa53c9b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 9 Nov 2018 20:28:51 +0100 Subject: [PATCH 0753/1669] Removed useless file. --- tools/merge.sh | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 tools/merge.sh diff --git a/tools/merge.sh b/tools/merge.sh deleted file mode 100644 index c03a40689..000000000 --- a/tools/merge.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -cd "$DIR" -cd .. -cd merged -echo "MERGING" -node merge.js -echo "UGLIFY" -node merge.js --minify -echo "DONE" \ No newline at end of file From d44093a271ae6f07752401aac069ac6cc849b2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 10 Nov 2018 12:59:10 +0100 Subject: [PATCH 0754/1669] Improved code. --- index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/index.js b/index.js index 818258b84..8dccfb8d7 100755 --- a/index.js +++ b/index.js @@ -6429,9 +6429,7 @@ F.exists = function(req, res, max, callback) { var name = req.$key = createTemporaryKey(req); var filename = F.path.temp(name); - var httpcachevalid = false; - - RELEASE && (req.headers['if-none-match'] === ETAG + CONF.etag_version) && (httpcachevalid = true); + var httpcachevalid = RELEASE && (req.headers['if-none-match'] === (ETAG + CONF.etag_version)); if (F.isProcessed(name) || httpcachevalid) { res.options.filename = filename; From 76fabb834e552cc90e8fe98ab6ed73d8e99fdfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Nov 2018 00:38:13 +0100 Subject: [PATCH 0755/1669] Fixed SMTP TLS. --- mail.js | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/mail.js b/mail.js index dc4eca0af..9448c66fa 100755 --- a/mail.js +++ b/mail.js @@ -21,7 +21,7 @@ /** * @module FrameworkMail - * @version 3.0.0 + * @version 3.1.0 */ 'use strict'; @@ -36,6 +36,8 @@ const REG_STATE = /\d+/; const REG_WINLINE = /\r\n/g; const REG_NEWLINE = /\n/g; const REG_AUTH = /(AUTH LOGIN|AUTH PLAIN)/i; +const REG_TLS = /TLS/; +const REG_STARTTLS = /STARTTLS/; const EMPTYARRAY = []; var INDEXSENDER = 0; @@ -577,6 +579,7 @@ Mailer.prototype.send = function(smtp, options, messages, callback) { return self; } + obj.smtpoptions = options; obj.socket.$host = smtp; obj.host = smtp.substring(smtp.lastIndexOf('.', smtp.lastIndexOf('.') - 1) + 1); obj.socket.on('error', function(err) { @@ -811,20 +814,28 @@ Mailer.prototype.$send = function(obj, options, autosend) { switch (code) { case 220: - if (obj.isTLS) { + if (obj.isTLS || REG_TLS.test(line)) { mailer.switchToTLS(obj, options); - return; + } else { + obj.secured = REG_ESMTP.test(line); + command = obj.isTLS || (options.user && options.password) || obj.secured ? 'EHLO' : 'HELO'; + mailer.$writeline(obj, command + ' ' + host); } - command = obj.isTLS || (options.user && options.password) || REG_ESMTP.test(line) ? 'EHLO' : 'HELO'; - mailer.$writeline(obj, command + ' ' + host); - break; + return; case 250: // OPERATION case 251: // FORWARD case 235: // VERIFY case 999: // Total.js again + if (obj.secured && !obj.isTLS && !obj.logged && obj.smtpoptions.user && obj.smtpoptions.password) { + // maybe TLS + obj.isTLS = true; + mailer.$writeline(obj, 'STARTTLS'); + return; + } + mailer.$writeline(obj, buffer.shift()); if (buffer.length) @@ -857,6 +868,7 @@ Mailer.prototype.$send = function(obj, options, autosend) { var value = auth.shift(); if (value) { + obj.logged = true; mailer.$writeline(obj, value); } else { var err = new Error('Forbidden.'); @@ -879,6 +891,12 @@ Mailer.prototype.$send = function(obj, options, autosend) { if (code < 400) return; + if (!obj.isTLS && code === 530 && REG_STARTTLS.test(line)) { + obj.isTLS = true; + mailer.$writeline(obj, 'STARTTLS'); + return; + } + var err = new Error(line); mailer.$events.error && !obj.try && mailer.emit('error', err, obj); From 0775ec42501e95c2e26f22b3a6506bf42d42c524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Nov 2018 00:38:26 +0100 Subject: [PATCH 0756/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 40259b217..b8873567e 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-28", + "version": "3.0.1-29", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 824688b845412029de003696bb9fe2f5c319ca1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Nov 2018 00:42:04 +0100 Subject: [PATCH 0757/1669] Added new changes. --- changes.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 1aae99a03..4c439f748 100755 --- a/changes.txt +++ b/changes.txt @@ -60,7 +60,8 @@ - fixed: `controller.autoclear()` - fixed: `controller.proxy()` - fixed: `repeat` mode in `SCHEDULE()` -- fixed: `--inspect` argument for Workers +- fixed: `--inspect` argument for Workers by Tema Smirnov +- fixed: TLS in SMTP mail sender - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute From 48a7b098ff8c4410fc56b7755492b27f033d392b Mon Sep 17 00:00:00 2001 From: Novak Tomas Date: Sun, 11 Nov 2018 10:44:55 +0100 Subject: [PATCH 0758/1669] Added Openplatform empty project command to totaljs bin file. --- bin/totaljs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/totaljs b/bin/totaljs index f17802003..750381dda 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -557,6 +557,11 @@ function main() { return; } + if (cmd === '-openplatform-app' || cmd === 'openplatform-app') { + git(dir, 'openplatform-application'); + return; + } + if (cmd === '-flow' || cmd === 'flow') { git(dir, 'emptyproject-flow'); return; From b3f6055f26b5be7612f23608eea622ec539873b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0irka?= Date: Sun, 11 Nov 2018 10:48:37 +0100 Subject: [PATCH 0759/1669] Revert "Added Openplatform empty project command to totaljs bin file." --- bin/totaljs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index 750381dda..f17802003 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -557,11 +557,6 @@ function main() { return; } - if (cmd === '-openplatform-app' || cmd === 'openplatform-app') { - git(dir, 'openplatform-application'); - return; - } - if (cmd === '-flow' || cmd === 'flow') { git(dir, 'emptyproject-flow'); return; From 7d8aa46f54c34ee3aea7f798a5f3ce2b502825d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Nov 2018 13:29:42 +0100 Subject: [PATCH 0760/1669] Improved NoSQL joins. --- nosql.js | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/nosql.js b/nosql.js index 72eaa611e..cebc7b96e 100755 --- a/nosql.js +++ b/nosql.js @@ -2256,7 +2256,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { var join = self.$join[key]; var response = self.$response; - var unique = []; + var unique = new Set(); if (response instanceof Array && response.length) { for (var i = 0; i < response.length; i++) { @@ -2264,18 +2264,16 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { var val = item[join.b]; if (val !== undefined) { if (val instanceof Array) { - for (var j = 0; j < val.length; j++) { - if (unique.indexOf(val[j]) === -1) - unique.push(val[j]); - } - } else if (unique.indexOf(val) === -1) - unique.push(val); + for (var j = 0; j < val.length; j++) + unique.add(val[j]); + } else + unique.add(val); } } } else if (response) { var val = response[join.b]; - if (val !== undefined && unique.indexOf(val) === -1) - unique.push(val); + if (val !== undefined) + unique.add(val); } var isTable = self.db instanceof Table; @@ -2283,8 +2281,8 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { if (join.scalartype) { join.items = []; - join.count = unique.length; - for (var i = 0; i < unique.length; i++) { + join.count = unique.size; + for (var m of unique.values()) { (function(val) { var builder = db.scalar(join.scalartype, join.scalarfield).callback(function(err, response) { join.items.push({ id: val, response: response }); @@ -2309,20 +2307,20 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { builder.$scope = join.builder.$scope; builder.where(join.a, val); - })(unique[i]); + })(m); } } else { - if (unique.length) { + if (unique.size) { join.builder.$options.fields && join.builder.$options.fields.push(join.a); join.builder.$callback = function(err, docs) { join.items = docs; next(); }; if (isTable) - db.find(join.builder).in(join.a, unique); + db.find(join.builder).in(join.a, Array.from(unique)); else - db.find(join.builder).in(join.a, unique); + db.find(join.builder).in(join.a, Array.from(unique)); } else { join.items = join.builder.$options.first ? null : []; next(); From c93c5b5be3379391ca9861564de7eba339c3128f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Nov 2018 22:31:41 +0100 Subject: [PATCH 0761/1669] Updated `F.path.mkdir()` by adding a cache(). --- changes.txt | 1 + index.js | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/changes.txt b/changes.txt index 4c439f748..021908e1a 100755 --- a/changes.txt +++ b/changes.txt @@ -34,6 +34,7 @@ - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method +- updated: `F.path.mkdir()` currently caches a value - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 8dccfb8d7..0bb1b283f 100755 --- a/index.js +++ b/index.js @@ -9660,6 +9660,13 @@ FrameworkPath.prototype.verify = function(name) { FrameworkPath.prototype.mkdir = function(p) { + var key = '$directory-' + p; + + if (F.temporary.path[key]) + return F; + + F.temporary.path[key] = true; + var is = F.isWindows; var s = ''; @@ -9683,6 +9690,9 @@ FrameworkPath.prototype.mkdir = function(p) { p = p.substring(0, l); } + if (existsSync(p)) + return F; + var arr = is ? p.replace(/\//g, '\\').split('\\') : p.split('/'); var directory = s; From 58c1c82694254e8be98ec633487d39260b0c023e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 14 Nov 2018 17:52:37 +0100 Subject: [PATCH 0762/1669] Fixed `versions` regexp. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 021908e1a..f5c6a2d45 100755 --- a/changes.txt +++ b/changes.txt @@ -63,6 +63,7 @@ - fixed: `repeat` mode in `SCHEDULE()` - fixed: `--inspect` argument for Workers by Tema Smirnov - fixed: TLS in SMTP mail sender +- fixed: applying of versions - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/index.js b/index.js index 0bb1b283f..420007fc0 100755 --- a/index.js +++ b/index.js @@ -49,7 +49,7 @@ const COMPRESSIONSPECIAL = { 'js': 1, 'css': 1 }; const REG_TEMPORARY = /\//g; const REG_MOBILE = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|Tablet/i; const REG_ROBOT = /search|agent|bot|crawler|spider/i; -const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; +const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.\_]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; const REG_NEWIMPL = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; From ef4c435e7b5cad0ebc64f76e9065c3ba527b4cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 15 Nov 2018 12:57:52 +0100 Subject: [PATCH 0763/1669] Improved cluster. --- cluster.js | 30 +++++++++++++++++++----------- index.js | 2 +- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cluster.js b/cluster.js index 95567a852..0c099d669 100644 --- a/cluster.js +++ b/cluster.js @@ -48,23 +48,31 @@ exports.on = function(name, callback) { return F; }; -exports.emit = function(name, data) { - if (Cluster.isMaster) { - CLUSTER_EMIT.name = name; - CLUSTER_EMIT.data = data; +exports.emit = function(name, a, b, c, d, e) { + + CLUSTER_EMIT.name = name; + CLUSTER_EMIT.a = a; + CLUSTER_EMIT.b = b; + CLUSTER_EMIT.c = c; + CLUSTER_EMIT.d = d; + CLUSTER_EMIT.e = e; + + if (Cluster.isMaster) message(CLUSTER_EMIT); - } else if (F.isCluster) { - CLUSTER_EMIT.name = name; - CLUSTER_EMIT.data = data; + else if (F.isCluster) process.send(CLUSTER_EMIT); - } + return F; }; -exports.master = function(name, data) { +exports.master = function(name, a, b, c, d, e) { if (F.isCluster) { CLUSTER_MASTER.name = name; - CLUSTER_MASTER.data = data; + CLUSTER_MASTER.a = a; + CLUSTER_MASTER.b = b; + CLUSTER_MASTER.c = c; + CLUSTER_MASTER.d = d; + CLUSTER_MASTER.e = e; process.send(CLUSTER_MASTER); } return F; @@ -233,7 +241,7 @@ function message(m) { if (m.TYPE === 'master') { if (MASTER && MASTER[m.name]) { for (var i = 0, length = MASTER[m.name].length; i < length; i++) - MASTER[m.name][i](m.data); + MASTER[m.name][i](m.a, m.b, m.c, m.d, m.e); } } else { if (m.target === 'master') { diff --git a/index.js b/index.js index 420007fc0..df2053386 100755 --- a/index.js +++ b/index.js @@ -16772,7 +16772,7 @@ process.on('message', function(msg, h) { msg.TYPE === 'cache-clear' && F.cache.clear(false); msg.TYPE === 'req' && F.cluster.req(msg); msg.TYPE === 'res' && msg.target === F.id && F.cluster.res(msg); - msg.TYPE === 'emit' && F.$events[msg.name] && EMIT(msg.name, msg.data); + msg.TYPE === 'emit' && F.$events[msg.name] && EMIT(msg.name, msg.a, msg.b, msg.c, msg.d, msg.e); msg.TYPE === 'nosql-meta' && NOSQL(msg.name).meta(msg.key, msg.value, true); msg.TYPE === 'table-meta' && TABLE(msg.name).meta(msg.key, msg.value, true); } From 00126637acfda5df5605f19835b5dd5aaef97951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 15 Nov 2018 16:04:04 +0100 Subject: [PATCH 0764/1669] Added build exceptions for `.file` or `file-bk.ext` --- changes.txt | 1 + index.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index f5c6a2d45..26d2083ac 100755 --- a/changes.txt +++ b/changes.txt @@ -26,6 +26,7 @@ - added: Bundles can be downloaded from URL addresses - added: `ONCE()` alias to `F.once()` - added: `image.define(value)` performs `convert -define 'value'` +- added: Total.js JS files (+ packages) tarted with `.` (dot) or ended with `-bk` won't be processed - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index df2053386..433fb7cb2 100755 --- a/index.js +++ b/index.js @@ -3730,8 +3730,9 @@ F.$load = function(types, targetdirectory, callback, packageName) { var ext = U.getExtension(o); if (ext) ext = '.' + ext; - if (ext !== extension) + if (ext !== extension || o[0] === '.' || o.endsWith('-bk' + extension)) return; + var name = (level ? U.$normalize(directory).replace(dir, '') + '/' : '') + o.substring(0, o.length - ext.length); output.push({ name: name[0] === '/' ? name.substring(1) : name, filename: Path.join(dir, name) + extension }); }); From 6e1d809d8ad4073505c8499740421883cdd7325b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 17 Nov 2018 17:21:08 +0100 Subject: [PATCH 0765/1669] Added `ROUTE()` dynamic schemas. --- builders.js | 31 +++++++++++-- changes.txt | 1 + index.js | 83 +++++++++++++++++++++++++--------- package.json | 2 +- test/controllers/default.js | 6 +++ test/definitions/initialize.js | 4 -- test/schemas/user.js | 21 ++++++++- test/test-framework-debug.js | 73 ++++++++++++++++++++++++------ test/test-framework-release.js | 73 ++++++++++++++++++++++++------ 9 files changed, 237 insertions(+), 57 deletions(-) diff --git a/builders.js b/builders.js index 217f1253d..bdaad14d5 100755 --- a/builders.js +++ b/builders.js @@ -36,7 +36,9 @@ const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); var schemas = {}; +var schemasall = {}; var operations = {}; +var schemacache = {}; var transforms = { pagination: {}, error: {}, restbuilder: {} }; function SchemaBuilder(name) { @@ -237,9 +239,10 @@ SchemaBuilder.prototype.remove = function(name) { var schema = this.collection[name]; schema && schema.destroy(); schema = null; + delete schemasall[name.toLowerCase()]; delete this.collection[name]; } else { - delete schemas[this.name]; + exports.remove(this.name); this.collection = null; } }; @@ -1665,6 +1668,7 @@ SchemaBuilderEntity.prototype.make = function(model, filter, callback, argument, } var builder = this.validate(output, undefined, undefined, undefined, filter); + if (builder.hasError()) { this.onError && this.onError(builder, model, 'make'); callback && callback(builder, null, argument); @@ -3169,7 +3173,7 @@ global.EACHSCHEMA = exports.eachschema = function(group, fn) { } }; -exports.getschema = function(group, name, fn, timeout) { +global.$$$ = global.GETSCHEMA = exports.getschema = function(group, name, fn, timeout) { if (!name || typeof(name) === 'function') { fn = name; @@ -3189,6 +3193,10 @@ exports.getschema = function(group, name, fn, timeout) { return g ? g.get(name) : undefined; }; +exports.findschema = function(groupname) { + return schemasall[groupname.toLowerCase()]; +}; + exports.newschema = function(group, name) { if (!group) @@ -3198,7 +3206,11 @@ exports.newschema = function(group, name) { schemas[group] = new SchemaBuilder(group); var o = schemas[group].create(name); + var key = group + '/' + name; + o.owner = F.$owner(); + schemasall[key.toLowerCase()] = o; + return o; }; @@ -3209,10 +3221,23 @@ exports.newschema = function(group, name) { */ exports.remove = function(group, name) { if (name) { + var g = schemas[group || DEFAULT_SCHEMA]; g && g.remove(name); - } else + var key = ((group || DEFAULT_SCHEMA) + '/' + name).toLowerCase(); + delete schemasall[key]; + + } else { + delete schemas[group]; + + var lower = group.toLowerCase(); + + Object.keys(schemasall).forEach(function(key) { + if (key.substring(0, group.length) === lower) + delete schemasall[key]; + }); + } }; global.EACHOPERATION = function(fn) { diff --git a/changes.txt b/changes.txt index 26d2083ac..eb36a952f 100755 --- a/changes.txt +++ b/changes.txt @@ -32,6 +32,7 @@ - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` +- updated: `ROUTE()` supports dynamic schemas - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method diff --git a/index.js b/index.js index 433fb7cb2..42cbcefe6 100755 --- a/index.js +++ b/index.js @@ -306,7 +306,6 @@ global.RESOURCE = (name, key) => F.resource(name, key); global.TRANSLATE = (name, key) => F.translate(name, key); global.TRANSLATOR = (name, text) => F.translator(name, text); global.TRACE = (message, name, uri, ip) => F.trace(message, name, uri, ip); -global.$$$ = global.GETSCHEMA = (group, name, fn, timeout) => framework_builders.getschema(group, name, fn, timeout); global.CREATE = (group, name) => framework_builders.getschema(group, name).default(); global.SCRIPT = (body, value, callback, param) => F.script(body, value, callback, param); global.SINGLETON = (name, def) => SINGLETONS[name] || (SINGLETONS[name] = (new Function('return ' + (def || '{}')))()); @@ -1942,7 +1941,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.replace(/\t/g, ' '); - url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { + url = url.replace(/(^|\s?)\*([{}a-z0-9}]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; @@ -2066,6 +2065,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var corsflags = []; var membertype = 0; var isGENERATOR = false; + var isDYNAMICSCHEMA = false; var description; var id = null; var groups = []; @@ -2134,6 +2134,12 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu if (schema.length === 1) { schema[1] = schema[0]; schema[0] = 'default'; + + // Is dynamic schema? + if (schema[1][0] === '{') { + isDYNAMICSCHEMA = true; + schema[1] = schema[1].substring(1, schema[1].length - 1).trim(); + } } index = schema[1].indexOf('#'); @@ -2378,6 +2384,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var params = []; var reg = null; var regIndex = null; + var dynamicidindex = -1; if (url.indexOf('{') !== -1) { routeURL.forEach(function(o, i) { @@ -2389,6 +2396,9 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var sub = o.substring(1, o.length - 1); var name = o.substring(1, o.length - 1).trim(); + if (name === 'id') + dynamicidindex = i; + params.push(name); if (sub[0] !== '/') @@ -2492,6 +2502,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.urlraw = urlraw; r.url = routeURL; r.param = arr; + r.paramidindex = dynamicidindex; r.paramnames = params.length ? params : null; r.flags = flags || EMPTYARRAY; r.flags2 = flags_to_object(flags); @@ -2525,6 +2536,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.isCACHE = !url.startsWith('/#') && !CUSTOM && !arr.length && !isWILDCARD; r.isPARAM = arr.length > 0; r.isDELAY = isDELAY; + r.isDYNAMICSCHEMA = isDYNAMICSCHEMA; r.CUSTOM = CUSTOM; r.options = options; r.regexp = reg; @@ -5539,12 +5551,21 @@ F.$onParseQueryUrl = function(req) { * @param {String} name * @param {Function(err, body)} callback */ -F.onSchema = function(req, group, name, callback, filter, novalidate, workflow) { - var schema = GETSCHEMA(group, name); +F.onSchema = function(req, route, callback) { + + var schema; + + if (route.isDYNAMICSCHEMA) { + var index = route.param[route.paramnames.indexOf(route.schema[1])]; + req.$schemaname = route.schema[0] + '/' + req.split[index]; + schema = framework_builders.findschema(req.$schemaname); + } else + schema = GETSCHEMA(route.schema[0], route.schema[1]); + if (schema) - schema.make(req.body, filter, onSchema_callback, callback, novalidate, workflow ? workflow.meta : null); + schema.make(req.body, route.schema[2], onSchema_callback, callback, route.novalidate, route.workflow ? route.workflow.meta : null); else - callback(new Error('Schema "' + group + '/' + name + '" not found.')); + callback('Schema "' + (route.isDYNAMICSCHEMA ? req.$schemaname : (route.schema[0] + '/' + route.schema[1])) + '" not found.'); }; function onSchema_callback(err, res, callback) { @@ -10556,7 +10577,7 @@ Controller.prototype.getSchema = function() { var route = this.route; if (!route.schema || !route.schema[1]) throw new Error('The controller\'s route does not define any schema.'); - var schema = GETSCHEMA(route.schema[0], route.schema[1]); + var schema = route.isDYNAMICSCHEMA ? framework_builders.findschema(route.schema[0] + '/' + this.params[route.schema[1]]) : GETSCHEMA(route.schema[0], route.schema[1]); if (schema) return schema; throw new Error('Schema "{0}" does not exist.'.format(route.schema[1])); @@ -15009,10 +15030,12 @@ function extend_request(PROTO) { return next(self, code); } - F.onSchema(self, self.$total_route.schema[0], self.$total_route.schema[1], function(err, body) { - + F.onSchema(self, self.$total_route, function(err, body) { if (err) { - self.$total_400(err); + if (self.$total_route.isDYNAMICSCHEMA) + self.$total_404(err); + else + self.$total_400(err); next = null; } else { F.stats.request.schema++; @@ -15020,8 +15043,7 @@ function extend_request(PROTO) { self.$total_schema = true; next(self, code); } - - }, route.schema[2], route.novalidate, route.workflow); + }); }; PROTO.$total_authorize = function(isLogged, user, roles) { @@ -15193,6 +15215,12 @@ function extend_request(PROTO) { this.$total_execute(400, true); }; + PROTO.$total_404 = function(problem) { + this.$total_route = F.lookup(this, '#404', EMPTYARRAY, 0); + this.$total_exception = problem; + this.$total_execute(404, true); + }; + PROTO.$total_500 = function(problem) { this.$total_route = F.lookup(this, '#500', EMPTYARRAY, 0); this.$total_exception = problem; @@ -17046,19 +17074,21 @@ function parseComponent(body, filename) { return response; } -function getSchemaName(schema) { - return schema[0] === 'default' ? schema[1] : schema[0] + '/' + schema[1]; +function getSchemaName(schema, params) { + return schema[0] === 'default' ? (params ? params[schema[1]] : schema[1]) : schema[0] + '/' + schema[1]; } // Default action for workflow routing function controller_json_workflow(id) { var self = this; - self.id = id; var w = self.route.workflow; + self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + CONF.logger && (self.req.$logger = []); if (w instanceof Object) { + if (!w.type) { // IS IT AN OPERATION? @@ -17067,10 +17097,13 @@ function controller_json_workflow(id) { return; } - var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); - + var schema = self.route.isDYNAMICSCHEMA ? framework_builders.findschema(self.req.$schemaname || (self.route.schema[0] + '/' + self.params[self.route.schema[1]])) : GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { - self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema))); + var err = 'Schema "{0}" not found.'.format(getSchemaName(self.route.schema, self.route.isDYNAMICSCHEMA ? self.params : null)); + if (self.route.isDYNAMICSCHEMA) + self.throw404(err); + else + self.throw500(err); return; } @@ -17090,6 +17123,7 @@ function controller_json_workflow(id) { w.name = w.id; } } + if (w.name) self[w.type](w.name, self.callback()); else { @@ -17098,16 +17132,22 @@ function controller_json_workflow(id) { else self.throw500('Operation @' + w.id + ' not found.'); } + + if (self.route.isDYNAMICSCHEMA) + w.type = ''; + } else self.$exec(w, null, self.callback()); } // Default action for workflow routing function controller_json_workflow_multiple(id) { + var self = this; - self.id = id; var w = self.route.workflow; + self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + CONF.logger && (self.req.$logger = []); if (w instanceof Object) { @@ -17119,10 +17159,9 @@ function controller_json_workflow_multiple(id) { return; } - var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); - + var schema = self.route.isDYNAMICSCHEMA ? framework_builders.findschema(self.route.schema[0] + '/' + self.params[self.route.schema[1]]) : GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { - self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema))); + self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema, self.isDYNAMICSCHEMA ? self.params : null))); return; } diff --git a/package.json b/package.json index b8873567e..3d3763e44 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-29", + "version": "3.0.1-30", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/test/controllers/default.js b/test/controllers/default.js index c535d0328..a258d92aa 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -101,6 +101,12 @@ exports.install = function() { F.route('/live/', viewLive); F.route('/live/incoming/', viewLiveIncoming, ['mixed']); + ROUTE('GET /api/static/orders/ *Orders --> @query'); + ROUTE('GET /api/static/users/ *Users --> @query'); + ROUTE('POST /api/static/orders/ *Orders --> @save'); + ROUTE('GET /api/dynamic/{schema}/ *{schema} --> @query'); + ROUTE('POST /api/dynamic/{schema}/ *{schema} --> @save'); + F.redirect('http://www.google.sk', 'http://www.petersirka.sk'); F.route('#408', function DEFER() { diff --git a/test/definitions/initialize.js b/test/definitions/initialize.js index 65c2b35a1..8e9aa5a80 100644 --- a/test/definitions/initialize.js +++ b/test/definitions/initialize.js @@ -71,10 +71,6 @@ framework.onPictureUrl = function(dimension, id, width, height, alt) { return dimension + '-' + id + '.jpg'; }; -framework.onValidate = function(name, value) { - return name + value; -}; - // Is read from http://www.totaljs.com/framework/include.js //framework.precompile('precompile.layout', 'http://www.totaljs.com/framework/_layout.html'); //framework.precompile('precompile.homepage', 'http://www.totaljs.com/framework/homepage.html'); \ No newline at end of file diff --git a/test/schemas/user.js b/test/schemas/user.js index b5f66fa17..15a631945 100644 --- a/test/schemas/user.js +++ b/test/schemas/user.js @@ -1 +1,20 @@ -F.global.schemas = 1; \ No newline at end of file +F.global.schemas = 1; + +NEWSCHEMA('Orders', function(schema) { + + schema.define('name', String, true); + + schema.setQuery(function($) { + $.success('orders'); + }); + + schema.setSave(function($) { + $.success('orders'); + }); +}); + +NEWSCHEMA('Users', function(schema) { + schema.setQuery(function($) { + $.success('users'); + }); +}); \ No newline at end of file diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 7b814a460..616d946bb 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -5,13 +5,6 @@ var url = 'http://127.0.0.1:8001/'; var errorStatus = 0; var max = 100; -//F.snapshot('/templates/localization.html', '/users/petersirka/desktop/localization.html'); - -// INSTALL('module', 'https://www.totaljs.com/framework/include.js', { test: true }); - -//framework.map('/minify/', '@testpackage', ['.html', 'js']); -//framework.map('/minify/', 'models'); -//framework.map('/minify/', F.path.models()); framework.onCompileView = function(name, html) { return html + 'COMPILED'; }; @@ -822,7 +815,7 @@ function test_routing(next) { }); async.await('theme-green', function(complete) { - utils.request(url + '/green/js/default.js', [], function(error, data, code, headers) { + utils.request(url + 'green/js/default.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with static files.'); @@ -831,7 +824,7 @@ function test_routing(next) { }); async.await('theme-green-merge', function(complete) { - utils.request(url + '/merge-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'merge-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data.indexOf('var a=1+1;') !== -1 && data.indexOf('var b=2+2;'), 'Themes: problem with merging static files'); @@ -840,7 +833,7 @@ function test_routing(next) { }); async.await('theme-green-map', function(complete) { - utils.request(url + '/map-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'map-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with mapping static files.'); @@ -848,11 +841,65 @@ function test_routing(next) { }); }); - async.await('theme-green', function(complete) { - utils.request(url + 'theme-green/', ['get'], function(error, data, code, headers) { + async.await('dynamic-schema-1', function(complete) { + utils.request(url + 'api/dynamic/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 1'); + complete(); + }); + }); + + async.await('dynamic-schema-2', function(complete) { + utils.request(url + 'api/dynamic/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'Dynamic schemas 2'); + complete(); + }); + }); + + async.await('dynamic-schema-3', function(complete) { + utils.request(url + 'api/dynamic/products/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 404, 'Dynamic schemas 3'); + complete(); + }); + }); + + async.await('dynamic-schema-4', function(complete) { + utils.request(url + 'api/dynamic/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 4'); + complete(); + }); + }); + + async.await('static-schema-1', function(complete) { + utils.request(url + 'api/static/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'static schemas 1'); + complete(); + }); + }); + + async.await('static-schema-2', function(complete) { + utils.request(url + 'api/static/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'static schemas 2'); + complete(); + }); + }); + + async.await('static-schema-3', function(complete) { + utils.request(url + 'api/static/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { if (error) throw error; - console.log('--->', data); + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 3'); complete(); }); }); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index e5a2efb7d..ee19598de 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -5,13 +5,6 @@ var url = 'http://127.0.0.1:8001/'; var errorStatus = 0; var max = 100; -//F.snapshot('/templates/localization.html', '/users/petersirka/desktop/localization.html'); - -// INSTALL('module', 'https://www.totaljs.com/framework/include.js', { test: true }); - -//framework.map('/minify/', '@testpackage', ['.html', 'js']); -//framework.map('/minify/', 'models'); -//framework.map('/minify/', F.path.models()); framework.onCompileView = function(name, html) { return html + 'COMPILED'; }; @@ -822,7 +815,7 @@ function test_routing(next) { }); async.await('theme-green', function(complete) { - utils.request(url + '/green/js/default.js', [], function(error, data, code, headers) { + utils.request(url + 'green/js/default.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with static files.'); @@ -831,7 +824,7 @@ function test_routing(next) { }); async.await('theme-green-merge', function(complete) { - utils.request(url + '/merge-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'merge-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data.indexOf('var a=1+1;') !== -1 && data.indexOf('var b=2+2;'), 'Themes: problem with merging static files'); @@ -840,7 +833,7 @@ function test_routing(next) { }); async.await('theme-green-map', function(complete) { - utils.request(url + '/map-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'map-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with mapping static files.'); @@ -848,11 +841,65 @@ function test_routing(next) { }); }); - async.await('theme-green', function(complete) { - utils.request(url + 'theme-green/', ['get'], function(error, data, code, headers) { + async.await('dynamic-schema-1', function(complete) { + utils.request(url + 'api/dynamic/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 1'); + complete(); + }); + }); + + async.await('dynamic-schema-2', function(complete) { + utils.request(url + 'api/dynamic/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'Dynamic schemas 2'); + complete(); + }); + }); + + async.await('dynamic-schema-3', function(complete) { + utils.request(url + 'api/dynamic/products/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 404, 'Dynamic schemas 3'); + complete(); + }); + }); + + async.await('dynamic-schema-4', function(complete) { + utils.request(url + 'api/dynamic/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 4'); + complete(); + }); + }); + + async.await('static-schema-1', function(complete) { + utils.request(url + 'api/static/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'static schemas 1'); + complete(); + }); + }); + + async.await('static-schema-2', function(complete) { + utils.request(url + 'api/static/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'static schemas 2'); + complete(); + }); + }); + + async.await('static-schema-3', function(complete) { + utils.request(url + 'api/static/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { if (error) throw error; - console.log('--->', data); + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 3'); complete(); }); }); From 3d4d852bf28423b7a37dbb829e5df0870c45cfb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 18 Nov 2018 20:11:28 +0100 Subject: [PATCH 0766/1669] Fixed routing. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 42cbcefe6..5f568ca57 100755 --- a/index.js +++ b/index.js @@ -2502,7 +2502,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.urlraw = urlraw; r.url = routeURL; r.param = arr; - r.paramidindex = dynamicidindex; + r.paramidindex = isDYNAMICSCHEMA ? dynamicidindex : -1; r.paramnames = params.length ? params : null; r.flags = flags || EMPTYARRAY; r.flags2 = flags_to_object(flags); From d17683fb9a1fec2fc130390ef479a5c140f316e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 18 Nov 2018 20:58:57 +0100 Subject: [PATCH 0767/1669] Fixed routing. --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 5f568ca57..b6645b9e1 100755 --- a/index.js +++ b/index.js @@ -17083,7 +17083,7 @@ function controller_json_workflow(id) { var self = this; var w = self.route.workflow; - self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + self.id = self.route.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; CONF.logger && (self.req.$logger = []); @@ -17146,7 +17146,7 @@ function controller_json_workflow_multiple(id) { var self = this; var w = self.route.workflow; - self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + self.id = self.route.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; CONF.logger && (self.req.$logger = []); From a9fa0c3c95cb955d907c1af16688815f3e23798c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 19 Nov 2018 08:35:11 +0100 Subject: [PATCH 0768/1669] Improved code. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index b6645b9e1..b925c2fdd 100755 --- a/index.js +++ b/index.js @@ -49,7 +49,7 @@ const COMPRESSIONSPECIAL = { 'js': 1, 'css': 1 }; const REG_TEMPORARY = /\//g; const REG_MOBILE = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|Tablet/i; const REG_ROBOT = /search|agent|bot|crawler|spider/i; -const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.\_]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; +const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-._]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; const REG_NEWIMPL = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; From 9faad8b5a908fdd592a6ce4cb8555bbc0bd23358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Nov 2018 09:59:24 +0100 Subject: [PATCH 0769/1669] Fixed test. --- test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test.js b/test.js index d4abd2114..a3e9c1fe5 100644 --- a/test.js +++ b/test.js @@ -178,6 +178,9 @@ exports.load = function() { return next(); } + if (U.getExtension(filename) !== 'js') + return next(); + T.current = { filename: filename, items: [] }; var m = require(filename); T.current.module = m; From fef98c9e3590555b8230b4e6f8ec4f83cab44f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Nov 2018 10:01:06 +0100 Subject: [PATCH 0770/1669] New changes. --- changes.txt | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index eb36a952f..ce2627ecc 100755 --- a/changes.txt +++ b/changes.txt @@ -66,6 +66,7 @@ - fixed: `--inspect` argument for Workers by Tema Smirnov - fixed: TLS in SMTP mail sender - fixed: applying of versions +- fixed: unit-tests reads only `javascript` files - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/package.json b/package.json index 3d3763e44..bb82596fb 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-30", + "version": "3.0.1-31", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From ba9646533d086151c0804b8091fb37e852787af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Nov 2018 12:42:18 +0100 Subject: [PATCH 0771/1669] Improved TaskBuilder. --- builders.js | 89 +++++++++++++++++++++++++++++++++++++++++++++++------ changes.txt | 1 + index.js | 6 ++++ 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/builders.js b/builders.js index bdaad14d5..63c31d994 100755 --- a/builders.js +++ b/builders.js @@ -38,7 +38,6 @@ const Qs = require('querystring'); var schemas = {}; var schemasall = {}; var operations = {}; -var schemacache = {}; var transforms = { pagination: {}, error: {}, restbuilder: {} }; function SchemaBuilder(name) { @@ -54,7 +53,22 @@ function SchemaOptions(error, model, options, callback, controller) { this.controller = controller; } -SchemaOptions.prototype = { +function TaskBuilder($) { + var t = this; + t.value = t.model = {}; + t.tasks = {}; + if ($ instanceof SchemaOptions || $ instanceof OperationOptions) { + t.error = $.error; + t.controller = $.controller; + } else { + if ($ instanceof Controller) + t.controller = $; + else if ($ instanceof ErrorBuilder) + t.error = $; + } +} + +SchemaOptions.prototype = TaskBuilder.prototype = { get user() { return this.controller ? this.controller.user : null; @@ -4953,6 +4967,10 @@ OperationOptions.prototype = { } }; +SchemaOptions.prototype.tasks = OperationOptions.prototype.tasks = function() { + return new TaskBuilder(this); +}; + OperationOptions.prototype.cancel = function() { var self = this; self.callback = null; @@ -5202,6 +5220,7 @@ global.Pagination = Pagination; global.Page = Page; global.UrlBuilder = global.URLBuilder = UrlBuilder; global.SchemaBuilder = SchemaBuilder; +global.TaskBuilder = TaskBuilder; // Uninstall owners exports.uninstall = function(owner) { @@ -5219,11 +5238,63 @@ exports.uninstall = function(owner) { }); }; -exports.restart = function() { - schemas = {}; - operations = {}; - Object.keys(transforms).forEach(function(key) { - if (key.indexOf('_') === -1) - transforms[key] = {}; - }); +TaskBuilder.prototype.invalid = function(error) { + var self = this; + if (!self.$done) { + !self.error && (self.error = new ErrorBuilder()); + self.error.push(self.current, error); + self.done(); + } + return self; +}; + +TaskBuilder.prototype.push = function(name, fn) { + var self = this; + self.tasks[name] = fn; + return self; +}; + +TaskBuilder.prototype.next = function() { + var self = this; + if (!self.$done) { + self.prev = self.current; + for (var i = 0; i < arguments.length; i++) { + var task = self.tasks[self.current = arguments[i]]; + if (task == null) + continue; + else { + task.call(self, self); + return self; + } + } + self.done(); + } + return self; +}; + +TaskBuilder.prototype.done = function(data) { + var self = this; + self.$callback && self.$callback(self.error && self.error.is ? self.error : null, data || self.value); + self.$done = true; + return self; +}; + +TaskBuilder.prototype.success = function(data) { + return this.done(SUCCESS(true, data)); }; + +TaskBuilder.prototype.callback = function(fn) { + var self = this; + self.$callback = fn; + return self; +}; + +TaskBuilder.prototype.exec = function(name, callback) { + var self = this; + + if (callback) + self.$callback = callback; + + self.next(name); + return self; +}; \ No newline at end of file diff --git a/changes.txt b/changes.txt index ce2627ecc..b4fcac9c0 100755 --- a/changes.txt +++ b/changes.txt @@ -27,6 +27,7 @@ - added: `ONCE()` alias to `F.once()` - added: `image.define(value)` performs `convert -define 'value'` - added: Total.js JS files (+ packages) tarted with `.` (dot) or ended with `-bk` won't be processed +- added: A new builder called `TaskBuilder` for creating custom tasks in Schemas or Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index b925c2fdd..22f5c4c27 100755 --- a/index.js +++ b/index.js @@ -10525,6 +10525,12 @@ Controller.prototype.operation = function(name, value, callback, options) { return this; }; +Controller.prototype.tasks = function() { + var tb = new TaskBuilder(this); + // tb.callback(this.callback()); + return tb; +}; + Controller.prototype.$operation2 = function(name, helper, callback) { if (callback == null && typeof(helper) === 'function') { From 039c51720ba15cb0dbe03fe7a3290e21325018b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Nov 2018 12:51:54 +0100 Subject: [PATCH 0772/1669] Fixed prototype. --- builders.js | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 63c31d994..49d350e06 100755 --- a/builders.js +++ b/builders.js @@ -68,7 +68,46 @@ function TaskBuilder($) { } } -SchemaOptions.prototype = TaskBuilder.prototype = { +TaskBuilder.prototype = { + + get user() { + return this.controller ? this.controller.user : null; + }, + + get session() { + return this.controller ? this.controller.session : null; + }, + + get language() { + return (this.controller ? this.controller.language : '') || ''; + }, + + get ip() { + return this.controller ? this.controller.ip : null; + }, + + get id() { + return this.controller ? this.controller.id : null; + }, + + get params() { + return this.controller ? this.controller.params : null; + }, + + get files() { + return this.controller ? this.controller.files : null; + }, + + get body() { + return this.controller ? this.controller.body : null; + }, + + get query() { + return this.controller ? this.controller.query : null; + } +}; + +SchemaOptions.prototype = { get user() { return this.controller ? this.controller.user : null; From 09b3565040876d2918ae63d4d7037306628a65a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Nov 2018 13:01:34 +0100 Subject: [PATCH 0773/1669] Added `logger` for `TaskBuiler`. --- builders.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builders.js b/builders.js index 49d350e06..a4e187727 100755 --- a/builders.js +++ b/builders.js @@ -5296,6 +5296,7 @@ TaskBuilder.prototype.push = function(name, fn) { TaskBuilder.prototype.next = function() { var self = this; if (!self.$done) { + self.current && self.controller && CONF.logger && F.ilogger((self.name || 'tasks') + '.' + self.current, self.controller, self.$now); self.prev = self.current; for (var i = 0; i < arguments.length; i++) { var task = self.tasks[self.current = arguments[i]]; @@ -5334,6 +5335,9 @@ TaskBuilder.prototype.exec = function(name, callback) { if (callback) self.$callback = callback; + if (CONF.logger) + self.$now = Date.now(); + self.next(name); return self; }; \ No newline at end of file From 57afb58ba8bee9fd633f2971a80ee7ffa2da8fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 10:24:39 +0100 Subject: [PATCH 0774/1669] Updated `F.path.mkdir()`. --- changes.txt | 2 +- index.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index b4fcac9c0..87dab9e5b 100755 --- a/changes.txt +++ b/changes.txt @@ -37,7 +37,7 @@ - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method -- updated: `F.path.mkdir()` currently caches a value +- updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 22f5c4c27..1bba46dff 100755 --- a/index.js +++ b/index.js @@ -9680,11 +9680,11 @@ FrameworkPath.prototype.verify = function(name) { return F; }; -FrameworkPath.prototype.mkdir = function(p) { +FrameworkPath.prototype.mkdir = function(p, cache) { var key = '$directory-' + p; - if (F.temporary.path[key]) + if (cache && F.temporary.path[key]) return F; F.temporary.path[key] = true; From 7d6b155cd2c75c59a052cfb82594b0ae54656c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:33:50 +0100 Subject: [PATCH 0775/1669] Fixed GZIP compression. --- index.js | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/index.js b/index.js index 1bba46dff..e119c1564 100755 --- a/index.js +++ b/index.js @@ -6095,13 +6095,15 @@ function compile_check(res) { if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && CONF.allow_compile && !REG_NOCOMPRESS.test(res.options.filename)) return compile_file(res); - var tmp = F.temporary.path[req.$key] = [res.options.filename, size, stats.mtime.toUTCString()]; + var tmp = [res.options.filename, size, stats.mtime.toUTCString()]; if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { compile_gzip(tmp, function() { + F.temporary.path[req.$key] = tmp; res.$file(); delete F.temporary.processing[req.$key]; }); } else { + F.temporary.path[req.$key] = tmp; res.$file(); delete F.temporary.processing[req.$key]; } @@ -13839,7 +13841,7 @@ WebSocket.prototype.destroy = function(problem) { return self; self.close(); - self.$events.destroy && selEMIT('destroy'); + self.$events.destroy && self.emit('destroy'); setTimeout(function() { @@ -15197,14 +15199,6 @@ function extend_request(PROTO) { file.execute(req, res, false); return; - - /* - } catch (err) { - F.error(err, file.controller, req.uri); - res.throw500(); - return; - } - */ } res.continue(); @@ -16004,6 +15998,8 @@ function extend_response(PROTO) { if (name[1] && !compress) headers[HEADER_LENGTH] = name[1]; + else if (compress && name[4]) + headers[HEADER_LENGTH] = name[4]; else if (headers[HEADER_LENGTH]) delete headers[HEADER_LENGTH]; From 5f0d0c965e17c7538eadfab91c6bf0ae17b36f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:41:42 +0100 Subject: [PATCH 0776/1669] Fixed GZIP compression. --- changes.txt | 1 + index.js | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/changes.txt b/changes.txt index 87dab9e5b..b22dff4eb 100755 --- a/changes.txt +++ b/changes.txt @@ -43,6 +43,7 @@ - fixed: a critical bug with JavaScript minificator - fixed: a critical bug with NoSQL counter and freezing app - fixed: a critical bug with rendering of multiple async components +- fixed: a critical bug with GZIP compression - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories diff --git a/index.js b/index.js index e119c1564..e670564ad 100755 --- a/index.js +++ b/index.js @@ -5887,8 +5887,9 @@ function compile_file(res) { F.path.verify('temp'); Fs.writeFileSync(file, compile_content(req.extension, framework_internal.parseBlock(F.routes.blocks[uri.pathname], buffer.toString(ENCODING)), res.options.filename), ENCODING); var stats = Fs.statSync(file); - var tmp = F.temporary.path[req.$key] = [file, stats.size, stats.mtime.toUTCString()]; - compile_gzip(tmp, function() { + var tmp = [file, stats.size, stats.mtime.toUTCString()]; + compile_gzip(tmp, function(tmp) { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); }); @@ -5905,9 +5906,10 @@ function compile_merge(res) { if (!CONF.debug && existsSync(filename)) { var stats = Fs.statSync(filename); - var tmp = F.temporary.path[req.$key] = [filename, stats.size, stats.mtime.toUTCString()]; - compile_gzip(tmp, function() { + var tmp = [filename, stats.size, stats.mtime.toUTCString()]; + compile_gzip(tmp, function(tmp) { delete F.temporary.processing[req.$key]; + F.temporary.path[req.$key] = tmp; res.$file(); }); return; @@ -5917,9 +5919,10 @@ function compile_merge(res) { writer.on('finish', function() { var stats = Fs.statSync(filename); - var tmp = F.temporary.path[req.$key] = [filename, stats.size, stats.mtime.toUTCString()]; + var tmp = [filename, stats.size, stats.mtime.toUTCString()]; this.destroy && this.destroy(); - compile_gzip(tmp, function() { + compile_gzip(tmp, function(tmp) { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); }); @@ -6063,13 +6066,15 @@ F.compile_virtual = function(res) { return compile_file(res); } - var tmp = F.temporary.path[req.$key] = [tmpname, size, stats.mtime.toUTCString()]; + var tmp = [tmpname, size, stats.mtime.toUTCString()]; if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { - compile_gzip(tmp, function() { + compile_gzip(tmp, function(tmp) { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); }); } else { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); } @@ -6097,7 +6102,7 @@ function compile_check(res) { var tmp = [res.options.filename, size, stats.mtime.toUTCString()]; if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { - compile_gzip(tmp, function() { + compile_gzip(tmp, function(tmp) { F.temporary.path[req.$key] = tmp; res.$file(); delete F.temporary.processing[req.$key]; @@ -6131,7 +6136,7 @@ function compile_gzip(arr, callback) { CLEANUP(writer, function() { fsFileExists(filename, function(e, size) { arr.push(size); - callback(); + callback(arr); }); }); From 81615fddb88b48423f5e73a2f5bf95eb5da291a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:42:40 +0100 Subject: [PATCH 0777/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bb82596fb..0f956763a 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-31", + "version": "3.0.1-32", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 784367cf2900263586ba69f1c31bb9cb862a22c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:43:27 +0100 Subject: [PATCH 0778/1669] Updated changelog. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index b22dff4eb..78ad0edc7 100755 --- a/changes.txt +++ b/changes.txt @@ -43,7 +43,7 @@ - fixed: a critical bug with JavaScript minificator - fixed: a critical bug with NoSQL counter and freezing app - fixed: a critical bug with rendering of multiple async components -- fixed: a critical bug with GZIP compression +- fixed: a critical bug with GZIP compression (sometimes appeared in Safari) - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories From e2f376ded9e5ba70d8e9e579544c865fcac4a701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Nov 2018 09:23:00 +0100 Subject: [PATCH 0779/1669] Improved `websocketclient.send()`. --- index.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/index.js b/index.js index e670564ad..81f31bc74 100755 --- a/index.js +++ b/index.js @@ -14501,27 +14501,29 @@ WebSocketClient.prototype.$onclose = function() { */ WebSocketClient.prototype.send = function(message, raw, replacer) { - if (this.isClosed) - return this; + var self = this; - if (this.type !== 1) { - var data = this.type === 3 ? (raw ? message : JSON.stringify(message, replacer)) : (message || '').toString(); + if (self.isClosed) + return self; + + if (self.type !== 1) { + var data = self.type === 3 ? (raw ? message : JSON.stringify(message, replacer)) : typeof(message) === 'object' ? JSON.stringify(message, replacer) : message.toString(); if (CONF.default_websocket_encodedecode === true && data) data = encodeURIComponent(data); - if (this.deflate) { - this.deflatepending.push(U.createBuffer(data)); - this.sendDeflate(); + if (self.deflate) { + self.deflatepending.push(U.createBuffer(data)); + self.sendDeflate(); } else - this.socket.write(U.getWebSocketFrame(0, data, 0x01)); + self.socket.write(U.getWebSocketFrame(0, data, 0x01)); } else if (message) { - if (this.deflate) { - this.deflatepending.push(U.createBuffer(message)); - this.sendDeflate(); + if (self.deflate) { + self.deflatepending.push(U.createBuffer(message)); + self.sendDeflate(); } else - this.socket.write(U.getWebSocketFrame(0, new Int8Array(message), 0x02)); + self.socket.write(U.getWebSocketFrame(0, new Int8Array(message), 0x02)); } - return this; + return self; }; WebSocketClient.prototype.sendDeflate = function() { From 7813128ba8596ec5022a705632614fffd5ab7d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 09:16:12 +0100 Subject: [PATCH 0780/1669] Updated readme. --- readme.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 8f620d01e..2302f17c3 100755 --- a/readme.md +++ b/readme.md @@ -13,8 +13,9 @@ $ npm install -g total.js [![Support](https://www.totaljs.com/img/button-support.png?v=3)](https://www.totaljs.com/support/) -- [__NEW__ Total.js CMS](http://www.totaljs.com/cms/) -- [__NEW__ Total.js Eshop](http://www.totaljs.com/eshop/) +- [__NEW__: Total.js Code editor](http://www.totaljs.com/code/) +- [Total.js CMS](http://www.totaljs.com/cms/) +- [Total.js Eshop](http://www.totaljs.com/eshop/) - [Total.js Wiki](https://www.totaljs.com/wiki/) - [Total.js Flow](https://www.totaljs.com/flow/) - [Total.js Flowboard](https://www.totaljs.com/flowboard/) @@ -29,7 +30,8 @@ $ npm install -g total.js - [Total.js modules and packages](https://modules.totaljs.com) - [Total.js +100 examples](https://github.com/totaljs/examples) - [NoSQL embedded database explorer](https://www.totaljs.com/nosql/) -- [Download +100 client-side components (jComponent) for free](https://componentator.com) +- [Download +100 UI components for free](https://componentator.com) +- [Download +100 CMS widgets for free](https://componentator.com/widgets/) --- From 7f5b00821d2fd265d433841668af90bef55b32ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 09:35:25 +0100 Subject: [PATCH 0781/1669] Removed useless code. --- changes.txt | 3 +++ index.js | 30 ++++++++++++------------------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/changes.txt b/changes.txt index 78ad0edc7..5f147260e 100755 --- a/changes.txt +++ b/changes.txt @@ -70,6 +70,9 @@ - fixed: applying of versions - fixed: unit-tests reads only `javascript` files +- removed: `F.config.debug` +- removed: `controller.isDebug` + - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute - improved: CORS diff --git a/index.js b/index.js index 81f31bc74..60cc6e199 100755 --- a/index.js +++ b/index.js @@ -4217,8 +4217,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); if (F.$bundling) { - content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); - content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (DEBUG ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (DEBUG ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); } if (content.js) @@ -4295,12 +4295,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp = F.components.groups[obj.group] = {}; if (content.js) { - Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (DEBUG ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); tmp.js = true; } if (content.css) { - Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (DEBUG ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); tmp.css = true; } @@ -4735,7 +4735,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } else { F.$configure_configs('@' + name + '/config'); - if (CONF.debug) + if (DEBUG) F.$configure_configs('@' + name + '/config-debug'); else F.$configure_configs('@' + name + '/config-release'); @@ -5747,7 +5747,7 @@ F.usage = function(detailed) { memoryTotal: (memory.heapTotal / 1024 / 1024).floor(2), memoryUsage: (memory.heapUsed / 1024 / 1024).floor(2), memoryRss: (memory.rss / 1024 / 1024).floor(2), - mode: CONF.debug ? 'debug' : 'release', + mode: DEBUG, port: F.port, ip: F.ip, directory: process.cwd() @@ -5904,7 +5904,7 @@ function compile_merge(res) { var merge = F.routes.merge[uri.pathname]; var filename = merge.filename; - if (!CONF.debug && existsSync(filename)) { + if (!DEBUG && existsSync(filename)) { var stats = Fs.statSync(filename); var tmp = [filename, stats.size, stats.mtime.toUTCString()]; compile_gzip(tmp, function(tmp) { @@ -6612,7 +6612,6 @@ global.LOAD = F.load = function(debug, types, pwd) { } F.isWorker = true; - CONF.debug = debug; F.isDebug = debug; global.DEBUG = debug; @@ -6709,7 +6708,6 @@ F.initialize = function(http, debug, options) { if (options.id) F.id = options.id; - CONF.debug = debug; F.isDebug = debug; if (options.bundling != null) @@ -6922,7 +6920,7 @@ F.mode = function(http, name, options) { debug = true; break; } - CONF.debug = debug; + DEBUG = debug; CONF.trace = debug; F.isDebug = debug; global.DEBUG = debug; @@ -7035,7 +7033,7 @@ F.console = function() { console.log('Version : ' + CONF.version); console.log('Author : ' + CONF.author); console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); - console.log('Mode : ' + (CONF.debug ? 'debug' : 'release')); + console.log('Mode : ' + (DEBUG ? 'debug' : 'release')); console.log('===================================================='); console.log('Directory : ' + process.cwd()); console.log('node_modules : ' + PATHMODULES); @@ -7437,7 +7435,7 @@ F.$requestcontinue = function(req, res, headers) { flags.push('sse'); } - if (CONF.debug) { + if (DEBUG) { req.$flags += 'h'; flags.push('debug'); } @@ -8793,7 +8791,7 @@ F.$configure_configs = function(arr, rewrite) { if (!arr) { var filenameA = U.combine('/', 'config'); - var filenameB = U.combine('/', 'config-' + (CONF.debug ? 'debug' : 'release')); + var filenameB = U.combine('/', 'config-' + (DEBUG ? 'debug' : 'release')); arr = []; @@ -10258,10 +10256,6 @@ Controller.prototype = { return F.controllers; }, - get isDebug() { - return CONF.debug; - }, - get isTest() { return this.req.headers['x-assertion-testing'] === '1'; }, @@ -13526,7 +13520,7 @@ WebSocket.prototype = { }, get isDebug() { - return CONF.debug; + return DEBUG; }, get path() { From 62321eb586bf1628d3f669093cc1abcd8d6ed94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 11:02:08 +0100 Subject: [PATCH 0782/1669] Updated `controller.all()`. --- changes.txt | 1 + index.js | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 5f147260e..65d7df0f2 100755 --- a/changes.txt +++ b/changes.txt @@ -38,6 +38,7 @@ - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method - updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) +- updated: `controller.all()` can return `Array` of all WebSocketClient - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 60cc6e199..206d413c9 100755 --- a/index.js +++ b/index.js @@ -13795,11 +13795,17 @@ WebSocket.prototype.change = function(message) { * @return {WebSocket} */ WebSocket.prototype.all = function(fn) { - if (this._keys) { - for (var i = 0, length = this._keys.length; i < length; i++) - fn(this.connections[this._keys[i]], i); + var arr = fn == null || fn == true ? [] : null; + var self = this; + if (self._keys) { + for (var i = 0, length = self._keys.length; i < length; i++) { + if (arr) + arr.push(self.connections[self._keys[i]]); + else + fn(self.connections[self._keys[i]], i); + } } - return this; + return arr ? arr : self; }; /** From 61477c6c78f11fb748ab8ef1e00caf12dbce1bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 19:36:37 +0100 Subject: [PATCH 0783/1669] Added error handling for unpacking. --- index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/index.js b/index.js index 206d413c9..0d04bbc78 100755 --- a/index.js +++ b/index.js @@ -6248,6 +6248,16 @@ F.restore = function(filename, target, callback, filter) { output.count++; + tmp.zlib.on('error', function(e) { + pending--; + var tmp = this.$self; + tmp.writer.end(); + tmp.writer = null; + tmp.zlib = null; + delete open[tmp.name]; + F.error(e, 'bundling', path); + }); + tmp.zlib.on('data', function(chunk) { this.$self.writer.write(chunk); }); From 1a365ffa068fb0fff673f4cc89c3fc5ec1f931c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 08:08:01 +0100 Subject: [PATCH 0784/1669] Fixed `controller.invalid()`. --- changes.txt | 1 + index.js | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/changes.txt b/changes.txt index 65d7df0f2..5894594e0 100755 --- a/changes.txt +++ b/changes.txt @@ -70,6 +70,7 @@ - fixed: TLS in SMTP mail sender - fixed: applying of versions - fixed: unit-tests reads only `javascript` files +- fixed: `controller.invalid()` a problem with ErrorBuilder as a argument - removed: `F.config.debug` - removed: `controller.isDebug` diff --git a/index.js b/index.js index 0d04bbc78..1776f4ebd 100755 --- a/index.js +++ b/index.js @@ -10804,6 +10804,12 @@ Controller.prototype.error = function(err) { Controller.prototype.invalid = function(status) { var self = this; + + if (status instanceof ErrorBuilder) { + setImmediate(next_controller_invalid, self, status); + return status; + } + var type = typeof(status); if (type === 'number') From 1aa7ed1820d93416a90c7974b11b02d7c2870058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 08:41:01 +0100 Subject: [PATCH 0785/1669] New improvements. --- changes.txt | 2 ++ index.js | 67 +++++++++++++++++++++++++++++-------- test/controllers/default.js | 10 +++--- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/changes.txt b/changes.txt index 5894594e0..22cd9476d 100755 --- a/changes.txt +++ b/changes.txt @@ -28,6 +28,8 @@ - added: `image.define(value)` performs `convert -define 'value'` - added: Total.js JS files (+ packages) tarted with `.` (dot) or ended with `-bk` won't be processed - added: A new builder called `TaskBuilder` for creating custom tasks in Schemas or Operations +- added: `WebSocket.send2(message, [comparer(client, message)], [replacer])` a new method for better sending frames +- addde: `PATH` as a global alias for `F.path` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index 1776f4ebd..d6e95da78 100755 --- a/index.js +++ b/index.js @@ -967,7 +967,7 @@ function Framework() { // intialize cache self.cache = new FrameworkCache(); - self.path = new FrameworkPath(); + self.path = global.PATH = new FrameworkPath(); self._request_check_redirect = false; self._request_check_referer = false; @@ -10219,6 +10219,7 @@ Controller.prototype = { }, get path() { + OBSOLETE('controller.path', 'Use: PATH'); return F.path; }, @@ -10255,10 +10256,12 @@ Controller.prototype = { }, get cache() { + OBSOLETE('controller.cache', 'Use: F.cache or CACHE()'); return F.cache; }, get config() { + OBSOLETE('controller.config', 'Use: CONF'); return CONF; }, @@ -10271,6 +10274,7 @@ Controller.prototype = { }, get isSecure() { + OBSOLETE('controller.isSecure', 'Use: controller.secured'); return this.req.isSecure; }, @@ -13524,30 +13528,32 @@ WebSocket.prototype = { }, get global() { + OBSOLETE('controller.global', 'Use: G'); return F.global; }, get config() { + OBSOLETE('controller.config', 'Use: CONF'); return CONF; }, get cache() { + OBSOLETE('controller.cache', 'Use: F.cache or CACHE()'); return F.cache; }, get isDebug() { + OBSOLETE('controller.isDebug', 'Use: DEBUG'); return DEBUG; }, get path() { + OBSOLETE('controller.path', 'Use: PATH'); return F.path; }, - get fs() { - return F.fs; - }, - get isSecure() { + OBSOLETE('controller.isSecure', 'Use: controller.secured'); return this.req.isSecure; }, @@ -13643,23 +13649,25 @@ WebSocket.prototype.removeAllListeners = function(name) { */ WebSocket.prototype.send = function(message, id, blacklist, replacer) { - var keys = this._keys; - if (!keys || !keys.length) - return this; + var self = this; + var keys = self._keys; + + if (!keys || !keys.length || message === undefined) + return self; var data; var raw = false; for (var i = 0, length = keys.length; i < length; i++) { - var conn = this.connections[keys[i]]; + var conn = self.connections[keys[i]]; if (id) { if (id instanceof Array) { if (!websocket_valid_array(conn.id, id)) continue; } else if (id instanceof Function) { - if (!websocket_valid_fn(conn.id, conn, id)) + if (!websocket_valid_fn(conn.id, conn, id, message)) continue; } else throw new Error('Invalid "id" argument.'); @@ -13670,7 +13678,7 @@ WebSocket.prototype.send = function(message, id, blacklist, replacer) { if (websocket_valid_array(conn.id, blacklist)) continue; } else if (blacklist instanceof Function) { - if (websocket_valid_fn(conn.id, conn, blacklist)) + if (websocket_valid_fn(conn.id, conn, blacklist, message)) continue; } else throw new Error('Invalid "blacklist" argument.'); @@ -13688,15 +13696,46 @@ WebSocket.prototype.send = function(message, id, blacklist, replacer) { F.stats.response.websocket++; } - return this; + return self; +}; + +WebSocket.prototype.send2 = function(message, comparer, replacer) { + + var self = this; + var keys = self._keys; + if (!keys || !keys.length || message === undefined) + return self; + + var data; + var raw = false; + + for (var i = 0, length = keys.length; i < length; i++) { + + var conn = self.connections[keys[i]]; + + if (data === undefined) { + if (conn.type === 3) { + raw = true; + data = JSON.stringify(message, replacer); + } else + data = message; + } + + if (comparer && comparer(conn, message)) { + conn.send(data, raw); + F.stats.response.websocket++; + } + } + + return self; }; function websocket_valid_array(id, arr) { return arr.indexOf(id) !== -1; } -function websocket_valid_fn(id, client, fn) { - return fn && fn(id, client) ? true : false; +function websocket_valid_fn(id, client, fn, msg) { + return fn && fn(id, client, msg) ? true : false; } /** diff --git a/test/controllers/default.js b/test/controllers/default.js index a258d92aa..fa70af164 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -156,7 +156,7 @@ function plain_options() { function *synchronize() { var self = this; - var content = (yield sync(require('fs').readFile)(self.path.public('file.txt'))).toString('utf8'); + var content = (yield sync(require('fs').readFile)(PATH.public('file.txt'))).toString('utf8'); self.plain(content); } @@ -439,9 +439,9 @@ function viewIndex() { var self = this; var name = 'controller: '; - assert.ok(self.path.public('file.txt').endsWith('/public/file.txt'), name + 'path.public'); - assert.ok(self.path.logs('file.txt').endsWith('/file.txt'), name + 'path.logs'); - assert.ok(self.path.temp('file.txt').endsWith('/file.txt'), name + 'path.temp'); + assert.ok(PATH.public('file.txt').endsWith('/public/file.txt'), name + 'path.public'); + assert.ok(PATH.logs('file.txt').endsWith('/file.txt'), name + 'path.logs'); + assert.ok(PATH.temp('file.txt').endsWith('/file.txt'), name + 'path.temp'); self.meta('A', 'B'); assert.ok(self.repository['$title'] === 'A' && self.repository['$description'] === 'B', name + 'meta() - write'); @@ -458,7 +458,7 @@ function viewIndex() { assert.ok(framework.model('other/products').ok === 2, 'framework: model() - 2'); assert.ok(self.secured === false, 'controller.secured'); - assert.ok(self.config.isDefinition === true, 'definitions()'); + assert.ok(CONF.isDefinition === true, 'definitions()'); assert.ok(!self.xhr, name + 'xhr'); assert.ok(self.flags.indexOf('get') !== -1, name + 'flags') From 2b17995cee4e24e1f8a0c8a4b26cdd0d0b5e0a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 09:00:42 +0100 Subject: [PATCH 0786/1669] Fixed cache. --- index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index d6e95da78..c8dd04258 100755 --- a/index.js +++ b/index.js @@ -11355,7 +11355,7 @@ Controller.prototype.$view = function(name, model, expire, key) { if (expire) { cache = '$view.' + name + '.' + (key || ''); - var output = self.cache.read2(cache); + var output = F.cache.read2(cache); if (output) return output.body; } @@ -11365,7 +11365,7 @@ Controller.prototype.$view = function(name, model, expire, key) { if (!value) return ''; - expire && self.cache.add(cache, { components: value instanceof Function, body: value instanceof Function ? '' : value }, expire, false); + expire && F.cache.add(cache, { components: value instanceof Function, body: value instanceof Function ? '' : value }, expire, false); return value; }; @@ -13382,7 +13382,7 @@ Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { self.themeName && (key += '#' + self.themeName); - var output = self.cache.read2(key); + var output = F.cache.read2(key); if (!output) return self.$memorize_prepare(key, expires, disabled, fnTo, fnFrom); @@ -13473,7 +13473,7 @@ Controller.prototype.$memorize_prepare = function(key, expires, disabled, fnTo, } } - self.cache.add(key, options, expires, false); + F.cache.add(key, options, expires, false); self.precache = null; delete F.temporary.processing[pk]; }; From 032b45b680415dc861d9c5b1dc56b52b284febe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 20:07:47 +0100 Subject: [PATCH 0787/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0f956763a..f25f66df8 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-32", + "version": "3.0.1-33", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 541f93014ec2fcc64b8ac65d424550e674f12820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 1 Dec 2018 10:51:45 +0100 Subject: [PATCH 0788/1669] Fixed `controller.send2()`. --- index.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index c8dd04258..dada5d503 100755 --- a/index.js +++ b/index.js @@ -13721,10 +13721,11 @@ WebSocket.prototype.send2 = function(message, comparer, replacer) { data = message; } - if (comparer && comparer(conn, message)) { - conn.send(data, raw); - F.stats.response.websocket++; - } + if (comparer && !comparer(conn, message)) + continue; + + conn.send(data, raw); + F.stats.response.websocket++; } return self; From 0ee9046606d1dc27e615fe89d23b6e0548a8f288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Dec 2018 08:12:54 +0100 Subject: [PATCH 0789/1669] Updated startup info. --- changes.txt | 1 + index.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 22cd9476d..9135b9d66 100755 --- a/changes.txt +++ b/changes.txt @@ -41,6 +41,7 @@ - updated: `ROUTING(id:|search, [flags])` method - updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) - updated: `controller.all()` can return `Array` of all WebSocketClient +- updated: startup info by adding user name - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index dada5d503..74488abae 100755 --- a/index.js +++ b/index.js @@ -7036,12 +7036,13 @@ F.console = function() { console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); + console.log('User : ' + Os.userInfo().username); CONF.nosql_worker && console.log('NoSQL PID : ' + framework_nosql.pid()); console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); console.log('===================================================='); console.log('Name : ' + CONF.name); console.log('Version : ' + CONF.version); - console.log('Author : ' + CONF.author); + CONF.author && console.log('Author : ' + CONF.author); console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); console.log('Mode : ' + (DEBUG ? 'debug' : 'release')); console.log('===================================================='); From 49bcf4dcc864298cb894d4e056ae633fbfcdf0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Dec 2018 08:31:31 +0100 Subject: [PATCH 0790/1669] Updated ordering. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 74488abae..50bf2e2d3 100755 --- a/index.js +++ b/index.js @@ -7036,9 +7036,9 @@ F.console = function() { console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); - console.log('User : ' + Os.userInfo().username); CONF.nosql_worker && console.log('NoSQL PID : ' + framework_nosql.pid()); console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); + console.log('User : ' + Os.userInfo().username); console.log('===================================================='); console.log('Name : ' + CONF.name); console.log('Version : ' + CONF.version); From 1110d9e961f6d8fed8102cedbcbeaa1969d16e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 5 Dec 2018 08:57:32 +0100 Subject: [PATCH 0791/1669] Updated `LOCALIZE()`. --- changes.txt | 1 + index.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 9135b9d66..52f44a486 100755 --- a/changes.txt +++ b/changes.txt @@ -42,6 +42,7 @@ - updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) - updated: `controller.all()` can return `Array` of all WebSocketClient - updated: startup info by adding user name +- updated: `LOCALIZE()` now `url` arg can be a function which replaces `F.onLocale` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 50bf2e2d3..4850ed7cf 100755 --- a/index.js +++ b/index.js @@ -3393,6 +3393,11 @@ function sitemapurl(url) { global.LOCALIZE = F.localize = function(url, flags, minify) { + if (typeof(url) === 'function') { + F.onLocale = url; + return; + } + if (url[0] === '#') url = sitemapurl(url.substring(1)); @@ -3482,7 +3487,6 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { }); }, flags); - return F; }; F.$notModified = function(req, res, date) { From b90209d921e471338b220e64acd6a577fd044d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 7 Dec 2018 11:20:54 +0100 Subject: [PATCH 0792/1669] Updated version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f25f66df8..ce28533b2 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-33", + "version": "3.0.1", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 493525b08d253b1f54a861ad3b3d4990c63a3ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 7 Dec 2018 11:28:39 +0100 Subject: [PATCH 0793/1669] Updated version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ce28533b2..adfc97195 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1", + "version": "3.1.0", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 2bc41ec6e96c65e367b0481bc6d391f36618e444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Dec 2018 22:20:28 +0100 Subject: [PATCH 0794/1669] Fixed system routing. --- changes.txt | 4 ++++ index.js | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 52f44a486..97618a25c 100755 --- a/changes.txt +++ b/changes.txt @@ -1,3 +1,7 @@ +======= 3.2.0 + +- fixed: system routing + ======= 3.1.0 - added: CSS variables support default values `border-radius: $radius || 10px` diff --git a/index.js b/index.js index 4850ed7cf..a26c12ea1 100755 --- a/index.js +++ b/index.js @@ -21,7 +21,7 @@ /** * @module Framework - * @version 3.1.0 + * @version 3.2.0 */ 'use strict'; @@ -4880,11 +4880,19 @@ F.install_make = function(key, name, obj, options, callback, skipEmit, type) { var id = (type === 'module' ? '#' : '') + name; var length = F.routes.web.length; + for (var i = 0; i < length; i++) { if (F.routes.web[i].controller === routeID) F.routes.web[i].controller = id; } + var tmp = Object.keys(F.routes.system); + length = tmp.length; + for (var i = 0; i < length; i++) { + if (F.routes.system[tmp[i]].controller === routeID) + F.routes.system[tmp[i]].controller = id; + } + length = F.routes.websockets.length; for (var i = 0; i < length; i++) { if (F.routes.websockets[i].controller === routeID) From cafd921dc1fe3dae91d6c533e8cbe79b72ea0fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 18 Dec 2018 15:02:53 +0100 Subject: [PATCH 0795/1669] Improved Workers. --- changes.txt | 5 +++++ index.js | 22 +++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 97618a25c..b64d6651d 100755 --- a/changes.txt +++ b/changes.txt @@ -1,5 +1,10 @@ ======= 3.2.0 +- added: `WORKER()` alias to `F.worker()` +- added: `WORKER2()` alias to `F.worker2()` + +- updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` + - fixed: system routing ======= 3.1.0 diff --git a/index.js b/index.js index a26c12ea1..ddde8f0b0 100755 --- a/index.js +++ b/index.js @@ -252,7 +252,7 @@ HEADERS.binary['X-Powered-By'] = 'Total.js'; HEADERS.authorization = { user: '', password: '', empty: true }; HEADERS.fsStreamRead = { flags: 'r', mode: '0666', autoClose: true }; HEADERS.fsStreamReadRange = { flags: 'r', mode: '0666', autoClose: true, start: 0, end: 0 }; -HEADERS.workers = { cwd: '' }; +HEADERS.workers = { cwd: '', silent: true }; HEADERS.responseLocalize = {}; HEADERS.responseNotModified = {}; HEADERS.responseNotModified[HEADER_CACHE] = 'public, max-age=11111111'; @@ -645,8 +645,8 @@ function Framework() { var self = this; self.$id = null; // F.id ==> property - self.version = 3100; - self.version_header = '3.1.0'; + self.version = 3200; + self.version_header = '3.2.0'; self.version_node = process.version.toString(); self.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); @@ -9489,7 +9489,7 @@ F.accept = function(extension, contentType) { * @param {Array} args Additional arguments, optional. * @return {ChildProcess} */ -F.worker = function(name, id, timeout, args) { +global.WORKER = F.worker = function(name, id, timeout, args) { var fork = null; var type = typeof(id); @@ -9550,7 +9550,7 @@ F.worker = function(name, id, timeout, args) { return fork; }; -F.worker2 = function(name, args, callback, timeout) { +global.WORKER2 = F.worker2 = function(name, args, callback, timeout) { if (typeof(args) === 'function') { timeout = callback; @@ -9569,14 +9569,22 @@ F.worker2 = function(name, args, callback, timeout) { if (fork.__worker2) return fork; + var output = U.createBufferSize(0); + fork.__worker2 = true; fork.on('error', function(e) { - callback && callback(e); + callback && callback(e, output); callback = null; }); + fork.stdout.on('data', function(data) { + CONCAT[0] = output; + CONCAT[1] = data; + output = Buffer.concat(CONCAT); + }); + fork.on('exit', function() { - callback && callback(); + callback && callback(null, output); callback = null; }); From 16622d1fdb48e923a29f486bd6b8092669213fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 18 Dec 2018 15:03:19 +0100 Subject: [PATCH 0796/1669] Updated version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index adfc97195..508badf4c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.1.0", + "version": "3.2.0-1", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From f8de73be43928a1c0fc760a42df709c05a146e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 18 Dec 2018 15:06:20 +0100 Subject: [PATCH 0797/1669] Added `F.cluster.https()`. --- changes.txt | 1 + cluster.js | 24 +++++++++++++++++++----- index.js | 3 +++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index b64d6651d..d1fbe6aad 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: `WORKER()` alias to `F.worker()` - added: `WORKER2()` alias to `F.worker2()` +- added: `F.cluster.https()` - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` diff --git a/cluster.js b/cluster.js index 0c099d669..dab97d35b 100644 --- a/cluster.js +++ b/cluster.js @@ -158,6 +158,17 @@ exports.http = function(count, mode, options, callback) { fork(); }; +exports.https = function(count, mode, options, callback) { + // Fork will obtain options automatically via event + if (Cluster.isMaster) { + CLUSTER_REQ.id = 'master'; + CLUSTER_RES.id = 'master'; + CLUSTER_EMIT.id = 'master'; + master(count, mode, options, callback, true); + } else + fork(); +}; + exports.restart = function(index) { if (index === undefined) { for (var i = 0; i < THREADS; i++) @@ -173,7 +184,7 @@ exports.restart = function(index) { } }; -function master(count, mode, options, callback) { +function master(count, mode, options, callback, https) { if (count == null || count === 'auto') count = require('os').cpus().length; @@ -222,7 +233,7 @@ function master(count, mode, options, callback) { }; count.async(function(i, next) { - exec(Math.abs(i - THREADS)); + exec(Math.abs(i - THREADS), https); can(next); }, function() { callback && callback(FORKS); @@ -258,7 +269,7 @@ function mastersend(m) { FORKS[i] && FORKS[i].send(m); } -function exec(index) { +function exec(index, https) { var fork = Cluster.fork(); fork.$id = index.toString(); fork.on('message', message); @@ -272,7 +283,7 @@ function exec(index) { (function(fork) { setTimeout(function() { OPTIONS.options.id = fork.$id; - fork.send({ TYPE: 'init', bundling: !CONTINUE, id: fork.$id, mode: OPTIONS.mode, options: OPTIONS.options, threads: OPTIONS.count, index: index }); + fork.send({ TYPE: 'init', bundling: !CONTINUE, id: fork.$id, mode: OPTIONS.mode, options: OPTIONS.options, threads: OPTIONS.count, index: index, https: https }); }, fork.$id * 500); })(fork); } @@ -290,7 +301,10 @@ function on_init(msg) { CLUSTER_RES.id = msg.id; THREADS = msg.threads; msg.options.bundling = msg.bundling; - F.http(msg.mode, msg.options); + if (msg.https) + F.https(msg.mode, msg.options); + else + F.http(msg.mode, msg.options); F.isCluster = true; F.removeListener(msg.TYPE, on_init); break; diff --git a/index.js b/index.js index ddde8f0b0..2139c4251 100755 --- a/index.js +++ b/index.js @@ -6920,6 +6920,9 @@ F.https = function(mode, options, middleware) { options == null && (options = {}); !options.port && (options.port = +process.argv[2]); + if (options.port && isNaN(options.port)) + options.port = 0; + if (typeof(middleware) === 'function') options.middleware = middleware; From 06436b67361b773f6131bf3b1df3e71cd7a034b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 20 Dec 2018 09:35:39 +0100 Subject: [PATCH 0798/1669] Fixed a strange bug with sorting in NoSQL. --- changes.txt | 1 + nosql.js | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index d1fbe6aad..f07fd6af1 100755 --- a/changes.txt +++ b/changes.txt @@ -7,6 +7,7 @@ - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` - fixed: system routing +- fixed: NoSQL sorting, solved a strange problem ======= 3.1.0 diff --git a/nosql.js b/nosql.js index cebc7b96e..4e601de95 100755 --- a/nosql.js +++ b/nosql.js @@ -6993,13 +6993,13 @@ NoSQLReader.prototype.callback = function(item) { } if (item.count) { - if (opt.sort.name) { + + if (opt.sort === null) + item.response.random(); + else if (opt.sort.name) { if (!builder.$inlinesort || opt.take !== item.response.length) item.response.quicksort(opt.sort.name, opt.sort.asc); - } else if (opt.sort === null) - item.response.random(); - else - item.response.sort(opt.sort); + } if (opt.skip && opt.take) item.response = item.response.splice(opt.skip, opt.take); From 188196eadf72e1f11e4a59208be224992f40464f Mon Sep 17 00:00:00 2001 From: Khaled Date: Thu, 27 Dec 2018 14:04:45 +0200 Subject: [PATCH 0799/1669] Added condition to exclude GET method Adding a body to GET requests may lead to problems with some servers. --- utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index a2131098f..7aa4ec253 100755 --- a/utils.js +++ b/utils.js @@ -607,7 +607,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined)) + if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined) && options.post) data = BUFEMPTYJSON; } @@ -6215,4 +6215,4 @@ exports.reader = function() { const BUFEMPTYJSON = exports.createBuffer('{}'); global.WAIT = exports.wait; -!global.F && require('./index'); \ No newline at end of file +!global.F && require('./index'); From 9841b7d6c69a3df97ae5186be3734b369d17dded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 1 Jan 2019 11:03:41 +0100 Subject: [PATCH 0800/1669] Updated `@{options()}` by adding `disabled`. --- changes.txt | 1 + index.js | 22 ++++++++++++++++------ package.json | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index f07fd6af1..44877c462 100755 --- a/changes.txt +++ b/changes.txt @@ -5,6 +5,7 @@ - added: `F.cluster.https()` - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` +- updated: `$options()` by adding `disabled` key - fixed: system routing - fixed: NoSQL sorting, solved a strange problem diff --git a/index.js b/index.js index 2139c4251..ac91dacc8 100755 --- a/index.js +++ b/index.js @@ -11817,7 +11817,7 @@ Controller.prototype.$isValue = function(bool, charBeg, charEnd, value) { return charBeg + value + charEnd; }; -Controller.prototype.$options = function(arr, selected, name, value) { +Controller.prototype.$options = function(arr, selected, name, value, disabled) { var type = typeof(arr); if (!arr) @@ -11832,7 +11832,7 @@ Controller.prototype.$options = function(arr, selected, name, value) { arr = Object.keys(arr); } - if (!U.isArray(arr)) + if (!(arr instanceof Array)) arr = [arr]; selected = selected || ''; @@ -11840,10 +11840,12 @@ Controller.prototype.$options = function(arr, selected, name, value) { var options = ''; if (!isObject) { - if (value === undefined) + if (value == null) value = value || name || 'value'; - if (name === undefined) - name = name || 'name'; + if (name == null) + name = 'name'; + if (disabled == null) + disabled = 'disabled'; } var isSelected = false; @@ -11858,6 +11860,7 @@ Controller.prototype.$options = function(arr, selected, name, value) { var text = ''; var val = ''; var sel = false; + var dis = false; if (isObject) { if (name === true) { @@ -11883,6 +11886,13 @@ Controller.prototype.$options = function(arr, selected, name, value) { if (typeof(val) === 'function') val = val(i, text); + dis = o[disabled]; + + if (typeof(disabled) === 'function') + dis = disabled(i, val, text); + else + dis = dis ? true : false; + } else { text = o; val = o; @@ -11893,7 +11903,7 @@ Controller.prototype.$options = function(arr, selected, name, value) { isSelected = sel; } - options += ''; + options += ''; } return options; diff --git a/package.json b/package.json index 508badf4c..80d0fbd12 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.2.0-1", + "version": "3.2.0-2", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 3ce3a79318246a6e3e1de636ea855ab9caddc7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0irka?= Date: Tue, 1 Jan 2019 11:24:51 +0100 Subject: [PATCH 0801/1669] Revert "Added condition to exclude body from GET requests" --- utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 7aa4ec253..a2131098f 100755 --- a/utils.js +++ b/utils.js @@ -607,7 +607,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined) && options.post) + if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined)) data = BUFEMPTYJSON; } @@ -6215,4 +6215,4 @@ exports.reader = function() { const BUFEMPTYJSON = exports.createBuffer('{}'); global.WAIT = exports.wait; -!global.F && require('./index'); +!global.F && require('./index'); \ No newline at end of file From f1ba5f9c7ee198fa61c3a92a8f5017d00d744051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 1 Jan 2019 11:27:34 +0100 Subject: [PATCH 0802/1669] Fixed `U.request()` for `GET` requests. --- changes.txt | 1 + package.json | 2 +- utils.js | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 44877c462..32644b857 100755 --- a/changes.txt +++ b/changes.txt @@ -9,6 +9,7 @@ - fixed: system routing - fixed: NoSQL sorting, solved a strange problem +- fixed: `U.request()` with `GET` method by @khaledkhalil94 (it doesn't send JSON data if `data` is null/undefined) ======= 3.1.0 diff --git a/package.json b/package.json index 80d0fbd12..6e2ef7ec6 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.2.0-2", + "version": "3.2.0-3", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/utils.js b/utils.js index a2131098f..1eca6fea2 100755 --- a/utils.js +++ b/utils.js @@ -597,6 +597,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, method = 'GET'; if (type < 3) { + if (typeof(data) !== 'string') data = type === 1 ? JSON.stringify(data) : Qs.stringify(data); else if (data[0] === '?') @@ -607,7 +608,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined)) + if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined) && options.post) data = BUFEMPTYJSON; } From 801e36eef769363c8a12c4b0037231d7c8f5b7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 2 Jan 2019 10:23:40 +0100 Subject: [PATCH 0803/1669] Extended `TaskBuilder` methods. --- builders.js | 34 ++++++++++++++++++++++++++++++++++ changes.txt | 3 +++ 2 files changed, 37 insertions(+) diff --git a/builders.js b/builders.js index a4e187727..3ac5bb41b 100755 --- a/builders.js +++ b/builders.js @@ -5312,6 +5312,20 @@ TaskBuilder.prototype.next = function() { return self; }; +TaskBuilder.prototype.next2 = function(name) { + var self = this; + return function(err) { + if (err) + self.invalid(err); + else { + if (name == null) + self.done(); + else + self.next(name); + } + }; +}; + TaskBuilder.prototype.done = function(data) { var self = this; self.$callback && self.$callback(self.error && self.error.is ? self.error : null, data || self.value); @@ -5319,10 +5333,30 @@ TaskBuilder.prototype.done = function(data) { return self; }; +TaskBuilder.prototype.done2 = function(send_value) { + var self = this; + return function(err, data) { + if (err) + self.invalid(err); + else + self.done(send_value ? data : null); + }; +}; + TaskBuilder.prototype.success = function(data) { return this.done(SUCCESS(true, data)); }; +TaskBuilder.prototype.success2 = function(send_value) { + var self = this; + return function(err, data) { + if (err) + self.invalid(err); + else + self.done(SUCCESS(true, send_value ? data : null)); + }; +}; + TaskBuilder.prototype.callback = function(fn) { var self = this; self.$callback = fn; diff --git a/changes.txt b/changes.txt index 32644b857..beb7580e0 100755 --- a/changes.txt +++ b/changes.txt @@ -3,6 +3,9 @@ - added: `WORKER()` alias to `F.worker()` - added: `WORKER2()` alias to `F.worker2()` - added: `F.cluster.https()` +- added: `TaskBuilder.done2([send_value])` returns `function` with wrapped `.done()` +- added: `TaskBuilder.success2([send_value])` returns `function` with wrapped `.success()` +- added: `TaskBuilder.next2(name)` returns `function` with wrapped `.next()` - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` - updated: `$options()` by adding `disabled` key From 0b2d64a0a0caaeb5418b4768d9303a6020740e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 4 Jan 2019 10:59:07 +0100 Subject: [PATCH 0804/1669] Removed useless REG EXP. --- utils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/utils.js b/utils.js index 1eca6fea2..e25b16ef3 100755 --- a/utils.js +++ b/utils.js @@ -62,7 +62,6 @@ const regexpINTEGER = /(^-|\s-)?[0-9]+/g; const regexpFLOAT = /(^-|\s-)?[0-9.,]+/g; const regexpALPHA = /^[A-Za-z0-9]+$/; const regexpSEARCH = /[^a-zA-Zá-žÁ-Ž\d\s:]/g; -const regexpUNICODE = /\\u([\d\w]{4})/gi; const regexpTERMINAL = /[\w\S]+/g; const regexpY = /y/g; const regexpN = /\n/g; From 667de1d2c2b0db3b720a149706525193e40bfc3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 4 Jan 2019 12:28:18 +0100 Subject: [PATCH 0805/1669] Added markdown parser for testing. --- test/test-tmp.js | 291 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 276 insertions(+), 15 deletions(-) diff --git a/test/test-tmp.js b/test/test-tmp.js index 97fa07942..aaac8a3c2 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,20 +1,281 @@ require('../index'); -// NOSQLMEMORY('test'); -// NOSQL('test').find2().take(10).callback(console.log); -// NOSQL('test').count().callback(console.log); -// NOSQL('test').remove().between('index', 0, 10).callback(console.log).backup('fet'); -// NOSQL('test').remove().between('index', 1, 3).callback(console.log); -// NOSQL('test').on('modify', console.log); -// NOSQL('test').update({ name: GUID(5) }).between('index', 1, 3).callback(console.log); -CONF['table.test'] = 'index:number | name:string'; +var md = `# Peter je kráľ -TABLE('test').find().take(10).skip(10).callback(console.log); +asdljasdlasjdlaj lsajd lsajd lsajskl dsalk jklsa -// TABLE('test').modify({ name: GUID(5) }).between('index', 3, 5).callback(console.log); -// TABLE('test').remove().between('index', 0, 2).callback(console.log); +- a +- b +- c + - d + - e + - f +- g +- h +`; -/* -for (var i = 0; i < 100; i++) - TABLE('test').insert({ index: i }); -*/ \ No newline at end of file +(function Markdown() { + + var links = /(!)?\[.*?\]\(.*?\)/g; + var imagelinks = /\[!\[.*?\]\(.*?\)\]\(.*?\)/g; + var format = /__.*?__|_.*?_|\*\*.*?\*\*|\*.*?\*|~~.*?~~|~.*?~/g; + var ordered = /^([a-z|0-9]{1,2}\.\s)|-\s/i; + var orderedsize = /^(\s|\t)+/; + var code = /`.*?`/g; + var encodetags = /<|>/g; + var formatclean = /_|\*|~/g; + var regid = /[^\w]+/g; + var regdash = /-{2,}/g; + var regtags = /<\/?[^>]+(>|$)/g; + + var encode = function(val) { + return '&' + (val === '<' ? 'lt' : 'gt') + ';'; + }; + + function markdown_code(value) { + return '' + value.substring(1, value.length - 1) + ''; + } + + function markdown_imagelinks(value) { + var end = value.indexOf(')') + 1; + var img = value.substring(1, end); + return '' + markdown_links(img) + ''; + } + + function markdown_table(value, align, ishead) { + + var columns = value.substring(1, value.length - 1).split('|'); + var builder = ''; + + for (var i = 0; i < columns.length; i++) { + var column = columns[i].trim(); + if (column.charAt(0) == '-') + continue; + var a = align[i]; + builder += '<' + (ishead ? 'th' : 'td') + (a && a !== 'left' ? (' class="' + a + '"') : '') + '>' + column + ''; + } + + return '' + builder + ''; + } + + function markdown_links(value) { + var end = value.lastIndexOf(']'); + var img = value.charAt(0) === '!'; + var text = value.substring(img ? 2 : 1, end); + var link = value.substring(end + 2, value.length - 1); + return img ? ('' + text + '') : ('' + text + ''); + } + + function markdown_format(value) { + switch (value[0]) { + case '_': + return '' + value.replace(formatclean, '') + ''; + case '*': + return '' + value.replace(formatclean, '') + ''; + case '~': + return '' + value.replace(formatclean, '') + ''; + } + return value; + } + + function markdown_id(value) { + + var end = ''; + var beg = ''; + + if (value.charAt(0) === '<') + beg = '-'; + + if (value.charAt(value.length - 1) === '>') + end = '-'; + + return (beg + value.replace(regtags, '').toLowerCase().replace(regid, '-') + end).replace(regdash, '-'); + } + + String.prototype.markdown = function() { + var lines = this.split('\n'); + var builder = []; + var ul = []; + var table = false; + var iscode = false; + var ishead = false; + var prev; + var prevsize = 0; + var tmp; + + for (var i = 0, length = lines.length; i < length; i++) { + + lines[i] = lines[i].replace(encodetags, encode); + + if (lines[i].substring(0, 3) === '```') { + + if (iscode) { + builder.push(''); + iscode = false; + continue; + } + + iscode = true; + builder.push('
');
+				prev = 'code';
+				continue;
+			}
+
+			if (iscode) {
+				builder.push(lines[i]);
+				continue;
+			}
+
+			var line = lines[i].replace(imagelinks, markdown_imagelinks).replace(links, markdown_links).replace(format, markdown_format).replace(code, markdown_code);
+			if (!line) {
+				if (table) {
+					table = null;
+					builder.push('');
+				}
+			}
+
+			if (line === '' && lines[i - 1] === '') {
+				builder.push('
'); + prev = 'br'; + continue; + } + + if (line[0] === '|') { + if (!table) { + var next = lines[i + 1]; + if (next[0] === '|') { + table = []; + var columns = next.substring(1, next.length - 1).split('|'); + for (var j = 0; j < columns.length; j++) { + var column = columns[j].trim(); + var align = 'left'; + if (column.charAt(column.length - 1) === ':') + align = column[0] === ':' ? 'center' : 'right'; + table.push(align); + } + builder.push(''); + prev = 'table'; + ishead = true; + i++; + } else + continue; + } + + if (ishead) + builder.push(markdown_table(line, table, true) + ''); + else + builder.push(markdown_table(line, table)); + ishead = false; + continue; + } + + if (line.charAt(0) === '#') { + + if (line.substring(0, 2) === '# ') { + tmp = line.substring(2).trim(); + builder.push('

' + tmp + '

'); + prev = '#'; + continue; + } + + if (line.substring(0, 3) === '## ') { + tmp = line.substring(3).trim(); + builder.push('

' + tmp + '

'); + prev = '##'; + continue; + } + + if (line.substring(0, 4) === '### ') { + tmp = line.substring(4).trim(); + builder.push('

' + tmp + '

'); + prev = '###'; + continue; + } + + if (line.substring(0, 5) === '#### ') { + tmp = line.substring(5).trim(); + builder.push('

' + tmp + '

'); + prev = '####'; + continue; + } + + if (line.substring(0, 6) === '##### ') { + tmp = line.substring(6).trim(); + builder.push('
' + tmp + '
'); + prev = '#####'; + continue; + } + } + + var tmp = line.substring(0, 3); + + if (tmp === '---' || tmp === '***') { + prev = 'hr'; + builder.push('
'); + continue; + } + + if (line[0] === '>' && line.substring(0, 2) === '> ') { + builder.push('
' + line.substring(2).trim() + '
'); + prev = '>'; + continue; + } + + var tmpline = line.trim(); + + if (ordered.test(tmpline)) { + + var size = line.match(orderedsize); + if (size) + size = size[0].length; + else + size = 0; + + var append = false; + + if (prevsize !== size) { + // NESTED + if (size > prevsize) { + prevsize = size; + append = true; + var index = builder.length - 1; + builder[index] = builder[index].substring(0, builder[index].length - 5); + prev = ''; + } else { + // back to normal + prevsize = size; + builder.push(''); + } + } + + var type = tmpline.charAt(0) === '-' ? 'ul' : 'ol'; + if (prev !== type) { + var subtype; + if (type === 'ol') + subtype = tmpline.charAt(0); + builder.push('<' + type + (subtype ? (' type="' + subtype + '"') : '') + '>'); + ul.push(type + (append ? '>' + (type === 'ol' ? tmpline.substring(tmpline.indexOf('.') + 1) : tmpline.substring(2)).trim() + ''); + + } else { + ul.length && builder.push(''); + line && builder.push('

' + line.trim() + '

'); + prev = 'p'; + } + } + + for (var i = 0; i < ul.length; i++) + builder.push(''); + + table && builder.push('
'); + iscode && builder.push('
'); + + return '
' + builder.join('\n') + '
'; + }; +})(); + +console.log(md.markdown()); \ No newline at end of file From e9e43974cc8faaaa581c58da807750fbdf542042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 5 Jan 2019 22:35:21 +0100 Subject: [PATCH 0806/1669] Improved `String.ROOT()`. --- changes.txt | 1 + utils.js | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index beb7580e0..4f6499242 100755 --- a/changes.txt +++ b/changes.txt @@ -9,6 +9,7 @@ - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` - updated: `$options()` by adding `disabled` key +- updated: `String.ROOT()` by adding a support for jComponent `AJAX()` calls - fixed: system routing - fixed: NoSQL sorting, solved a strange problem diff --git a/utils.js b/utils.js index e25b16ef3..5ee10a6fa 100755 --- a/utils.js +++ b/utils.js @@ -87,6 +87,7 @@ const PROXYOPTIONSHTTP = {}; const REG_ROOT = /@\{#\}(\/)?/g; const REG_NOREMAP = /@\{noremap\}(\n)?/g; const REG_REMAP = /href=".*?"|src=".*?"/gi; +const REG_AJAX = /('|")+(!)?(GET|POST|PUT|DELETE|PATH)\s(\(.*?\)\s)?\//g; const REG_URLEXT = /(https|http|wss|ws|file):\/\/|\/\/[a-z0-9]|[a-z]:/i; const REG_TEXTAPPLICATION = /text|application/i; @@ -3024,7 +3025,7 @@ SP.ROOT = function(noremap) { }).replace(REG_ROOT, $urlmaker); if (!noremap && CONF.default_root) - str = str.replace(REG_REMAP, $urlremap); + str = str.replace(REG_REMAP, $urlremap).replace(REG_AJAX, $urlajax); return str; }; @@ -3034,6 +3035,10 @@ function $urlremap(text) { return REG_URLEXT.test(text) ? text : ((text[0] === 'h' ? 'href' : 'src') + '="' + CONF.default_root + (text[pos] === '/' ? text.substring(pos + 1) : text)); } +function $urlajax(text) { + return text.substring(0, text.length - 1) + CONF.default_root; +} + function $urlmaker(text) { var c = text[4]; return CONF.default_root ? CONF.default_root : (c || ''); From aa01faac4846cf04e7ec23b8b626587c34ca0768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 7 Jan 2019 09:46:53 +0100 Subject: [PATCH 0807/1669] Improved `WORKER2()`. --- index.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index ac91dacc8..5ba0da970 100755 --- a/index.js +++ b/index.js @@ -7270,6 +7270,9 @@ F.service = function(count) { CONF.allow_debug && F.consoledebug('gc()'); }, 1000); + if (WORKERID > 9999999999) + WORKERID = 0; + // Run schedules if (!F.schedules.length) return F; @@ -9484,6 +9487,10 @@ F.accept = function(extension, contentType) { return F; }; +// A temporary variable for generating Worker ID +// It's faster than Date.now() +var WORKERID = 0; + /** * Run worker * @param {String} name @@ -9528,7 +9535,7 @@ global.WORKER = F.worker = function(name, id, timeout, args) { fork = Child.fork(filename[filename.length - 3] === '.' ? filename : filename + '.js', args, HEADERS.workers); if (!id) - id = name + '_' + new Date().getTime(); + id = name + '_' + (WORKERID++); fork.__id = id; F.workers[id] = fork; @@ -9568,7 +9575,7 @@ global.WORKER2 = F.worker2 = function(name, args, callback, timeout) { if (args && !(args instanceof Array)) args = [args]; - var fork = F.worker(name, name, timeout, args); + var fork = F.worker(name, 'worker' + (WORKERID++), timeout, args); if (fork.__worker2) return fork; From 7779027c630382214fa9d5c174e7b97213cf2242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 12 Jan 2019 22:52:55 +0100 Subject: [PATCH 0808/1669] Fixed `F.wait()` in WebSocket. --- changes.txt | 1 + index.js | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 4f6499242..d233cba3a 100755 --- a/changes.txt +++ b/changes.txt @@ -14,6 +14,7 @@ - fixed: system routing - fixed: NoSQL sorting, solved a strange problem - fixed: `U.request()` with `GET` method by @khaledkhalil94 (it doesn't send JSON data if `data` is null/undefined) +- fixed: `F.wait()` in WebSocket ======= 3.1.0 diff --git a/index.js b/index.js index ac91dacc8..18df615b7 100755 --- a/index.js +++ b/index.js @@ -7701,7 +7701,7 @@ F.$cors = function(req, res, fn, arg) { */ F.$upgrade = function(req, socket, head) { - if ((req.headers.upgrade || '').toLowerCase() !== 'websocket') + if ((req.headers.upgrade || '').toLowerCase() !== 'websocket' || F._length_wait) return; // disables timeout diff --git a/package.json b/package.json index 6e2ef7ec6..51693f9ab 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.2.0-3", + "version": "3.2.0-4", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 0c8159f50394f14ab255ec2c33c22b4adaaa1cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 13 Jan 2019 23:53:09 +0100 Subject: [PATCH 0809/1669] Added new RESTBuilder aliases. --- builders.js | 84 +++++++++++++++++++++++++++-------------------------- changes.txt | 1 + 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/builders.js b/builders.js index 3ac5bb41b..61c3eb4d6 100755 --- a/builders.js +++ b/builders.js @@ -4286,7 +4286,9 @@ RESTBuilder.setDefaultTransform = function(name) { delete transforms['restbuilder_default']; }; -RESTBuilder.prototype.promise = function(fn) { +var RESTP = RESTBuilder.prototype; + +RESTP.promise = function(fn) { var self = this; return new Promise(function(resolve, reject) { self.exec(function(err, result) { @@ -4298,24 +4300,24 @@ RESTBuilder.prototype.promise = function(fn) { }); }; -RESTBuilder.prototype.proxy = function(value) { +RESTP.proxy = function(value) { this.$proxy = value; return this; }; -RESTBuilder.prototype.setTransform = function(name) { +RESTP.setTransform = function(name) { this.$transform = name; return this; }; -RESTBuilder.prototype.url = function(url) { +RESTP.url = function(url) { if (url === undefined) return this.$url; this.$url = url; return this; }; -RESTBuilder.prototype.file = function(name, filename, buffer) { +RESTP.file = function(name, filename, buffer) { var obj = { name: name, filename: filename, buffer: buffer }; if (this.$files) this.$files.push(obj); @@ -4324,7 +4326,7 @@ RESTBuilder.prototype.file = function(name, filename, buffer) { return this; }; -RESTBuilder.prototype.maketransform = function(obj, data) { +RESTP.maketransform = function(obj, data) { if (this.$transform) { var fn = transforms['restbuilder'][this.$transform]; return fn ? fn(obj, data) : obj; @@ -4332,67 +4334,67 @@ RESTBuilder.prototype.maketransform = function(obj, data) { return obj; }; -RESTBuilder.prototype.timeout = function(number) { +RESTP.timeout = function(number) { this.$timeout = number; return this; }; -RESTBuilder.prototype.maxlength = function(number) { +RESTP.maxlength = function(number) { this.$length = number; this.$flags = null; return this; }; -RESTBuilder.prototype.auth = function(user, password) { +RESTP.auth = function(user, password) { this.$headers['authorization'] = 'Basic ' + framework_utils.createBuffer(user + ':' + password).toString('base64'); return this; }; -RESTBuilder.prototype.schema = function(group, name) { +RESTP.schema = function(group, name) { this.$schema = exports.getschema(group, name); if (!this.$schema) throw Error('RESTBuilder: Schema "{0}" not found.'.format(name ? (group + '/' + name) : group)); return this; }; -RESTBuilder.prototype.noDnsCache = function() { +RESTP.noDnsCache = function() { this.$nodnscache = true; this.$flags = null; return this; }; -RESTBuilder.prototype.noCache = function() { +RESTP.noCache = function() { this.$nocache = true; return this; }; -RESTBuilder.prototype.make = function(fn) { +RESTP.make = function(fn) { fn.call(this, this); return this; }; -RESTBuilder.prototype.xhr = function() { +RESTP.xhr = function() { this.$headers['X-Requested-With'] = 'XMLHttpRequest'; return this; }; -RESTBuilder.prototype.method = function(method) { +RESTP.method = function(method) { this.$method = method.toLowerCase(); this.$flags = null; return this; }; -RESTBuilder.prototype.referer = RESTBuilder.prototype.referrer = function(value) { +RESTP.referer = RESTP.referrer = function(value) { this.$headers['Referer'] = value; return this; }; -RESTBuilder.prototype.origin = function(value) { +RESTP.origin = function(value) { this.$headers['Origin'] = value; return this; }; -RESTBuilder.prototype.robot = function() { +RESTP.robot = function() { if (this.$headers['User-Agent']) this.$headers['User-Agent'] += ' Bot'; else @@ -4400,7 +4402,7 @@ RESTBuilder.prototype.robot = function() { return this; }; -RESTBuilder.prototype.mobile = function() { +RESTP.mobile = function() { if (this.$headers['User-Agent']) this.$headers['User-Agent'] += ' iPhone'; else @@ -4408,7 +4410,7 @@ RESTBuilder.prototype.mobile = function() { return this; }; -RESTBuilder.prototype.put = function(data) { +RESTP.put = RESTP.PUT = function(data) { if (this.$method !== 'put') { this.$flags = null; this.$method = 'put'; @@ -4418,7 +4420,7 @@ RESTBuilder.prototype.put = function(data) { return this; }; -RESTBuilder.prototype.delete = function(data) { +RESTP.delete = RESTP.DELETE = function(data) { if (this.$method !== 'delete') { this.$flags = null; this.$method = 'delete'; @@ -4428,7 +4430,7 @@ RESTBuilder.prototype.delete = function(data) { return this; }; -RESTBuilder.prototype.get = function(data) { +RESTP.get = RESTP.GET = function(data) { if (this.$method !== 'get') { this.$flags = null; this.$method = 'get'; @@ -4437,7 +4439,7 @@ RESTBuilder.prototype.get = function(data) { return this; }; -RESTBuilder.prototype.post = function(data) { +RESTP.post = RESTP.POST = function(data) { if (this.$method !== 'post') { this.$flags = null; this.$method = 'post'; @@ -4447,7 +4449,7 @@ RESTBuilder.prototype.post = function(data) { return this; }; -RESTBuilder.prototype.patch = function(data) { +RESTP.patch = RESTP.PATCH = function(data) { if (this.$method !== 'patch') { this.$flags = null; this.$method = 'patch'; @@ -4457,7 +4459,7 @@ RESTBuilder.prototype.patch = function(data) { return this; }; -RESTBuilder.prototype.json = function(data) { +RESTP.json = function(data) { if (this.$type !== 1) this.$flags = null; @@ -4471,7 +4473,7 @@ RESTBuilder.prototype.json = function(data) { return this; }; -RESTBuilder.prototype.urlencoded = function(data) { +RESTP.urlencoded = function(data) { if (this.$type !== 2) this.$flags = null; @@ -4484,7 +4486,7 @@ RESTBuilder.prototype.urlencoded = function(data) { return this; }; -RESTBuilder.prototype.accept = function(ext) { +RESTP.accept = function(ext) { var type; @@ -4502,7 +4504,7 @@ RESTBuilder.prototype.accept = function(ext) { return this; }; -RESTBuilder.prototype.xml = function(data) { +RESTP.xml = function(data) { if (this.$type !== 3) this.$flags = null; @@ -4515,54 +4517,54 @@ RESTBuilder.prototype.xml = function(data) { return this; }; -RESTBuilder.prototype.redirect = function(value) { +RESTP.redirect = function(value) { this.$redirect = value; return this; }; -RESTBuilder.prototype.raw = function(value) { +RESTP.raw = function(value) { this.$data = value && value.$clean ? value.$clean() : value; return this; }; -RESTBuilder.prototype.plain = function() { +RESTP.plain = function() { this.$plain = true; return this; }; -RESTBuilder.prototype.cook = function(value) { +RESTP.cook = function(value) { this.$flags = null; this.$persistentcookies = value !== false; return this; }; -RESTBuilder.prototype.cookies = function(obj) { +RESTP.cookies = function(obj) { this.$cookies = obj; return this; }; -RESTBuilder.prototype.cookie = function(name, value) { +RESTP.cookie = function(name, value) { !this.$cookies && (this.$cookies = {}); this.$cookies[name] = value; return this; }; -RESTBuilder.prototype.header = function(name, value) { +RESTP.header = function(name, value) { this.$headers[name] = value; return this; }; -RESTBuilder.prototype.type = function(value) { +RESTP.type = function(value) { this.$headers['Content-Type'] = value; return this; }; -RESTBuilder.prototype.cache = function(expire) { +RESTP.cache = function(expire) { this.$cache_expire = expire; return this; }; -RESTBuilder.prototype.set = function(name, value) { +RESTP.set = function(name, value) { if (!this.$data) this.$data = {}; if (typeof(name) !== 'object') { @@ -4575,13 +4577,13 @@ RESTBuilder.prototype.set = function(name, value) { return this; }; -RESTBuilder.prototype.rem = function(name) { +RESTP.rem = function(name) { if (this.$data && this.$data[name]) this.$data[name] = undefined; return this; }; -RESTBuilder.prototype.stream = function(callback) { +RESTP.stream = function(callback) { var self = this; var flags = self.$flags ? self.$flags : [self.$method]; @@ -4602,7 +4604,7 @@ RESTBuilder.prototype.stream = function(callback) { return U.download(self.$url, flags, self.$data, callback, self.$cookies, self.$headers, undefined, self.$timeout); }; -RESTBuilder.prototype.exec = function(callback) { +RESTP.exec = function(callback) { if (!callback) callback = NOOP; diff --git a/changes.txt b/changes.txt index d233cba3a..d86841dd6 100755 --- a/changes.txt +++ b/changes.txt @@ -6,6 +6,7 @@ - added: `TaskBuilder.done2([send_value])` returns `function` with wrapped `.done()` - added: `TaskBuilder.success2([send_value])` returns `function` with wrapped `.success()` - added: `TaskBuilder.next2(name)` returns `function` with wrapped `.next()` +- added: new `RESTBuilder` aliases `.DELETE()`, `.PUT()`, `.POST()`, `.PATCH() and `.GET()` - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` - updated: `$options()` by adding `disabled` key From 6188c0d32b21981724d03b5a1fc1c5a85dab1d74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 Jan 2019 13:44:45 +0100 Subject: [PATCH 0810/1669] Fixed `String.capitalize()`. --- changes.txt | 1 + utils.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index d86841dd6..7a757c38e 100755 --- a/changes.txt +++ b/changes.txt @@ -16,6 +16,7 @@ - fixed: NoSQL sorting, solved a strange problem - fixed: `U.request()` with `GET` method by @khaledkhalil94 (it doesn't send JSON data if `data` is null/undefined) - fixed: `F.wait()` in WebSocket +- fixed: `String.capitalize(true)` ======= 3.1.0 diff --git a/utils.js b/utils.js index 5ee10a6fa..fe71ad654 100755 --- a/utils.js +++ b/utils.js @@ -3751,7 +3751,7 @@ SP.parseFloat = function(def) { SP.capitalize = function(first) { if (first) - return this[0].toUpperCase() + this.substring(1); + return (this[0] || '').toUpperCase() + this.substring(1); var builder = ''; var c; From c5db7ddd81a62334a95897a6f60466cfc21b4f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 Jan 2019 13:46:11 +0100 Subject: [PATCH 0811/1669] Extend `RESTBuilder.method()`. --- builders.js | 3 ++- changes.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 61c3eb4d6..1d1c1efd8 100755 --- a/builders.js +++ b/builders.js @@ -4378,9 +4378,10 @@ RESTP.xhr = function() { return this; }; -RESTP.method = function(method) { +RESTP.method = function(method, data) { this.$method = method.toLowerCase(); this.$flags = null; + data && this.raw(data); return this; }; diff --git a/changes.txt b/changes.txt index 7a757c38e..473b1d09b 100755 --- a/changes.txt +++ b/changes.txt @@ -11,6 +11,7 @@ - updated: `F.worker2()` returns entire `stdout` buffer in the `callback(err, buffer)` - updated: `$options()` by adding `disabled` key - updated: `String.ROOT()` by adding a support for jComponent `AJAX()` calls +- updated: `RESTBuilder.method(method, [data])` added `data` argument - fixed: system routing - fixed: NoSQL sorting, solved a strange problem From 0ac5d8f724cd92b3a136f41cf5042873646a8d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 Jan 2019 23:48:45 +0100 Subject: [PATCH 0812/1669] Improved `process.env.NODE_TLS_REJECT_UNAUTHORIZED`. --- index.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 6f3d5a028..601d8a8be 100755 --- a/index.js +++ b/index.js @@ -6660,6 +6660,9 @@ global.LOAD = F.load = function(debug, types, pwd) { F.cache.init(); EMIT('init'); + if (CONF.allow_ssc_validation === false) + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + F.$load(types, directory, function() { F.isLoaded = true; @@ -6813,6 +6816,10 @@ F.initialize = function(http, debug, options) { // clears static files F.consoledebug('clear temporary'); F.clear(function() { + + if (CONF.allow_ssc_validation === false) + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + F.consoledebug('clear temporary (done)'); F.$load(undefined, directory, function() { @@ -9177,9 +9184,6 @@ F.$configure_configs = function(arr, rewrite) { CONF.nosql_inmemory && CONF.nosql_inmemory.forEach(n => framework_nosql.inmemory(n)); accepts && accepts.length && accepts.forEach(accept => CONF.static_accepts[accept] = true); - if (CONF.allow_ssc_validation === false) - process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; - if (CONF.allow_performance) http.globalAgent.maxSockets = 9999; From 8a80af3a2f394f45cb10a16b2334940d8e1f0cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 14 Jan 2019 23:52:10 +0100 Subject: [PATCH 0813/1669] Fixed `allow_ssc_validation`. --- index.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 601d8a8be..5b5856495 100755 --- a/index.js +++ b/index.js @@ -6660,9 +6660,6 @@ global.LOAD = F.load = function(debug, types, pwd) { F.cache.init(); EMIT('init'); - if (CONF.allow_ssc_validation === false) - process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; - F.$load(types, directory, function() { F.isLoaded = true; @@ -6817,9 +6814,6 @@ F.initialize = function(http, debug, options) { F.consoledebug('clear temporary'); F.clear(function() { - if (CONF.allow_ssc_validation === false) - process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; - F.consoledebug('clear temporary (done)'); F.$load(undefined, directory, function() { @@ -9171,6 +9165,8 @@ F.$configure_configs = function(arr, rewrite) { CONF.mail_smtp_options = tmp; } + process.env.NODE_TLS_REJECT_UNAUTHORIZED = CONF.allow_ssc_validation === false ? '0' : '1'; + if (!CONF.directory_temp) CONF.directory_temp = '~' + U.path(Path.join(Os.tmpdir(), 'totaljs' + F.directory.hash())); From 68f431f18f36a6dd0059d41447a3dcbba9def03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 Jan 2019 13:55:12 +0100 Subject: [PATCH 0814/1669] Fixed GET params. --- utils.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utils.js b/utils.js index fe71ad654..69bc091c9 100755 --- a/utils.js +++ b/utils.js @@ -492,6 +492,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, var def; var proxy; + if (headers) { headers = exports.extend({}, headers); def = headers[CT]; @@ -604,7 +605,12 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = data.substring(1); if (!options.post) { - data.length && url.indexOf('?') === -1 && (url += '?' + data); + if (data.length) { + if (url.indexOf('?') === -1) + url += '?' + data; + else + url += '&' + data; + } data = ''; } From d8668fd505d1854f101ab8d23542528795a16583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 15 Jan 2019 17:49:47 +0100 Subject: [PATCH 0815/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 51693f9ab..66ba10e2a 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.2.0-4", + "version": "3.2.0-5", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From b17e0b212fb658a2fb19e3c40949b0e941108a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 16 Jan 2019 19:36:54 +0100 Subject: [PATCH 0816/1669] Fixed view engine conditions. --- changes.txt | 1 + internal.js | 6 ++++-- package.json | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 473b1d09b..3408f9c5a 100755 --- a/changes.txt +++ b/changes.txt @@ -18,6 +18,7 @@ - fixed: `U.request()` with `GET` method by @khaledkhalil94 (it doesn't send JSON data if `data` is null/undefined) - fixed: `F.wait()` in WebSocket - fixed: `String.capitalize(true)` +- fixed: view engine conditions defined in `') : value; }; +/** + * Serialize object into the JSON + * @private + * @param {Object} obj + * @param {String} id Optional. + * @param {Boolean} beautify Optional. + * @return {String} + */ +Controller.prototype.$json2 = function(obj, id) { + + if (obj && obj.$$schema) + obj = obj.$clean(); + + var data = {}; + + for (var i = 2; i < arguments.length; i++) { + var key = arguments[i]; + data[key] = obj[key]; + } + + return ''; +}; + /** * Append FAVICON tag * @private @@ -12620,7 +12643,7 @@ Controller.prototype.json = function(obj, headers, beautify, replacer) { F.stats.response.errorBuilder++; } else { - if (framework_builders.isSchema(obj)) + if (obj && obj.$$schema) obj = obj.$clean(); if (beautify) @@ -12681,7 +12704,7 @@ Controller.prototype.jsonp = function(name, obj, headers, beautify, replacer) { F.stats.response.errorBuilder++; } else { - if (framework_builders.isSchema(obj)) + if (obj && obj.$$schema) obj = obj.$clean(); if (beautify) @@ -12832,7 +12855,7 @@ Controller.prototype.plain = function(body, headers) { if (body == null) body = ''; else if (type === 'object') { - if (framework_builders.isSchema(body)) + if (body && body.$$schema) body = body.$clean(); body = body ? JSON.stringify(body, null, 4) : ''; } else @@ -16231,7 +16254,7 @@ function extend_response(PROTO) { PROTO.json = function(obj) { var res = this; F.stats.response.json++; - if (framework_builders.isSchema(obj)) + if (obj && obj.$$schema) obj = obj.$clean(); res.options.body = JSON.stringify(obj); res.options.type = CT_JSON; diff --git a/internal.js b/internal.js index 436f58914..7dcc9ad1e 100755 --- a/internal.js +++ b/internal.js @@ -2251,6 +2251,7 @@ function view_prepare(command, dynamicCommand, functions, controller, components case 'translate': return 'self.' + command; case 'json': + case 'json2': case 'sitemap_change': case 'sitemap_replace': case 'sitemap_add': diff --git a/nosql.js b/nosql.js index 974757a4f..e54ab8ecd 100755 --- a/nosql.js +++ b/nosql.js @@ -332,7 +332,7 @@ exports.worker = function() { var self = this; var builder; - if (framework_builders.isSchema(doc)) + if (doc.$$schema) doc = doc.$clean(); if (unique) { @@ -371,14 +371,14 @@ exports.worker = function() { }; TP.update = DP.update = function(doc, insert) { - var val = framework_builders.isSchema(doc) ? doc.$clean() : doc; + var val = doc.$$schema ? doc.$clean() : doc; if (typeof(val) === 'function') val = val.toString(); return send(this, 'update', val, insert).builder = new DatabaseBuilder(this); }; TP.modify = DP.modify = function(doc, insert) { - var val = framework_builders.isSchema(doc) ? doc.$clean() : doc; + var val = doc.$$schema ? doc.$clean() : doc; if (typeof(val) === 'function') val = val.toString(); return send(this, 'modify', val, insert).builder = new DatabaseBuilder(this); @@ -483,7 +483,7 @@ exports.worker = function() { }; SP.insert = function(doc) { - notify(this.db, 'storage.insert', framework_builders.isSchema(doc) ? doc.$clean() : doc); + notify(this.db, 'storage.insert', doc.$$schema ? doc.$clean() : doc); return this; }; @@ -862,7 +862,7 @@ DP.insert = function(doc, unique) { } builder = new DatabaseBuilder2(self); - var json = framework_builders.isSchema(doc) ? doc.$clean() : doc; + var json = doc.$$schema ? doc.$clean() : doc; self.pending_append.push({ doc: JSON.stringify(json).replace(REGBOOL, JSONBOOL), raw: doc, builder: builder }); setImmediate(next_operation, self, 1); self.$events.insert && self.emit('insert', json); @@ -877,7 +877,7 @@ DP.update = function(doc, insert) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; + var data = doc.$$schema ? doc.$clean() : doc; builder.$options.readertype = 1; if (typeof(data) === 'string') data = new Function('doc', 'repository', 'arg', data.indexOf('return ') === -1 ? ('return (' + data + ')') : data); @@ -890,7 +890,7 @@ DP.modify = function(doc, insert) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; + var data = doc.$$schema ? doc.$clean() : doc; var keys = Object.keys(data); var inc = null; From 5b27374f951325defe8790ed45c5d809a5b18fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 26 Apr 2019 23:54:30 +0200 Subject: [PATCH 1010/1669] Fixed logging. --- index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/index.js b/index.js index e8740f65b..aae89366b 100755 --- a/index.js +++ b/index.js @@ -17703,7 +17703,6 @@ function controller_json_workflow_multiple(id) { } } w.async = op; - console.log(w.async); } var async = self.$async(self.callback(w.view), w.index); From 71408d2ef27f5276321e4192270261e31b73634a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Apr 2019 21:49:44 +0200 Subject: [PATCH 1011/1669] Fixed `session.setcookie()` and custom `options`. --- session.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/session.js b/session.js index d55863179..3ac198d59 100644 --- a/session.js +++ b/session.js @@ -113,6 +113,7 @@ Session.prototype.getcookie = function(req, opt, callback) { // opt.key {String} Encrypt key // opt.extendcookie {Boolean} Extends cookie expiration (default: true) // opt.removecookie {Boolean} Removes cookie if isn't valid (default: true) + // opt.options {Object} A cookie options (default: undefined) if (req.req) req = req.req; @@ -134,7 +135,7 @@ Session.prototype.getcookie = function(req, opt, callback) { if (value && typeof(value) === 'string') { value = value.split(';'); if (req.res && opt.expire && opt.extendcookie !== false) - req.res.cookie(opt.name, token, opt.expire, COOKIEOPTIONS); + req.res.cookie(opt.name, token, opt.expire, opt.options || COOKIEOPTIONS); this.get(value[0], opt.expire, function(err, data, meta, init) { if ((err || !data)) { if (req.res && opt.removecookie !== false) @@ -262,7 +263,7 @@ Session.prototype.setcookie = function(res, opt, callback) { // opt.data {Object} A session data // opt.note {String} A simple note for this session // opt.settings {String} Settings data for the session - // opt.options {String} A cookie options (default: undefined) + // opt.options {Object} A cookie options (default: undefined) if (res.res) res = res.res; From 511ddda9948f293beb9fe8022d11b660dd6fb40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Apr 2019 21:19:29 +0200 Subject: [PATCH 1012/1669] Fixed route operation. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index aae89366b..18ec93cbc 100755 --- a/index.js +++ b/index.js @@ -17607,7 +17607,7 @@ function controller_json_workflow(id) { // IS IT AN OPERATION? if (!self.route.schema.length) { - OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); + OPERATION(w.id, self.body, w.view ? self.callback(w.view) : self.callback(), self); return; } From 393d5c0e9274d6ce94ccf66ba89a963fd6741185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Apr 2019 21:22:09 +0200 Subject: [PATCH 1013/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 477828d67..79169f82d 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.3.0-18", + "version": "3.3.0-19", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 6c2f4d43f9f5361ff62a73387a362b58acb361f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 1 May 2019 09:40:48 +0200 Subject: [PATCH 1014/1669] Fixed `Bad File Descriptor` in `res.image()`. --- index.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 18ec93cbc..9fdc9d820 100755 --- a/index.js +++ b/index.js @@ -17049,8 +17049,13 @@ function $image_stream(exists, size, isFile, stats, res) { delete F.temporary.processing[req.$key]; F.temporary.path[req.$key] = [options.name, stats.size, stats.mtime.toUTCString()]; res.options.filename = options.name; - res.options.stream && DESTROY(options.stream); - res.options.stream = null; + + if (options.stream) { + options.stream.once('error', NOOP); // sometimes is throwed: Bad description + DESTROY(options.stream); + options.stream = null; + } + res.$file(); DEBUG && (F.temporary.path[req.$key] = undefined); return; @@ -17073,7 +17078,11 @@ function $image_stream(exists, size, isFile, stats, res) { F.stats.response.image++; image.save(options.name, function(err) { - options.stream && DESTROY(options.stream); + if (options.stream) { + options.stream.once('error', NOOP); // sometimes is throwed: Bad description + DESTROY(options.stream); + options.stream = null; + } delete F.temporary.processing[req.$key]; if (err) { From 92ab1a0a8c5a87a7e6df9df4750fe781a1685168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0irka?= Date: Thu, 2 May 2019 15:09:29 +0200 Subject: [PATCH 1015/1669] Added `PATCH` support. --- builders.js | 78 ++++++++++++++++++++++++++++++++++++++++++++-------- changes.txt | 1 + index.js | 23 ++++++++++++++++ package.json | 2 +- utils.js | 2 +- 5 files changed, 93 insertions(+), 13 deletions(-) diff --git a/builders.js b/builders.js index 6954571c5..a5eb95b3c 100755 --- a/builders.js +++ b/builders.js @@ -28,14 +28,14 @@ const REQUIRED = 'The field "@" is invalid.'; const DEFAULT_SCHEMA = 'default'; -const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true, $$workflow: true, $$parent: true }; +const SKIP = { $$schema: 1, $$async: 1, $$repository: 1, $$controller: 1, $$workflow: 1, $$parent: 1, $$keys: 1 }; const REGEXP_CLEAN_EMAIL = /\s/g; const REGEXP_CLEAN_PHONE = /\s|\.|-|\(|\)/g; const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?\([a-zA-Z$\s]+\)|^function anonymous\(\$|^\([a-zA-Z$\s]+\)/; const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); const MSG_OBSOLETE_NEW = 'You used older declaration of this delegate and you must rewrite it. Read more in docs.'; -const BOOL = { 'true': true, 'on': true, '1': true }; +const BOOL = { true: 1, on: 1, '1': 1 }; var schemas = {}; var schemasall = {}; @@ -136,6 +136,10 @@ SchemaOptions.prototype = { return this.controller ? this.controller.session : null; }, + get keys() { + return this.model.$$keys; + }, + get sessionid() { return this.controller && this.controller ? this.controller.req.sessionid : null; }, @@ -224,6 +228,10 @@ SchemaOptions.prototype.$update = function(helper, callback, async) { return this.model.$update(helper, callback, async); }; +SchemaOptions.prototype.$patch = function(helper, callback, async) { + return this.model.$patch(helper, callback, async); +}; + SchemaOptions.prototype.$query = function(helper, callback, async) { return this.model.$query(helper, callback, async); }; @@ -1042,6 +1050,19 @@ SchemaBuilderEntity.prototype.setUpdate = function(fn, description) { return this; }; +/** + * Set patch handler + * @param {Function(error, model, helper, next(value), controller)} fn + * @return {SchemaBuilderEntity} + */ +SchemaBuilderEntity.prototype.setPatch = function(fn, description) { + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); + this.onPatch = fn; + this.meta.patch = description || null; + !fn.$newversion && OBSOLETE('Schema("{0}").setPatch()'.format(this.name), MSG_OBSOLETE_NEW); + return this; +}; + /** * Set error handler * @param {Function(error)} fn @@ -1274,6 +1295,10 @@ SchemaBuilderEntity.prototype.update = function(model, options, callback, contro return this.execute('onUpdate', model, options, callback, controller, skip); }; +SchemaBuilderEntity.prototype.patch = function(model, options, callback, controller, skip) { + return this.execute('onPatch', model, options, callback, controller, skip); +}; + SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, controller, skip) { if (typeof(callback) === 'boolean') { @@ -1304,6 +1329,9 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, case 'onUpdate': $type = 'update'; break; + case 'onPatch': + $type = 'patch'; + break; default: $type = 'save'; break; @@ -1914,11 +1942,19 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies, req) { var entity; var item = new self.CurrentSchemaInstance(); var defaults = self.onDefault || self.$onDefault ? true : false; + var keys = req && req.$patch ? [] : null; for (var property in obj) { var val = model[property]; + + if (req && req.$patch && val === undefined) { + delete item[property]; + continue; + } + var type = obj[property]; + keys && keys.push(property); // IS PROTOTYPE? The problem was in e.g. "search" property, because search is in String prototypes. if (!hasOwnProperty.call(model, property)) @@ -2223,8 +2259,7 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies, req) { if (entity) { tmp = entity.prepare(tmp, dependencies); tmp.$$parent = item; - if (dependencies) - dependencies.push({ name: type.raw, value: self.$onprepare(property, tmp, j, model, req) }); + dependencies && dependencies.push({ name: type.raw, value: self.$onprepare(property, tmp, j, model, req) }); } else tmp = null; @@ -2247,11 +2282,16 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies, req) { for (var i = 0, length = self.fields_allow.length; i < length; i++) { var name = self.fields_allow[i]; var val = model[name]; - if (val !== undefined) + if (val !== undefined) { item[name] = val; + keys && keys.push(name); + } } } + if (keys) + item.$$keys = keys; + return item; }; @@ -2979,12 +3019,15 @@ SchemaInstance.prototype.$stop = function() { return this; }; +const PUSHTYPE1 = { save: 1, insert: 1, update: 1, patch: 1 }; +const PUSHTYPE2 = { query: 1, get: 1, read: 1, remove: 1 }; + SchemaInstance.prototype.$push = function(type, name, helper, first, async, callback) { var self = this; var fn; - if (type === 'save' || type === 'insert' || type === 'update') { + if (PUSHTYPE1[type]) { fn = function(next, indexer) { self.$$schema[type](self, helper, function(err, result) { var a = self.$$async; @@ -2999,7 +3042,7 @@ SchemaInstance.prototype.$push = function(type, name, helper, first, async, call }, self.$$controller); }; - } else if (type === 'query' || type === 'get' || type === 'read' || type === 'remove') { + } else if (PUSHTYPE2[type]) { fn = function(next, indexer) { self.$$schema[type](helper, function(err, result) { var a = self.$$async; @@ -3111,9 +3154,7 @@ SchemaInstance.prototype.$insert = function(helper, callback, async) { }; SchemaInstance.prototype.$update = function(helper, callback, async) { - if (this.$$async && !this.$$async.running) { - if (typeof(helper) === 'function') { async = callback; callback = helper; @@ -3123,14 +3164,29 @@ SchemaInstance.prototype.$update = function(helper, callback, async) { async = true; callback = a; } - this.$push('update', null, helper, null, async, callback); - } else this.$$schema.update(this, helper, callback, this.$$controller); return this; }; +SchemaInstance.prototype.$patch = function(helper, callback, async) { + if (this.$$async && !this.$$async.running) { + if (typeof(helper) === 'function') { + async = callback; + callback = helper; + helper = null; + } else if (callback === true) { + var a = async; + async = true; + callback = a; + } + this.$push('patch', null, helper, null, async, callback); + } else + this.$$schema.patch(this, helper, callback, this.$$controller); + return this; +}; + SchemaInstance.prototype.$query = function(helper, callback, async) { if (this.$$async && !this.$$async.running) { diff --git a/changes.txt b/changes.txt index 456432b36..71414c807 100755 --- a/changes.txt +++ b/changes.txt @@ -36,6 +36,7 @@ - added: usage of currency formatter `Number.currency(currency)` - added: new schema type `Number2` with default value is `null`, not zero `0` - added: `@{json2(model, elementID, key1, key2, key3)}` can serialize data with keys defined into the ` - @{js('default.js', '#test')} + @{js('default.js')} @{favicon('favicon.ico')} From 989c0e9d5000a2db874ab8c1d0f1306ab80907a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Jun 2019 10:43:53 +0200 Subject: [PATCH 1082/1669] Improved timeout in `U.request()`, `U.download()` and `U.upload()`. --- utils.js | 153 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 45 deletions(-) diff --git a/utils.js b/utils.js index 61a8178d6..3dd066590 100755 --- a/utils.js +++ b/utils.js @@ -855,24 +855,39 @@ function request_call(uri, options) { return; } - req.on('error', function(err) { - if (options.callback && !options.done) { - options.callback(err, '', 0, undefined, uri.host); + var timeout = function() { + if (options.callback) { + if (options.timeoutid) { + clearTimeout(options.timeoutid); + options.timeoutid = null; + } + req.abort(); + options.canceled = true; + options.callback(new Error(exports.httpStatus(408)), '', 0, undefined, uri.host); options.callback = null; options.evt.removeAllListeners(); options.evt = null; } - }); + }; - req.setTimeout(options.timeout, function() { - if (options.callback) { - options.callback(new Error(exports.httpStatus(408)), '', 0, undefined, uri.host); + req.on('error', function(err) { + if (options.callback && !options.done) { + if (options.timeoutid) { + clearTimeout(options.timeoutid); + options.timeoutid = null; + } + options.canceled = true; + options.callback(err, '', 0, undefined, uri.host); options.callback = null; options.evt.removeAllListeners(); options.evt = null; } }); + options.timeoutid && clearTimeout(options.timeoutid); + options.timeoutid = setTimeout(timeout, options.timeout); + req.setTimeout(options.timeout, timeout); + req.on('response', (response) => response.req = req); if (options.upload) { @@ -928,6 +943,9 @@ function request_response(res, uri, options) { if (options.noredirect) { + options.timeoutid && clearTimeout(options.timeoutid); + options.canceled = true; + if (options.callback) { options.callback(null, '', res.statusCode, res.headers, uri.host, EMPTYOBJECT); options.callback = null; @@ -939,14 +957,17 @@ function request_response(res, uri, options) { } res.req.removeAllListeners(); - res.req = null; res.removeAllListeners(); + res.req = null; res = null; return; } if (options.redirect > 3) { + options.timeoutid && clearTimeout(options.timeoutid); + options.canceled = true; + if (options.callback) { options.callback(new Error('Too many redirects.'), '', 0, undefined, uri.host, EMPTYOBJECT); options.callback = null; @@ -958,8 +979,8 @@ function request_response(res, uri, options) { } res.req.removeAllListeners(); - res.req = null; res.removeAllListeners(); + res.req = null; res = null; return; } @@ -1034,17 +1055,24 @@ function request_response(res, uri, options) { var onend = function() { - if (options.socket) - options.uri.agent.destroy(); - var self = this; var data; - if (!self.headers['content-type'] || REG_TEXTAPPLICATION.test(self.headers['content-type'])) + options.socket && options.uri.agent.destroy(); + options.timeoutid && clearTimeout(options.timeoutid); + + if (options.canceled) + return; + + var ct = self.headers['content-type']; + + if (!ct || REG_TEXTAPPLICATION.test(ct)) data = self._buffer ? (options.encoding === 'binary' ? self._buffer : self._buffer.toString(options.encoding)) : ''; else data = self._buffer; + options.canceled = true; + self._buffer = undefined; if (options.evt) { @@ -1072,7 +1100,7 @@ function request_response(res, uri, options) { var ondata = function(chunk) { var self = this; - if (options.max && self._bufferlength > options.max) + if (options.canceled || (options.max && self._bufferlength > options.max)) return; if (self._buffer) { CONCAT[0] = self._buffer; @@ -1349,24 +1377,32 @@ function download_call(uri, options) { return; } - req.on('error', function(err) { + var timeout = function() { if (options.callback) { - options.callback(err); + options.callback(new Error(exports.httpStatus(408))); options.callback = null; options.evt.removeAllListeners(); options.evt = null; + options.canceled = true; } - }); + }; - req.setTimeout(options.timeout, function() { + req.on('error', function(err) { if (options.callback) { - options.callback(new Error(exports.httpStatus(408))); + options.timeoutid && clearTimeout(options.timeoutid); + options.timeoutid = null; + options.callback(err); options.callback = null; options.evt.removeAllListeners(); options.evt = null; + options.canceled = true; } }); + options.timeoutid && clearTimeout(options.timeoutid); + options.timeoutid = setTimeout(timeout, options.timeout); + req.setTimeout(options.timeout, timeout); + req.on('response', function(response) { response.req = req; options.length = +response.headers['content-length'] || 0; @@ -1384,6 +1420,8 @@ function download_response(res, uri, options) { if (res.statusCode === 301 || res.statusCode === 302) { if (options.redirect > 3) { + options.canceled = true; + options.timeoutid && clearTimeout(options.timeoutid); options.callback && options.callback(new Error('Too many redirects.')); res.req.removeAllListeners(); res.req = null; @@ -1432,19 +1470,24 @@ function download_response(res, uri, options) { } res.on('data', function(chunk) { - var self = this; - self._bufferlength += chunk.length; - options.evt && options.evt.$events.data && options.evt.emit('data', chunk, options.length ? (self._bufferlength / options.length) * 100 : 0); + if (!options.canceled) { + var self = this; + self._bufferlength += chunk.length; + options.evt && options.evt.$events.data && options.evt.emit('data', chunk, options.length ? (self._bufferlength / options.length) * 100 : 0); + } }); res.on('end', function() { + var self = this; - var str = self._buffer ? self._buffer.toString(options.encoding) : ''; - self._buffer = undefined; + if (!options.canceled) { + var str = self._buffer ? self._buffer.toString(options.encoding) : ''; + self._buffer = undefined; + options.evt && options.evt.$events.end && options.evt.emit('end', str, self.statusCode, self.headers, uri.host); + } if (options.evt) { - options.evt.$events.end && options.evt.emit('end', str, self.statusCode, self.headers, uri.host); options.evt.removeAllListeners(); options.evt = null; } @@ -1454,6 +1497,7 @@ function download_response(res, uri, options) { }); res.resume(); + options.timeoutid && clearTimeout(options.timeoutid); options.callback && options.callback(null, res, res.statusCode, res.headers, uri.host); } @@ -1598,6 +1642,8 @@ exports.upload = function(files, url, callback, cookies, headers, method, timeou var uri = Url.parse(url); var options = { protocol: uri.protocol, auth: uri.auth, method: method || 'POST', hostname: uri.hostname, port: uri.port, path: uri.path, agent: false, headers: h }; var responseLength = 0; + var timeoutid; + var done = false; var response = function(res) { @@ -1605,20 +1651,26 @@ exports.upload = function(files, url, callback, cookies, headers, method, timeou res._bufferlength = 0; res.on('data', function(chunk) { - CONCAT[0] = res.body; - CONCAT[1] = chunk; - res.body = Buffer.concat(CONCAT); - res._bufferlength += chunk.length; - e.$events.data && e.emit('data', chunk, responseLength ? (res._bufferlength / responseLength) * 100 : 0); + if (!done) { + CONCAT[0] = res.body; + CONCAT[1] = chunk; + res.body = Buffer.concat(CONCAT); + res._bufferlength += chunk.length; + e.$events.data && e.emit('data', chunk, responseLength ? (res._bufferlength / responseLength) * 100 : 0); + } }); res.on('end', function() { - var self = this; - e.$events.end && e.emit('end', self.statusCode, self.headers); - e.removeAllListeners(); - e = null; - callback && callback(null, self.body.toString('utf8'), self.statusCode, self.headers, uri.host); - self.body = null; + if (!done) { + var self = this; + e.$events.end && e.emit('end', self.statusCode, self.headers); + e.removeAllListeners(); + callback && callback(null, self.body.toString('utf8'), self.statusCode, self.headers, uri.host); + timeoutid && clearTimeout(timeoutid); + self.body = null; + e = null; + done = true; + } }); }; @@ -1630,20 +1682,31 @@ exports.upload = function(files, url, callback, cookies, headers, method, timeou e.$events.begin && e.emit('begin', responseLength); }); - req.setTimeout(timeout || 60000, function() { - req.removeAllListeners(); - req = null; - e.removeAllListeners(); - e = null; - callback && callback(new Error(exports.httpStatus(408)), '', 408, undefined, uri.host); - }); + var timeoutcallback = function() { + if (!done) { + req.removeAllListeners(); + e.removeAllListeners(); + callback && callback(new Error(exports.httpStatus(408)), '', 408, undefined, uri.host); + timeoutid && clearTimeout(timeoutid); + req = null; + e = null; + done = true; + } + }; + + if (timeout) + timeoutid = setTimeout(timeoutcallback, timeout); + + req.setTimeout(timeout || 60000, timeoutcallback); req.on('error', function(err) { + done = true; req.removeAllListeners(); - req = null; e.removeAllListeners(); - e = null; callback && callback(err, '', 0, undefined, uri.host); + timeoutid && clearTimeout(timeoutid); + req = null; + e = null; }); req.on('close', function() { From bc075de241af69ee27484c9b3c46973e5729c52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Jun 2019 16:49:54 +0200 Subject: [PATCH 1083/1669] Improved code. --- mail.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail.js b/mail.js index 76fe6458a..e4267cc19 100755 --- a/mail.js +++ b/mail.js @@ -38,7 +38,7 @@ const REG_NEWLINE = /\n/g; const REG_AUTH = /(AUTH LOGIN|AUTH PLAIN|PLAIN LOGIN)/i; const REG_TLS = /TLS/; const REG_STARTTLS = /STARTTLS/; -const REG_PREVIEW = //i; +// const REG_PREVIEW = //i; const EMPTYARRAY = []; var INDEXSENDER = 0; From c9e5ef02e50958468fd6bf685bfd793501c4e709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Jun 2019 16:50:16 +0200 Subject: [PATCH 1084/1669] Added `REQIURE()`. --- changes.txt | 1 + index.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 551bcea1b..ca608772b 100755 --- a/changes.txt +++ b/changes.txt @@ -49,6 +49,7 @@ x added: `AUDIT(name, $, [type], message)` a method for audit logs x added: __obsolete__ message to older declaration of middleware, schemas and operations x added: `U.diffarr(prop, arr_A, arr_B)` for comparing of two arrays x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` +- added: `global.REQUIRE()` for local modules within app directory x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index 53e886acb..d37c565f0 100755 --- a/index.js +++ b/index.js @@ -107,6 +107,11 @@ Object.freeze(EMPTYREQUEST); global.EMPTYOBJECT = EMPTYOBJECT; global.EMPTYARRAY = EMPTYARRAY; global.NOW = new Date(); + +global.REQUIRE = function(path) { + return require('./' + path); +}; + var DEF = global.DEF = {}; DEF.currencies = {}; From 389c39035b9698c929242577e0f16962e7f5f698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Jun 2019 09:39:13 +0200 Subject: [PATCH 1085/1669] Renamed `*.routeSomething` to `public_something`. --- changes.txt | 8 ++ index.js | 181 +++++++++++++++++++++--------------- internal.js | 12 ++- test/controllers/default.js | 14 +-- test/controllers/share.js | 2 - test/modules/inline-view.js | 1 - test/views/a.html | 2 +- 7 files changed, 134 insertions(+), 86 deletions(-) diff --git a/changes.txt b/changes.txt index ca608772b..caa79d4cd 100755 --- a/changes.txt +++ b/changes.txt @@ -80,6 +80,14 @@ x updated: `String.hash(true)` returns unsigned int - fixed: renamed `F.stats.request.path` to `F.stats.request.patch` - fixed: SMTP sender (a problem with auth mechanism with some mail servers) +- renamed: `*.routeScript` to `*.public_js` +- renamed: `*.routeStyle` to `*.public_css` +- renamed: `*.routeFont` to `*.public_font` +- renamed: `*.routeVideo` to `*.public_video` +- renamed: `*.routeImage` to `*.public_image` +- renamed: `*.routeDownload` to `*.public_download` +- renamed: `*.routeStatic` to `*.public` + x __removed: backward compatibility__ with older version of Node.js < 10 x removed: `F.hash()`, alternative `String.prototype.hash()` x removed: `controller.hash()`, alternative `String.prototype.hash()` diff --git a/index.js b/index.js index d37c565f0..131a36f57 100755 --- a/index.js +++ b/index.js @@ -4890,10 +4890,11 @@ global.INSTALL = F.install = function(type, name, declaration, options, callback F.install_prepare = function(noRecursive) { var keys = Object.keys(F.temporary.dependencies); - if (!keys.length) return; + OBSOLETE('exports.dependencies()', 'Module dependencies will be removed in v4: "' + keys.join(', ') + '"'); + // check dependencies for (var i = 0, length = keys.length; i < length; i++) { @@ -9532,7 +9533,8 @@ function obsolete_config(name) { * @return {String} */ F.routeScript = function(name, theme) { - return F.$routeStatic(name, CONF.static_url_script, theme); + OBSOLETE('F.routeScript()', 'Renamed to F.public_js'); + return F.$public(name, CONF.static_url_script, theme); }; /** @@ -9541,30 +9543,64 @@ F.routeScript = function(name, theme) { * @return {String} */ F.routeStyle = function(name, theme) { - return F.$routeStatic(name, CONF.static_url_style, theme); + OBSOLETE('F.routeStyle()', 'Renamed to F.public_css'); + return F.$public(name, CONF.static_url_style, theme); }; F.routeImage = function(name, theme) { - return F.$routeStatic(name, CONF.static_url_image, theme); + OBSOLETE('F.routeImage()', 'Renamed to F.public_image'); + return F.$public(name, CONF.static_url_image, theme); }; F.routeVideo = function(name, theme) { - return F.$routeStatic(name, CONF.static_url_video, theme); + OBSOLETE('F.routeVideo()', 'Renamed to F.public_video'); + return F.$public(name, CONF.static_url_video, theme); }; F.routeFont = function(name, theme) { - return F.$routeStatic(name, CONF.static_url_font, theme); + OBSOLETE('F.routeFont()', 'Renamed to F.public_font'); + return F.$public(name, CONF.static_url_font, theme); }; F.routeDownload = function(name, theme) { - return F.$routeStatic(name, CONF.static_url_download, theme); + OBSOLETE('F.routeDownload()', 'Renamed to F.public_download'); + return F.$public(name, CONF.static_url_download, theme); }; F.routeStatic = function(name, theme) { - return F.$routeStatic(name, CONF.static_url, theme); + OBSOLETE('F.routeStatic()', 'Renamed to F.public'); + return F.$public(name, CONF.static_url, theme); +}; + +F.public_js = function(name, theme) { + return F.$public(name, CONF.static_url_script, theme); +}; + +F.public_css = function(name, theme) { + return F.$public(name, CONF.static_url_style, theme); +}; + +F.public_image = function(name, theme) { + return F.$public(name, CONF.static_url_image, theme); +}; + +F.public_video = function(name, theme) { + return F.$public(name, CONF.static_url_video, theme); +}; + +F.public_font = function(name, theme) { + return F.$public(name, CONF.static_url_font, theme); +}; + +F.public_download = function(name, theme) { + return F.$public(name, CONF.static_url_download, theme); +}; + +F.public = function(name, theme) { + return F.$public(name, CONF.static_url, theme); }; -F.$routeStatic = function(name, directory, theme) { +F.$public = function(name, directory, theme) { var key = name + directory + '$' + theme; var val = F.temporary.other[key]; if (RELEASE && val) @@ -12148,9 +12184,9 @@ Controller.prototype.head = function() { var is = (tmp[0] !== '/' && tmp[1] !== '/') && tmp !== 'http://' && tmp !== 'https:/'; var ext = U.getExtension(val); if (ext === 'css') - header += ''; + header += ''; else if (ext === 'js') - header += ''; + header += ''; } self.repository[REPOSITORY_HEAD] = header; @@ -12280,7 +12316,7 @@ Controller.prototype.$js = function() { var self = this; var builder = ''; for (var i = 0; i < arguments.length; i++) - builder += self.routeScript(arguments[i], true); + builder += self.public_js(arguments[i], true); return builder; }; @@ -12306,13 +12342,13 @@ Controller.prototype.$absolute = function(files, base) { for (var i = 0, length = files.length; i < length; i++) { switch (ftype) { case 'js': - builder += self.routeScript(files[i], true, base); + builder += self.public_js(files[i], true, base); break; case 'css': - builder += self.routeStyle(files[i], true, base); + builder += self.public_css(files[i], true, base); break; default: - builder += self.routeStatic(files[i], base); + builder += self.public(files[i], base); break; } } @@ -12324,12 +12360,12 @@ Controller.prototype.$absolute = function(files, base) { switch (ftype) { case 'js': - return self.routeScript(files, true, base); + return self.public_js(files, true, base); case 'css': - return self.routeStyle(files, true, base); + return self.public_css(files, true, base); } - return self.routeStatic(files, base); + return self.public(files, base); }; Controller.prototype.$import = function() { @@ -12383,10 +12419,10 @@ Controller.prototype.$import = function() { switch (extension) { case '.js': - builder += self.routeScript(filename, tag); + builder += self.public_js(filename, tag); break; case '.css': - builder += self.routeStyle(filename, tag); + builder += self.public_css(filename, tag); break; case '.ico': builder += self.$favicon(filename); @@ -12400,7 +12436,7 @@ Controller.prototype.$import = function() { case '.webp': case '.heic': case '.apng': - builder += self.routeImage(filename); + builder += self.public_image(filename); break; case '.mp4': case '.avi': @@ -12411,10 +12447,10 @@ Controller.prototype.$import = function() { case '.mpe': case '.mpeg': case '.m4v': - builder += self.routeVideo(filename); + builder += self.public_video(filename); break; default: - builder += self.routeStatic(filename); + builder += self.public(filename); break; } } @@ -12433,7 +12469,7 @@ Controller.prototype.$css = function() { var builder = ''; for (var i = 0; i < arguments.length; i++) - builder += self.routeStyle(arguments[i], true); + builder += self.public_css(arguments[i], true); return builder; }; @@ -12450,7 +12486,7 @@ Controller.prototype.$image = function(name, width, height, alt, className) { width = width.width; } - var builder = ' 0) builder += ' width="' + width + ATTR_END; @@ -12480,7 +12516,7 @@ Controller.prototype.$image = function(name, width, height, alt, className) { * @return {String} */ Controller.prototype.$download = function(filename, innerHTML, downloadName, className) { - var builder = ''; + return F.temporary.other[key] = ''; }; /** @@ -12575,18 +12611,16 @@ Controller.prototype.$favicon = function(name) { * @param {Function} fn * @return {String} */ -Controller.prototype._routeHelper = function(name, fn) { +Controller.prototype.$static = function(name, fn) { return fn.call(framework, prepare_staticurl(name, false), this.themeName); }; -/** - * Create URL: JavaScript - * @param {String} name - * @param {Boolean} tag Optional, default "false" - * @param {String} path Optional, default undefined - * @return {String} - */ Controller.prototype.routeScript = function(name, tag, path) { + OBSOLETE('controller.routeScript()', 'Was renamed to "controller.public_js()"'); + return this.public_js(name, tag, path); +}; + +Controller.prototype.public_js = function(name, tag, path) { if (name === undefined) name = 'default.js'; @@ -12610,7 +12644,7 @@ Controller.prototype.routeScript = function(name, tag, path) { return ''; } } else { - url = this._routeHelper(name, F.routeScript); + url = this.$static(name, F.public_js); if (path && U.isRelative(url)) url = F.isWindows ? U.join(path, url) : U.join(path, url).substring(1); } @@ -12618,68 +12652,68 @@ Controller.prototype.routeScript = function(name, tag, path) { return tag ? ('') : url; }; -/** - * Create URL: CSS - * @param {String} name - * @param {Boolean} tag Append tag? - * @return {String} - */ Controller.prototype.routeStyle = function(name, tag, path) { + OBSOLETE('controller.routeStyle()', 'Was renamed to "controller.public_css()"'); + return this.public_css(name, tag, path); +}; + +Controller.prototype.public_css = function(name, tag, path) { + var self = this; if (name === undefined) name = 'default.css'; - var url = self._routeHelper(name, F.routeStyle); + var url = self.$static(name, F.public_css); if (path && U.isRelative(url)) url = F.isWindows ? U.join(path, url) : U.join(path, url).substring(1); return tag ? '' : url; }; -/** - * Create URL: IMG - * @param {String} name - * @return {String} - */ Controller.prototype.routeImage = function(name) { - return this._routeHelper(name, F.routeImage); + OBSOLETE('controller.routeImage()', 'Was renamed to "controller.public_image()"'); + return this.public_image(name); +}; + +Controller.prototype.public_image = function(name) { + return this.$static(name, F.public_image); }; -/** - * Create URL: VIDEO - * @param {String} name - * @return {String} - */ Controller.prototype.routeVideo = function(name) { - return this._routeHelper(name, F.routeVideo); + OBSOLETE('controller.routeVideo()', 'Was renamed to "controller.public_video()"'); + return this.public_video(name); +}; + +Controller.prototype.public_video = function(name) { + return this.$static(name, F.public_video); }; -/** - * Create URL: FONT - * @param {String} name - * @return {String} - */ Controller.prototype.routeFont = function(name) { - return F.routeFont(name); + OBSOLETE('controller.routeFont()', 'Was renamed to "controller.public_font()"'); + return this.public_font(name); +}; + +Controller.prototype.public_font = function(name) { + return F.public_font(name); }; -/** - * Create URL: DOWNLOAD - * @param {String} name - * @return {String} - */ Controller.prototype.routeDownload = function(name) { - return this._routeHelper(name, F.routeDownload); + OBSOLETE('controller.routeDownload()', 'Was renamed to "controller.public_download()"'); + return this.public_download(name); +}; + +Controller.prototype.public_download = function(name) { + return this.$static(name, F.public_download); }; -/** - * Create URL: static files (by the config['static-url']) - * @param {String} name - * @return {String} - */ Controller.prototype.routeStatic = function(name, path) { - var url = this._routeHelper(name, F.routeStatic); + OBSOLETE('controller.routeStatic()', 'Was renamed to "controller.public()"'); + return this.public(name, path); +}; + +Controller.prototype.public = function(name, path) { + var url = this.$static(name, F.public); if (path && U.isRelative(url)) return F.isWindows ? U.join(path, url) : U.join(path, url).substring(1); return url; @@ -12692,6 +12726,7 @@ Controller.prototype.routeStatic = function(name, path) { * @return {String} */ Controller.prototype.template = function(name, model) { + OBSOLETE('controller.template()', 'This method will be removed in v4.'); return this.view(name, model, true); }; diff --git a/internal.js b/internal.js index 7dcc9ad1e..fbd3d59b3 100755 --- a/internal.js +++ b/internal.js @@ -2239,14 +2239,22 @@ function view_prepare(command, dynamicCommand, functions, controller, components return 'self.' + command; case 'routeJS': - case 'routeCSS': case 'routeScript': + case 'routeCSS': case 'routeStyle': case 'routeImage': case 'routeFont': case 'routeDownload': - case 'routeVideo': case 'routeStatic': + case 'routeVideo': + return 'self.' + command; + case 'public_js': + case 'public_css': + case 'public_image': + case 'public_font': + case 'public_download': + case 'public_video': + case 'public': return 'self.' + command; case 'translate': return 'self.' + command; diff --git a/test/controllers/default.js b/test/controllers/default.js index 0de1fdc5b..d2a59f9cd 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -470,13 +470,13 @@ function viewIndex() { var date = new Date(); date.setFullYear(1984); - assert.ok(self.routeScript('p.js') === '/js/p.js', name + 'routeScript()'); - assert.ok(self.routeStyle('p.css') === '/css/p.css', name + 'routeStyle()'); - assert.ok(self.routeImage('p.jpg') === '/img/p.jpg', name + 'routeImage()'); - assert.ok(self.routeVideo('p.avi') === '/video/p.avi', name + 'routeVideo()'); - assert.ok(self.routeFont('p.woff') === '/fonts/p.woff', name + 'routeFont()'); - assert.ok(self.routeDownload('p.pdf') === '/download/p.pdf', name + 'routeDownload()'); - assert.ok(self.routeStatic('/p.zip') === '/p.zip', name + 'routeStatic()'); + assert.ok(self.public_js('p.js') === '/js/p.js', name + 'public_js()'); + assert.ok(self.public_css('p.css') === '/css/p.css', name + 'public_css()'); + assert.ok(self.public_image('p.jpg') === '/img/p.jpg', name + 'public_image()'); + assert.ok(self.public_video('p.avi') === '/video/p.avi', name + 'public_video()'); + assert.ok(self.public_font('p.woff') === '/fonts/p.woff', name + 'public_font()'); + assert.ok(self.public_download('p.pdf') === '/download/p.pdf', name + 'public_download()'); + assert.ok(self.public('/p.zip') === '/p.zip', name + 'public()'); self.layout(''); assert.ok(self.view('test', null, true) === 'Total.js', name + 'view'); diff --git a/test/controllers/share.js b/test/controllers/share.js index 9591e7451..071aa367b 100755 --- a/test/controllers/share.js +++ b/test/controllers/share.js @@ -1,5 +1,3 @@ -exports.dependencies = ['test']; - exports.install = function() { framework.route('/share/', view_share); framework.route('/router/', view_router); diff --git a/test/modules/inline-view.js b/test/modules/inline-view.js index 23c441057..39b6fac8a 100644 --- a/test/modules/inline-view.js +++ b/test/modules/inline-view.js @@ -1,6 +1,5 @@ var assert = require('assert'); -exports.dependencies = ['test']; exports.installed = false; exports.install = function() { diff --git a/test/views/a.html b/test/views/a.html index 984340e26..2eb785f5f 100755 --- a/test/views/a.html +++ b/test/views/a.html @@ -32,7 +32,7 @@ #options@{options(R.options, 'C', 'k', 'v')}# #view-toggle@{viewToggle(false, 'b')}# #view@{view('current/b', 'model')}# -#routejs-@{routeScript('p.js')}# +#routejs-@{public_js('p.js')}# #title@{title}# #mobile@{mobile}# #@{head}# From 30f43124aca4d776be20b1b1d86651c60d8a2cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Jun 2019 09:48:16 +0200 Subject: [PATCH 1086/1669] Updated changelog. --- changes.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index caa79d4cd..dd68e3ff8 100755 --- a/changes.txt +++ b/changes.txt @@ -80,13 +80,13 @@ x updated: `String.hash(true)` returns unsigned int - fixed: renamed `F.stats.request.path` to `F.stats.request.patch` - fixed: SMTP sender (a problem with auth mechanism with some mail servers) -- renamed: `*.routeScript` to `*.public_js` -- renamed: `*.routeStyle` to `*.public_css` -- renamed: `*.routeFont` to `*.public_font` -- renamed: `*.routeVideo` to `*.public_video` -- renamed: `*.routeImage` to `*.public_image` -- renamed: `*.routeDownload` to `*.public_download` -- renamed: `*.routeStatic` to `*.public` +x renamed: `*.routeScript` to `*.public_js` +x renamed: `*.routeStyle` to `*.public_css` +x renamed: `*.routeFont` to `*.public_font` +x renamed: `*.routeVideo` to `*.public_video` +x renamed: `*.routeImage` to `*.public_image` +x renamed: `*.routeDownload` to `*.public_download` +x renamed: `*.routeStatic` to `*.public` x __removed: backward compatibility__ with older version of Node.js < 10 x removed: `F.hash()`, alternative `String.prototype.hash()` From 8bf1bc9cab55a322185b56436ff382d1e85de428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Jun 2019 11:22:24 +0200 Subject: [PATCH 1087/1669] Renamed `controller.viewCompile()` to `controller.view_compile()`. --- changes.txt | 1 + index.js | 19 ++++++++++++++++--- internal.js | 3 +-- test/controllers/default.js | 3 +-- test/views/a.html | 3 +-- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index dd68e3ff8..1c2475c2e 100755 --- a/changes.txt +++ b/changes.txt @@ -87,6 +87,7 @@ x renamed: `*.routeVideo` to `*.public_video` x renamed: `*.routeImage` to `*.public_image` x renamed: `*.routeDownload` to `*.public_download` x renamed: `*.routeStatic` to `*.public` +- renamed: `controller.viewCompile()` to `controller.view_compile()` x __removed: backward compatibility__ with older version of Node.js < 10 x removed: `F.hash()`, alternative `String.prototype.hash()` diff --git a/index.js b/index.js index 131a36f57..7f1765a0a 100755 --- a/index.js +++ b/index.js @@ -8177,7 +8177,7 @@ F.view = function(name, model, layout, repository, language) { * @param {String} language Optional. * @return {String} */ -global.VIEWCOMPILE = F.viewCompile = function(body, model, layout, repository, language) { +global.VIEWCOMPILE = function(body, model, layout, repository, language) { var controller = EMPTYCONTROLLER; @@ -8192,7 +8192,7 @@ global.VIEWCOMPILE = F.viewCompile = function(body, model, layout, repository, l controller.themeName = undefined; controller.repository = typeof(repository) === 'object' && repository ? repository : EMPTYOBJECT; - return controller.viewCompile(body, model, true); + return controller.view_compile(body, model, true); }; F.viewCompile = function(body, model, layout, repository, language) { @@ -11753,10 +11753,12 @@ Controller.prototype.mail = function(address, subject, view, model, callback) { }; Controller.prototype.$template = function(name, model, expire, key) { + OBSOLETE('@{template()}', 'The method will be removed in v4'); return this.$viewToggle(true, name, model, expire, key); }; Controller.prototype.$templateToggle = function(visible, name, model, expire, key) { + OBSOLETE('@{templateToggle()}', 'The method will be removed in v4'); return this.$viewToggle(visible, name, model, expire, key); }; @@ -11781,15 +11783,21 @@ Controller.prototype.$view = function(name, model, expire, key) { }; Controller.prototype.$viewCompile = function(body, model, key) { + OBSOLETE('@{viewCompile()}', 'Was renamed to @{view_compile()}.'); + return this.$view_compile(body, model, key); +}; + +Controller.prototype.$view_compile = function(body, model, key) { var self = this; var layout = self.layoutName; self.layoutName = ''; - var value = self.viewCompile(body, model, null, true, key); + var value = self.view_compile(body, model, null, true, key); self.layoutName = layout; return value || ''; }; Controller.prototype.$viewToggle = function(visible, name, model, expire, key, async) { + OBSOLETE('@{viewToggle()}', 'The method will be removed in v4'); return visible ? this.$view(name, model, expire, key, async) : ''; }; @@ -13612,6 +13620,11 @@ Controller.prototype.view = function(name, model, headers, partial, noasync, cac }; Controller.prototype.viewCompile = function(body, model, headers, partial, key) { + OBSOLETE('controller.viewCompile()', 'Was renamed to `controller.view_compile()`.'); + return this.view_compile(body, model, headers, partial, key); +}; + +Controller.prototype.view_compile = function(body, model, headers, partial, key) { if (headers === true) { key = partial; diff --git a/internal.js b/internal.js index fbd3d59b3..0e4ea1615 100755 --- a/internal.js +++ b/internal.js @@ -2247,7 +2247,6 @@ function view_prepare(command, dynamicCommand, functions, controller, components case 'routeDownload': case 'routeStatic': case 'routeVideo': - return 'self.' + command; case 'public_js': case 'public_css': case 'public_image': @@ -2255,7 +2254,6 @@ function view_prepare(command, dynamicCommand, functions, controller, components case 'public_download': case 'public_video': case 'public': - return 'self.' + command; case 'translate': return 'self.' + command; case 'json': @@ -2270,6 +2268,7 @@ function view_prepare(command, dynamicCommand, functions, controller, components case 'template': case 'templateToggle': case 'viewCompile': + case 'view_compile': case 'viewToggle': case 'download': case 'selected': diff --git a/test/controllers/default.js b/test/controllers/default.js index d2a59f9cd..1431f06a2 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -426,7 +426,7 @@ function viewTest() { } function viewDynamic() { - this.viewCompile('@{model.name}', { name: 'Peter' }); + this.viewcompile('@{model.name}', { name: 'Peter' }); } function viewTranslate() { @@ -561,7 +561,6 @@ function viewViews() { assert.ok(output.contains('#options-empty#'), name + 'options() - without property name and value'); assert.ok(output.contains('#options#'), name + 'options() - with property name and value'); assert.ok(output.contains('#view#bmodel##'), name + 'view() with model'); - assert.ok(output.contains('#view-toggle#'), name + 'viewToggle()'); assert.ok(output.contains('#titleTITLE#'), name + 'title'); assert.ok(output.contains('#routejs-/js/p.js#'), name + 'route to static'); assert.ok(output.contains('#content#'), name + 'download'); diff --git a/test/views/a.html b/test/views/a.html index 2eb785f5f..ab2b5c586 100755 --- a/test/views/a.html +++ b/test/views/a.html @@ -30,7 +30,6 @@ #resource@{resource('name')}# #options-empty@{options(repository.optionsEmpty, 'B')}# #options@{options(R.options, 'C', 'k', 'v')}# -#view-toggle@{viewToggle(false, 'b')}# #view@{view('current/b', 'model')}# #routejs-@{public_js('p.js')}# #title@{title}# @@ -39,7 +38,7 @@ #@{download('test.pdf', 'content', 'test')}# a @{place('footer', 'PLACE')} -#dynamic@{viewCompile('OK')}# +#dynamic@{view_compile('OK')}# ') : url; }; -Controller.prototype.routeStyle = function(name, tag, path) { +ControllerProto.routeStyle = function(name, tag, path) { OBSOLETE('controller.routeStyle()', 'Was renamed to "controller.public_css()"'); return this.public_css(name, tag, path); }; -Controller.prototype.public_css = function(name, tag, path) { +ControllerProto.public_css = function(name, tag, path) { var self = this; @@ -12679,48 +12685,48 @@ Controller.prototype.public_css = function(name, tag, path) { return tag ? '' : url; }; -Controller.prototype.routeImage = function(name) { +ControllerProto.routeImage = function(name) { OBSOLETE('controller.routeImage()', 'Was renamed to "controller.public_image()"'); return this.public_image(name); }; -Controller.prototype.public_image = function(name) { +ControllerProto.public_image = function(name) { return this.$static(name, F.public_image); }; -Controller.prototype.routeVideo = function(name) { +ControllerProto.routeVideo = function(name) { OBSOLETE('controller.routeVideo()', 'Was renamed to "controller.public_video()"'); return this.public_video(name); }; -Controller.prototype.public_video = function(name) { +ControllerProto.public_video = function(name) { return this.$static(name, F.public_video); }; -Controller.prototype.routeFont = function(name) { +ControllerProto.routeFont = function(name) { OBSOLETE('controller.routeFont()', 'Was renamed to "controller.public_font()"'); return this.public_font(name); }; -Controller.prototype.public_font = function(name) { +ControllerProto.public_font = function(name) { return F.public_font(name); }; -Controller.prototype.routeDownload = function(name) { +ControllerProto.routeDownload = function(name) { OBSOLETE('controller.routeDownload()', 'Was renamed to "controller.public_download()"'); return this.public_download(name); }; -Controller.prototype.public_download = function(name) { +ControllerProto.public_download = function(name) { return this.$static(name, F.public_download); }; -Controller.prototype.routeStatic = function(name, path) { +ControllerProto.routeStatic = function(name, path) { OBSOLETE('controller.routeStatic()', 'Was renamed to "controller.public()"'); return this.public(name, path); }; -Controller.prototype.public = function(name, path) { +ControllerProto.public = function(name, path) { var url = this.$static(name, F.public); if (path && U.isRelative(url)) return F.isWindows ? U.join(path, url) : U.join(path, url).substring(1); @@ -12733,7 +12739,7 @@ Controller.prototype.public = function(name, path) { * @param {Object} model A model, optional. * @return {String} */ -Controller.prototype.template = function(name, model) { +ControllerProto.template = function(name, model) { OBSOLETE('controller.template()', 'This method will be removed in v4.'); return this.view(name, model, true); }; @@ -12743,7 +12749,7 @@ Controller.prototype.template = function(name, model) { * @param {String} name A helper name. * @return {String} */ -Controller.prototype.helper = function(name) { +ControllerProto.helper = function(name) { var helper = F.helpers[name]; if (!helper) return ''; @@ -12763,7 +12769,7 @@ Controller.prototype.helper = function(name) { * @param {Function(key, value)} replacer JSON replacer. * @return {Controller} */ -Controller.prototype.json = function(obj, headers, beautify, replacer) { +ControllerProto.json = function(obj, headers, beautify, replacer) { var self = this; var res = self.res; @@ -12819,7 +12825,7 @@ Controller.prototype.json = function(obj, headers, beautify, replacer) { return self; }; -Controller.prototype.success = function(is, value) { +ControllerProto.success = function(is, value) { return this.json(SUCCESS(is === undefined ? true : is, value)); }; @@ -12832,7 +12838,7 @@ Controller.prototype.success = function(is, value) { * @param {Function} replacer Optional, the JSON replacer. * @return {Controller} */ -Controller.prototype.jsonp = function(name, obj, headers, beautify, replacer) { +ControllerProto.jsonp = function(name, obj, headers, beautify, replacer) { var self = this; var res = self.res; @@ -12886,7 +12892,7 @@ Controller.prototype.jsonp = function(name, obj, headers, beautify, replacer) { * @param {String} view Optional, undefined or null returns JSON. * @return {Function} */ -Controller.prototype.callback = function(view) { +ControllerProto.callback = function(view) { var self = this; return function(err, data) { @@ -12922,7 +12928,7 @@ Controller.prototype.callback = function(view) { }; }; -Controller.prototype.custom = function() { +ControllerProto.custom = function() { if (this.res.success) return false; this.res.$custom(); @@ -12934,22 +12940,22 @@ Controller.prototype.custom = function() { * @param {Boolean} enable Optional, default `true`. * @return {Controller} */ -Controller.prototype.noClear = function(enable) { +ControllerProto.noClear = function(enable) { OBSOLETE('controller.noClear()', 'You need to use controller.autoclear(false)'); this.req._manual = enable === undefined ? true : enable; return this; }; -Controller.prototype.autoclear = function(enable) { +ControllerProto.autoclear = function(enable) { this.req._manual = enable === false; return this; }; -Controller.prototype.html = function(body, headers) { +ControllerProto.html = function(body, headers) { return this.content(body, 'text/html', headers); }; -Controller.prototype.content = function(body, type, headers) { +ControllerProto.content = function(body, type, headers) { var self = this; var res = self.res; @@ -12993,7 +12999,7 @@ Controller.prototype.content = function(body, type, headers) { * @param {Boolean} headers A custom headers. * @return {Controller} */ -Controller.prototype.plain = function(body, headers) { +ControllerProto.plain = function(body, headers) { var self = this; var res = self.res; @@ -13033,7 +13039,7 @@ Controller.prototype.plain = function(body, headers) { * @param {Object/Number} headers A custom headers or a custom HTTP status. * @return {Controller} */ -Controller.prototype.empty = function(headers) { +ControllerProto.empty = function(headers) { var self = this; var res = self.res; @@ -13058,7 +13064,7 @@ Controller.prototype.empty = function(headers) { * @param {Object/Number} headers A custom headers or a custom HTTP status. * @return {Controller} */ -Controller.prototype.nocontent = function(headers) { +ControllerProto.nocontent = function(headers) { var self = this; var res = self.res; res.writeHead(204, headers); @@ -13073,7 +13079,7 @@ Controller.prototype.nocontent = function(headers) { * @param {String} problem Optional. * @return {Controller} */ -Controller.prototype.destroy = function(problem) { +ControllerProto.destroy = function(problem) { var self = this; problem && self.problem(problem); @@ -13094,7 +13100,7 @@ Controller.prototype.destroy = function(problem) { * @param {Function} done Optinoal, callback. * @return {Controller} */ -Controller.prototype.file = function(filename, download, headers, done) { +ControllerProto.file = function(filename, download, headers, done) { if (filename[0] === '~') filename = filename.substring(1); @@ -13118,7 +13124,7 @@ Controller.prototype.file = function(filename, download, headers, done) { * @param {Function} done Optional, callback. * @return {Controller} */ -Controller.prototype.image = function(filename, make, headers, done) { +ControllerProto.image = function(filename, make, headers, done) { var res = this.res; @@ -13148,7 +13154,7 @@ Controller.prototype.image = function(filename, make, headers, done) { * @param {Function} done Optinoal, callback. * @return {Controller} */ -Controller.prototype.stream = function(type, stream, download, headers, done, nocompress) { +ControllerProto.stream = function(type, stream, download, headers, done, nocompress) { var res = this.res; res.options.type = type; res.options.stream = stream; @@ -13165,7 +13171,7 @@ Controller.prototype.stream = function(type, stream, download, headers, done, no * @param {String} problem Description of problem (optional) * @return {Controller} */ -Controller.prototype.throw400 = Controller.prototype.view400 = function(problem) { +ControllerProto.throw400 = ControllerProto.view400 = function(problem) { return controller_error_status(this, 400, problem); }; @@ -13174,7 +13180,7 @@ Controller.prototype.throw400 = Controller.prototype.view400 = function(problem) * @param {String} problem Description of problem (optional) * @return {Controller} */ -Controller.prototype.throw401 = Controller.prototype.view401 = function(problem) { +ControllerProto.throw401 = ControllerProto.view401 = function(problem) { return controller_error_status(this, 401, problem); }; @@ -13183,7 +13189,7 @@ Controller.prototype.throw401 = Controller.prototype.view401 = function(problem) * @param {String} problem Description of problem (optional) * @return {Controller} */ -Controller.prototype.throw403 = Controller.prototype.view403 = function(problem) { +ControllerProto.throw403 = ControllerProto.view403 = function(problem) { return controller_error_status(this, 403, problem); }; @@ -13192,7 +13198,7 @@ Controller.prototype.throw403 = Controller.prototype.view403 = function(problem) * @param {String} problem Description of problem (optional) * @return {Controller} */ -Controller.prototype.throw404 = Controller.prototype.view404 = function(problem) { +ControllerProto.throw404 = ControllerProto.view404 = function(problem) { return controller_error_status(this, 404, problem); }; @@ -13201,7 +13207,7 @@ Controller.prototype.throw404 = Controller.prototype.view404 = function(problem) * @param {String} problem Description of problem (optional) * @return {Controller} */ -Controller.prototype.throw409 = Controller.prototype.view409 = function(problem) { +ControllerProto.throw409 = ControllerProto.view409 = function(problem) { return controller_error_status(this, 409, problem); }; @@ -13210,7 +13216,7 @@ Controller.prototype.throw409 = Controller.prototype.view409 = function(problem) * @param {Error} error * @return {Controller} */ -Controller.prototype.throw500 = Controller.prototype.view500 = function(error) { +ControllerProto.throw500 = ControllerProto.view500 = function(error) { var self = this; F.error(error instanceof Error ? error : new Error((error || '').toString()), self.name, self.req.uri); return controller_error_status(self, 500, error); @@ -13221,7 +13227,7 @@ Controller.prototype.throw500 = Controller.prototype.view500 = function(error) { * @param {String} problem Description of the problem (optional) * @return {Controller} */ -Controller.prototype.throw501 = Controller.prototype.view501 = function(problem) { +ControllerProto.throw501 = ControllerProto.view501 = function(problem) { return controller_error_status(this, 501, problem); }; @@ -13230,7 +13236,7 @@ Controller.prototype.throw501 = Controller.prototype.view501 = function(problem) * @param {String} problem Description of the problem (optional) * @return {Controller} */ -Controller.prototype.throw503 = Controller.prototype.view503 = function(problem) { +ControllerProto.throw503 = ControllerProto.view503 = function(problem) { return controller_error_status(this, 503, problem); }; @@ -13240,7 +13246,7 @@ Controller.prototype.throw503 = Controller.prototype.view503 = function(problem) * @param {Boolean} permanent Is permanent? Default: `false` * @return {Controller} */ -Controller.prototype.redirect = function(url, permanent) { +ControllerProto.redirect = function(url, permanent) { this.precache && this.precache(null, null, null); var res = this.res; res.options.url = url; @@ -13258,7 +13264,7 @@ Controller.prototype.redirect = function(url, permanent) { * @param {Object} headers Optional, additional headers. * @return {Controller} */ -Controller.prototype.binary = function(buffer, type, encoding, download, headers) { +ControllerProto.binary = function(buffer, type, encoding, download, headers) { var res = this.res; @@ -13288,7 +13294,7 @@ Controller.prototype.binary = function(buffer, type, encoding, download, headers * @param {String} label * @return {Object} */ -Controller.prototype.baa = function(label) { +ControllerProto.baa = function(label) { var self = this; self.precache && self.precache(null, null, null); @@ -13319,7 +13325,7 @@ Controller.prototype.baa = function(label) { * @param {Number} retry A reconnection timeout in milliseconds when is an unexpected problem. * @return {Controller} */ -Controller.prototype.sse = function(data, eventname, id, retry) { +ControllerProto.sse = function(data, eventname, id, retry) { var self = this; var res = self.res; @@ -13376,7 +13382,7 @@ Controller.prototype.sse = function(data, eventname, id, retry) { * @param {Boolean} end * @return {Controller} */ -Controller.prototype.close = function(end) { +ControllerProto.close = function(end) { var self = this; if (end === undefined) @@ -13417,7 +13423,7 @@ Controller.prototype.close = function(end) { * @param {Number} timeout Optional, timeout (default: 10000) * @return {EventEmitter} */ -Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callback, headers, timeout) { +ControllerProto.proxy = ControllerProto.proxy2 = function(url, callback, headers, timeout) { if (typeof(callback) === 'object') { timeout = headers; @@ -13505,7 +13511,7 @@ Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callbac * @param {Boolean} isPartial When is `true` the method returns rendered HTML as `String` * @return {Controller/String} */ -Controller.prototype.view = function(name, model, headers, partial, noasync, cachekey) { +ControllerProto.view = function(name, model, headers, partial, noasync, cachekey) { var self = this; @@ -13619,12 +13625,12 @@ Controller.prototype.view = function(name, model, headers, partial, noasync, cac return self.$viewrender(filename, framework_internal.viewEngine(name, filename, self), model, headers, partial, isLayout, noasync, cachekey); }; -Controller.prototype.viewCompile = function(body, model, headers, partial, key) { +ControllerProto.viewCompile = function(body, model, headers, partial, key) { OBSOLETE('controller.viewCompile()', 'Was renamed to `controller.view_compile()`.'); return this.view_compile(body, model, headers, partial, key); }; -Controller.prototype.view_compile = function(body, model, headers, partial, key) { +ControllerProto.view_compile = function(body, model, headers, partial, key) { if (headers === true) { key = partial; @@ -13641,7 +13647,7 @@ Controller.prototype.view_compile = function(body, model, headers, partial, key) return this.$viewrender('[dynamic view]', framework_internal.viewEngineCompile(body, this.language, this, key), model, headers, partial); }; -Controller.prototype.$viewrender = function(filename, generator, model, headers, partial, isLayout, noasync, cachekey) { +ControllerProto.$viewrender = function(filename, generator, model, headers, partial, isLayout, noasync, cachekey) { var self = this; var err; @@ -13873,7 +13879,7 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, * @param {Function()} fnFrom This method is executed when the content is readed from the cache. * @return {Controller} */ -Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { +ControllerProto.memorize = function(key, expires, disabled, fnTo, fnFrom) { var self = this; @@ -13943,7 +13949,7 @@ Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { return self; }; -Controller.prototype.$memorize_prepare = function(key, expires, disabled, fnTo, fnFrom) { +ControllerProto.$memorize_prepare = function(key, expires, disabled, fnTo, fnFrom) { var self = this; var pk = '$memorize' + key; @@ -14081,12 +14087,15 @@ WebSocket.prototype = { } }; -WebSocket.prototype.operation = function(name, value, callback, options) { + +const WebSocketProto = WebSocket.prototype; + +WebSocketProto.operation = function(name, value, callback, options) { OPERATION(name, value, callback, options, this); return this; }; -WebSocket.prototype.emit = function(name, a, b, c, d, e, f, g) { +WebSocketProto.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -14106,7 +14115,7 @@ WebSocket.prototype.emit = function(name, a, b, c, d, e, f, g) { return this; }; -WebSocket.prototype.on = function(name, fn) { +WebSocketProto.on = function(name, fn) { if (this.$events[name]) this.$events[name].push(fn); else @@ -14114,12 +14123,12 @@ WebSocket.prototype.on = function(name, fn) { return this; }; -WebSocket.prototype.once = function(name, fn) { +WebSocketProto.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -WebSocket.prototype.removeListener = function(name, fn) { +WebSocketProto.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -14131,7 +14140,7 @@ WebSocket.prototype.removeListener = function(name, fn) { return this; }; -WebSocket.prototype.removeAllListeners = function(name) { +WebSocketProto.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -14149,7 +14158,7 @@ WebSocket.prototype.removeAllListeners = function(name) { * @param {Function(key, value)} replacer for JSON (optional) * @return {WebSocket} */ -WebSocket.prototype.send = function(message, id, blacklist, replacer) { +WebSocketProto.send = function(message, id, blacklist, replacer) { var self = this; var keys = self._keys; @@ -14201,7 +14210,7 @@ WebSocket.prototype.send = function(message, id, blacklist, replacer) { return self; }; -WebSocket.prototype.send2 = function(message, comparer, replacer) { +WebSocketProto.send2 = function(message, comparer, replacer) { var self = this; var keys = self._keys; @@ -14245,7 +14254,7 @@ function websocket_valid_fn(id, client, fn, msg) { * Sends a ping message * @return {WebSocket} */ -WebSocket.prototype.ping = function() { +WebSocketProto.ping = function() { var keys = this._keys; if (!keys) @@ -14269,7 +14278,7 @@ WebSocket.prototype.ping = function() { * @param {Number} code Optional default 1000. * @return {Websocket} */ -WebSocket.prototype.close = function(id, message, code) { +WebSocketProto.close = function(id, message, code) { var keys = this._keys; @@ -14322,7 +14331,7 @@ WebSocket.prototype.close = function(id, message, code) { * @param {Error/String} err * @return {WebSocket/Function} */ -WebSocket.prototype.error = function(err) { +WebSocketProto.error = function(err) { var result = F.error(typeof(err) === 'string' ? new Error(err) : err, this.name, this.path); return err ? this : result; }; @@ -14332,7 +14341,7 @@ WebSocket.prototype.error = function(err) { * @param {String} message * @return {WebSocket} */ -WebSocket.prototype.wtf = WebSocket.prototype.problem = function(message) { +WebSocketProto.wtf = WebSocketProto.problem = function(message) { F.problem(message, this.name, this.uri); return this; }; @@ -14342,7 +14351,7 @@ WebSocket.prototype.wtf = WebSocket.prototype.problem = function(message) { * @param {String} message * @return {WebSocket} */ -WebSocket.prototype.change = function(message) { +WebSocketProto.change = function(message) { F.change(message, this.name, this.uri, this.ip); return this; }; @@ -14352,7 +14361,7 @@ WebSocket.prototype.change = function(message) { * @param {Function(connection, index)} fn * @return {WebSocket} */ -WebSocket.prototype.all = function(fn) { +WebSocketProto.all = function(fn) { var arr = fn == null || fn == true ? [] : null; var self = this; if (self._keys) { @@ -14371,7 +14380,7 @@ WebSocket.prototype.all = function(fn) { * @param {String} id * @return {WebSocketClient} */ -WebSocket.prototype.find = function(id) { +WebSocketProto.find = function(id) { var self = this; if (!self._keys) @@ -14396,7 +14405,7 @@ WebSocket.prototype.find = function(id) { * @param {String} problem Optional. * @return {WebSocket} */ -WebSocket.prototype.destroy = function(problem) { +WebSocketProto.destroy = function(problem) { var self = this; problem && self.problem(problem); @@ -14432,7 +14441,7 @@ WebSocket.prototype.destroy = function(problem) { * @param {Function} callback * @return {WebSocket] */ -WebSocket.prototype.autodestroy = function(callback) { +WebSocketProto.autodestroy = function(callback) { var self = this; var key = 'websocket:' + self.id; self.on('open', () => clearTimeout2(key)); @@ -14449,7 +14458,7 @@ WebSocket.prototype.autodestroy = function(callback) { * Internal function * @return {WebSocket} */ -WebSocket.prototype.$refresh = function() { +WebSocketProto.$refresh = function() { if (this.connections) { this._keys = Object.keys(this.connections); this.online = this._keys.length; @@ -14463,7 +14472,7 @@ WebSocket.prototype.$refresh = function() { * @param {String} id * @return {WebSocket} */ -WebSocket.prototype.$remove = function(id) { +WebSocketProto.$remove = function(id) { if (this.connections) delete this.connections[id]; return this; @@ -14474,7 +14483,7 @@ WebSocket.prototype.$remove = function(id) { * @param {WebSocketClient} client * @return {WebSocket} */ -WebSocket.prototype.$add = function(client) { +WebSocketProto.$add = function(client) { this.connections[client._id] = client; return this; }; @@ -14485,21 +14494,21 @@ WebSocket.prototype.$add = function(client) { * @param {String} key A resource key. * @return {String} */ -WebSocket.prototype.resource = function(name, key) { +WebSocketProto.resource = function(name, key) { return F.resource(name, key); }; -WebSocket.prototype.log = function() { +WebSocketProto.log = function() { F.log.apply(framework, arguments); return this; }; -WebSocket.prototype.logger = function() { +WebSocketProto.logger = function() { F.logger.apply(framework, arguments); return this; }; -WebSocket.prototype.check = function() { +WebSocketProto.check = function() { this.$ping && this.all(websocketcheck_ping); return this; }; @@ -14560,13 +14569,19 @@ WebSocketClient.prototype = { }, get config() { + OBSOLETE('controller.config', 'Use: CONF'); return this.container.config; }, get global() { + OBSOLETE('controller.global', 'Use: G'); return this.container.global; }, + get sessionid() { + return this.req.sessionid; + }, + get session() { return this.req.session; }, @@ -14588,13 +14603,15 @@ WebSocketClient.prototype = { } }; -WebSocketClient.prototype.isWebSocket = true; +const WebSocketClientProto = WebSocketClient.prototype; + +WebSocketClientProto.isWebSocket = true; -WebSocketClient.prototype.cookie = function(name) { +WebSocketClientProto.cookie = function(name) { return this.req.cookie(name); }; -WebSocketClient.prototype.$close = function(code, message) { +WebSocketClientProto.$close = function(code, message) { var self = this; @@ -14616,7 +14633,7 @@ WebSocketClient.prototype.$close = function(code, message) { return self; }; -WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { +WebSocketClientProto.prepare = function(flags, protocols, allow, length) { flags = flags || EMPTYARRAY; protocols = protocols || EMPTYARRAY; @@ -14710,7 +14727,7 @@ function websocket_deflate(data) { * @param {WebSocket} container * @return {WebSocketClient} */ -WebSocketClient.prototype.upgrade = function(container) { +WebSocketClientProto.upgrade = function(container) { var self = this; self.req.on('error', websocket_onerror); self.container = container; @@ -14740,7 +14757,7 @@ function websocket_close() { this.$websocket.$onclose(); } -WebSocketClient.prototype.$ondata = function(data) { +WebSocketClientProto.$ondata = function(data) { if (this.isClosed) return; @@ -14845,7 +14862,7 @@ function buffer_concat(buffers, length) { // MIT // Written by Jozef Gula // Optimized by Peter Sirka -WebSocketClient.prototype.$parse = function() { +WebSocketClientProto.$parse = function() { var self = this; var current = self.current; @@ -14916,7 +14933,7 @@ WebSocketClient.prototype.$parse = function() { return true; }; -WebSocketClient.prototype.$readbody = function() { +WebSocketClientProto.$readbody = function() { var current = this.current; var length = current.data.length; @@ -14948,7 +14965,7 @@ WebSocketClient.prototype.$readbody = function() { } }; -WebSocketClient.prototype.$decode = function() { +WebSocketClientProto.$decode = function() { var data = this.current.body; switch (this.type) { @@ -14978,7 +14995,7 @@ WebSocketClient.prototype.$decode = function() { this.current.body = null; }; -WebSocketClient.prototype.parseInflate = function() { +WebSocketClientProto.parseInflate = function() { var self = this; if (self.inflatelock) @@ -15019,7 +15036,7 @@ WebSocketClient.prototype.parseInflate = function() { } }; -WebSocketClient.prototype.$onerror = function(err) { +WebSocketClientProto.$onerror = function(err) { if (this.isClosed) return; @@ -15031,7 +15048,7 @@ WebSocketClient.prototype.$onerror = function(err) { this.container.$events.error && this.container.emit('error', err, this); }; -WebSocketClient.prototype.$onclose = function() { +WebSocketClientProto.$onclose = function() { if (this._isClosed) return; @@ -15063,7 +15080,7 @@ WebSocketClient.prototype.$onclose = function() { * @param {Boolean} raw The message won't be converted e.g. to JSON. * @return {WebSocketClient} */ -WebSocketClient.prototype.send = function(message, raw, replacer) { +WebSocketClientProto.send = function(message, raw, replacer) { var self = this; @@ -15090,7 +15107,7 @@ WebSocketClient.prototype.send = function(message, raw, replacer) { return self; }; -WebSocketClient.prototype.sendDeflate = function() { +WebSocketClientProto.sendDeflate = function() { var self = this; if (self.deflatelock) @@ -15119,7 +15136,7 @@ WebSocketClient.prototype.sendDeflate = function() { * Ping message * @return {WebSocketClient} */ -WebSocketClient.prototype.ping = function() { +WebSocketClientProto.ping = function() { if (!this.isClosed) { this.socket.write(U.getWebSocketFrame(0, 'PING', 0x09)); this.$ping = false; @@ -15133,7 +15150,7 @@ WebSocketClient.prototype.ping = function() { * @param {Number} code WebSocket code. * @return {WebSocketClient} */ -WebSocketClient.prototype.close = function(message, code) { +WebSocketClientProto.close = function(message, code) { var self = this; if (!self.isClosed) { self.isClosed = true; @@ -15151,7 +15168,7 @@ WebSocketClient.prototype.close = function(message, code) { * @param {Request} req * @return {String} */ -WebSocketClient.prototype.$websocket_key = function(req) { +WebSocketClientProto.$websocket_key = function(req) { var sha1 = Crypto.createHash('sha1'); sha1.update((req.headers['sec-websocket-key'] || '') + SOCKET_HASH); return sha1.digest('base64'); @@ -17469,12 +17486,14 @@ MiddlewareOptions.prototype = { } }; -MiddlewareOptions.prototype.callback = function() { +const MiddlewareOptionsProto = MiddlewareOptions.prototype; + +MiddlewareOptionsProto.callback = function() { this.next(); return this; }; -MiddlewareOptions.prototype.cancel = function() { +MiddlewareOptionsProto.cancel = function() { this.next(false); return this; }; diff --git a/internal.js b/internal.js index 0e4ea1615..bb99f1acd 100755 --- a/internal.js +++ b/internal.js @@ -1347,6 +1347,7 @@ function MultipartParser() { } exports.MultipartParser = MultipartParser; +const MultipartParserProto = MultipartParser.prototype; MultipartParser.stateToString = function(stateNumber) { for (var state in S) { @@ -1355,7 +1356,7 @@ MultipartParser.stateToString = function(stateNumber) { } }; -MultipartParser.prototype.initWithBoundary = function(str) { +MultipartParserProto.initWithBoundary = function(str) { var self = this; self.boundary = Buffer.alloc(str.length + 4); self.boundary.write('\r\n--', 0, 'ascii'); @@ -1367,7 +1368,7 @@ MultipartParser.prototype.initWithBoundary = function(str) { self.boundaryChars[self.boundary[i]] = true; }; -MultipartParser.prototype.write = function(buffer) { +MultipartParserProto.write = function(buffer) { var self = this, i = 0, len = buffer.length, @@ -1596,7 +1597,7 @@ MultipartParser.prototype.write = function(buffer) { return len; }; -MultipartParser.prototype.end = function() { +MultipartParserProto.end = function() { if ((this.state === S.HEADER_FIELD_START && this.index === 0) || (this.state === S.PART_DATA && this.index == this.boundary.length)) { this.onPartEnd && this.onPartEnd(); this.onEnd && this.onEnd(); @@ -1607,7 +1608,7 @@ MultipartParser.prototype.end = function() { } }; -MultipartParser.prototype.explain = function() { +MultipartParserProto.explain = function() { return 'state = ' + MultipartParser.stateToString(this.state); }; diff --git a/nosqlstream.js b/nosqlstream.js index f210bcd7f..437cb1ac9 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -54,8 +54,10 @@ function NoSQLStream(filename) { // this.indexer = 0; } +const NoSQLStreamProto = NoSQLStream.prototype; + // Because of performance -NoSQLStream.prototype.readhelpers = function() { +NoSQLStreamProto.readhelpers = function() { var self = this; @@ -357,7 +359,7 @@ NoSQLStream.prototype.readhelpers = function() { }; // Because of performance -NoSQLStream.prototype.writehelpers = function() { +NoSQLStreamProto.writehelpers = function() { var self = this; @@ -511,7 +513,7 @@ NoSQLStream.prototype.writehelpers = function() { }; }; -NoSQLStream.prototype.openread = function() { +NoSQLStreamProto.openread = function() { var self = this; self.type = 'r'; self.position = self.start; @@ -519,7 +521,7 @@ NoSQLStream.prototype.openread = function() { return self; }; -NoSQLStream.prototype.openreadreverse = function() { +NoSQLStreamProto.openreadreverse = function() { var self = this; self.type = 'r'; self.position = self.start; @@ -528,7 +530,7 @@ NoSQLStream.prototype.openreadreverse = function() { return self; }; -NoSQLStream.prototype.openupdate = function() { +NoSQLStreamProto.openupdate = function() { var self = this; self.type = 'r+'; Fs.open(self.filename, self.type, function(err, fd) { @@ -568,7 +570,7 @@ NoSQLStream.prototype.openupdate = function() { }; // For e.g. files on URL address -NoSQLStream.prototype.openstream = function(stream) { +NoSQLStreamProto.openstream = function(stream) { var self = this; @@ -592,7 +594,7 @@ NoSQLStream.prototype.openstream = function(stream) { return self; }; -NoSQLStream.prototype.open = function() { +NoSQLStreamProto.open = function() { var self = this; Fs.open(self.filename, self.type, function(err, fd) { @@ -624,7 +626,7 @@ NoSQLStream.prototype.open = function() { }); }; -NoSQLStream.prototype.close = function() { +NoSQLStreamProto.close = function() { var self = this; @@ -656,21 +658,21 @@ NoSQLStream.prototype.close = function() { return self; }; -NoSQLStream.prototype.write = function(doc, position) { +NoSQLStreamProto.write = function(doc, position) { var self = this; self.bufferstack.push({ position: position, data: doc }); !self.writing && self.$write(); return self; }; -NoSQLStream.prototype.write2 = function(doc) { +NoSQLStreamProto.write2 = function(doc) { var self = this; self.bufferstacknew.push(Buffer.from(doc)); !self.writing && self.$write(); return self; }; -NoSQLStream.prototype.$write = function() { +NoSQLStreamProto.$write = function() { var self = this; if (self.bufferstacknew.length && self.bufferstack.length) { self.writing = true; @@ -689,7 +691,7 @@ NoSQLStream.prototype.$write = function() { } }; -NoSQLStream.prototype.flush = function() { +NoSQLStreamProto.flush = function() { var self = this; if (self.writing) setTimeout(self.cb_flush, 100); @@ -698,7 +700,7 @@ NoSQLStream.prototype.flush = function() { return self; }; -NoSQLStream.prototype.read = function() { +NoSQLStreamProto.read = function() { var self = this; var size = self.stats.size - self.position; @@ -727,14 +729,14 @@ NoSQLStream.prototype.read = function() { } }; -NoSQLStream.prototype.readreverse = function() { +NoSQLStreamProto.readreverse = function() { var self = this; self.position = self.stats.size; self.readreverse2(); return self; }; -NoSQLStream.prototype.readreverse2 = function() { +NoSQLStreamProto.readreverse2 = function() { var self = this; if (!self.fd || self.position <= self.start || self.canceled) { @@ -762,7 +764,7 @@ NoSQLStream.prototype.readreverse2 = function() { } }; -NoSQLStream.prototype.readupdate = function() { +NoSQLStreamProto.readupdate = function() { var self = this; var size = self.stats.size - self.position; diff --git a/session.js b/session.js index 6d0dcc5ae..30efd0128 100644 --- a/session.js +++ b/session.js @@ -43,6 +43,8 @@ function Session(name, ondata) { }; } +const SessionProto = Session.prototype; + /* Session.prototype.find = function(filter, callback) { @@ -74,7 +76,7 @@ Session.prototype.find = function(filter, callback) { callback(null, arr); };*/ -Session.prototype.list = function(id, callback) { +SessionProto.list = function(id, callback) { var self = this; var arr = []; @@ -93,11 +95,11 @@ Session.prototype.list = function(id, callback) { callback(null, arr); }; -Session.prototype.has = function(sessionid, callback) { +SessionProto.has = function(sessionid, callback) { callback(null, this.items.has(sessionid)); }; -Session.prototype.has2 = function(id, callback) { +SessionProto.has2 = function(id, callback) { for (var m of this.items.values()) { if (m && m.expire >= NOW && m.id === id) { callback(null, true); @@ -107,7 +109,7 @@ Session.prototype.has2 = function(id, callback) { callback(null, false); }; -Session.prototype.getcookie = function(req, opt, callback) { +SessionProto.getcookie = function(req, opt, callback) { // opt.name {String} A cookie name // opt.expire {String} Expiration @@ -153,7 +155,7 @@ Session.prototype.getcookie = function(req, opt, callback) { } }; -Session.prototype.usage = function() { +SessionProto.usage = function() { var o = {}; o.used = 0; o.free = 0; @@ -167,7 +169,7 @@ Session.prototype.usage = function() { return o; }; -Session.prototype.release = function(sessionid, expire, callback) { +SessionProto.release = function(sessionid, expire, callback) { if (sessionid && sessionid.sessionid) sessionid = sessionid.sessionid; @@ -212,7 +214,7 @@ Session.prototype.release = function(sessionid, expire, callback) { } }; -Session.prototype.release2 = function(id, expire, callback) { +SessionProto.release2 = function(id, expire, callback) { if (typeof(expire) === 'function') { callback = expire; @@ -242,7 +244,7 @@ Session.prototype.release2 = function(id, expire, callback) { callback && callback(null, count); }; -Session.prototype.releaseunused = function(lastusage, callback) { +SessionProto.releaseunused = function(lastusage, callback) { var self = this; var count = 0; @@ -259,7 +261,7 @@ Session.prototype.releaseunused = function(lastusage, callback) { callback && callback(null, count); }; -Session.prototype.setcookie = function(res, opt, callback) { +SessionProto.setcookie = function(res, opt, callback) { // opt.name {String} A cookie name // opt.sessionid {String} A unique session ID @@ -291,7 +293,7 @@ Session.prototype.setcookie = function(res, opt, callback) { }); }; -Session.prototype.set2 = function(id, data, expire, note, settings, callback) { +SessionProto.set2 = function(id, data, expire, note, settings, callback) { if (typeof(expire) === 'function') { callback = expire; @@ -324,7 +326,7 @@ Session.prototype.set2 = function(id, data, expire, note, settings, callback) { updated && self.$save(); }; -Session.prototype.set = function(sessionid, id, data, expire, note, settings, callback) { +SessionProto.set = function(sessionid, id, data, expire, note, settings, callback) { if (typeof(id) === 'object') { callback = settings; @@ -357,7 +359,7 @@ Session.prototype.set = function(sessionid, id, data, expire, note, settings, ca self.$save(); }; -Session.prototype.get2 = function(id, callback) { +SessionProto.get2 = function(id, callback) { var self = this; var output = []; for (var m of self.items.values()) { @@ -369,7 +371,7 @@ Session.prototype.get2 = function(id, callback) { callback && callback(null, output); }; -Session.prototype.get = function(sessionid, expire, callback) { +SessionProto.get = function(sessionid, expire, callback) { if (typeof(expire) === 'function') { callback = expire; @@ -406,7 +408,7 @@ Session.prototype.get = function(sessionid, expire, callback) { item.used = NOW; }; -Session.prototype.update2 = function(id, data, expire, note, settings, callback) { +SessionProto.update2 = function(id, data, expire, note, settings, callback) { if (typeof(expire) === 'function') { callback = expire; @@ -444,7 +446,7 @@ Session.prototype.update2 = function(id, data, expire, note, settings, callback) updated && self.$save(); }; -Session.prototype.update = function(sessionid, data, expire, note, settings, callback) { +SessionProto.update = function(sessionid, data, expire, note, settings, callback) { if (typeof(expire) === 'function') { callback = expire; @@ -486,7 +488,7 @@ Session.prototype.update = function(sessionid, data, expire, note, settings, cal callback(); }; -Session.prototype.count = function(id, callback) { +SessionProto.count = function(id, callback) { if (!callback) { callback = id; @@ -510,7 +512,7 @@ Session.prototype.count = function(id, callback) { callback(null, o); }; -Session.prototype.remove2 = function(id, callback) { +SessionProto.remove2 = function(id, callback) { var self = this; var count = 0; for (var m of self.items.values()) { @@ -524,7 +526,7 @@ Session.prototype.remove2 = function(id, callback) { self.$save(); }; -Session.prototype.remove = function(sessionid, callback) { +SessionProto.remove = function(sessionid, callback) { if (sessionid && sessionid.sessionid) sessionid = sessionid.sessionid; @@ -541,7 +543,7 @@ Session.prototype.remove = function(sessionid, callback) { self.onremove && self.onremove(item); }; -Session.prototype.clear = function(lastusage, callback) { +SessionProto.clear = function(lastusage, callback) { if (typeof(lastusage) === 'function') { callback = lastusage; @@ -577,7 +579,7 @@ Session.prototype.clear = function(lastusage, callback) { self.$save(); }; -Session.prototype.clean = function() { +SessionProto.clean = function() { var self = this; var is = false; for (var m of self.items.values()) { @@ -590,7 +592,7 @@ Session.prototype.clean = function() { is && self.$save(); }; -Session.prototype.load = function(callback) { +SessionProto.load = function(callback) { var self = this; var removed = 0; diff --git a/websocketclient.js b/websocketclient.js index fc992d5b5..aa305a6e9 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -50,7 +50,9 @@ function WebSocketClient() { this.headers = {}; } -WebSocketClient.prototype.connect = function(url, protocol, origin) { +const WebSocketClientProto = WebSocketClient.prototype; + +WebSocketClientProto.connect = function(url, protocol, origin) { var self = this; var options = {}; @@ -165,7 +167,7 @@ function websocket_close() { }, ws.options.reconnect, ws); } -WebSocketClient.prototype.emit = function(name, a, b, c, d, e, f, g) { +WebSocketClientProto.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -185,7 +187,7 @@ WebSocketClient.prototype.emit = function(name, a, b, c, d, e, f, g) { return this; }; -WebSocketClient.prototype.on = function(name, fn) { +WebSocketClientProto.on = function(name, fn) { if (this.$events[name]) this.$events[name].push(fn); else @@ -193,12 +195,12 @@ WebSocketClient.prototype.on = function(name, fn) { return this; }; -WebSocketClient.prototype.once = function(name, fn) { +WebSocketClientProto.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -WebSocketClient.prototype.removeListener = function(name, fn) { +WebSocketClientProto.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -210,7 +212,7 @@ WebSocketClient.prototype.removeListener = function(name, fn) { return this; }; -WebSocketClient.prototype.removeAllListeners = function(name) { +WebSocketClientProto.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -220,7 +222,7 @@ WebSocketClient.prototype.removeAllListeners = function(name) { return this; }; -WebSocketClient.prototype.free = function() { +WebSocketClientProto.free = function() { var self = this; self.socket && self.socket.destroy(); self.socket = null; @@ -234,7 +236,7 @@ WebSocketClient.prototype.free = function() { * @param {Buffer} data * @return {Framework} */ -WebSocketClient.prototype.$ondata = function(data) { +WebSocketClientProto.$ondata = function(data) { if (this.isClosed) return; @@ -334,7 +336,7 @@ function buffer_concat(buffers, length) { // MIT // Written by Jozef Gula // Optimized by Peter Sirka -WebSocketClient.prototype.$parse = function() { +WebSocketClientProto.$parse = function() { var self = this; var current = self.current; @@ -400,7 +402,7 @@ WebSocketClient.prototype.$parse = function() { return true; }; -WebSocketClient.prototype.$readbody = function() { +WebSocketClientProto.$readbody = function() { var current = this.current; var length = current.data.length; @@ -433,7 +435,7 @@ WebSocketClient.prototype.$readbody = function() { }; -WebSocketClient.prototype.$decode = function() { +WebSocketClientProto.$decode = function() { var data = this.current.body; switch (this.options.type) { @@ -459,7 +461,7 @@ WebSocketClient.prototype.$decode = function() { this.current.body = null; }; -WebSocketClient.prototype.parseInflate = function() { +WebSocketClientProto.parseInflate = function() { var self = this; if (self.inflatelock) @@ -495,7 +497,7 @@ WebSocketClient.prototype.parseInflate = function() { } }; -WebSocketClient.prototype.$onerror = function(err) { +WebSocketClientProto.$onerror = function(err) { this.$events.error && this.emit('error', err); if (!this.isClosed) { this.isClosed = true; @@ -503,7 +505,7 @@ WebSocketClient.prototype.$onerror = function(err) { } }; -WebSocketClient.prototype.$onclose = function() { +WebSocketClientProto.$onclose = function() { if (this._isClosed) return; @@ -533,7 +535,7 @@ WebSocketClient.prototype.$onclose = function() { * @param {Boolean} raw The message won't be converted e.g. to JSON. * @return {WebSocketClient} */ -WebSocketClient.prototype.send = function(message, raw, replacer) { +WebSocketClientProto.send = function(message, raw, replacer) { if (this.isClosed) return this; @@ -560,7 +562,7 @@ WebSocketClient.prototype.send = function(message, raw, replacer) { return this; }; -WebSocketClient.prototype.sendDeflate = function() { +WebSocketClientProto.sendDeflate = function() { var self = this; if (self.deflatelock) @@ -589,7 +591,7 @@ WebSocketClient.prototype.sendDeflate = function() { * Ping message * @return {WebSocketClient} */ -WebSocketClient.prototype.ping = function() { +WebSocketClientProto.ping = function() { if (!this.isClosed) { this.socket.write(U.getWebSocketFrame(0, '', 0x09)); this.$ping = false; @@ -613,7 +615,7 @@ function websocket_deflate(data) { * @param {Number} code WebSocket code. * @return {WebSocketClient} */ -WebSocketClient.prototype.close = function(message, code) { +WebSocketClientProto.close = function(message, code) { if (message !== true) { this.options.reconnect = 0; From 41266363d6a1bc7ab6b4714533b04597a25b54e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 26 Jun 2019 21:03:41 +0200 Subject: [PATCH 1094/1669] Updated changelog. --- changes.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 7d36da5e5..1e0837941 100755 --- a/changes.txt +++ b/changes.txt @@ -49,7 +49,7 @@ x added: `AUDIT(name, $, [type], message)` a method for audit logs x added: __obsolete__ message to older declaration of middleware, schemas and operations x added: `U.diffarr(prop, arr_A, arr_B)` for comparing of two arrays x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` -- added: `global.REQUIRE()` for local modules within app directory +x added: `global.REQUIRE()` for local modules within app directory x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS @@ -63,8 +63,8 @@ x updated: `REQUEST()` supports `keepalive` flag x updated: `bundles` mechanism supports merging files `--filename` between `bundles` files x updated: `String.hash(true)` returns unsigned int x updated: `CONF:default_image_converter` supports `magick` for new version of ImageMagick -- updated: `CONF.default_image_consumption` can contain ZERO value which means disabled optimialization of CPU/memory consumption -- updated: `CONF.default_image_consumption` has changed a default value to `0` +x updated: `CONF.default_image_consumption` can contain ZERO value which means disabled optimialization of CPU/memory consumption +x updated: `CONF.default_image_consumption` has changed a default value to `0` - fixed: schemas validation - fixed: error handling in dynamic schemas @@ -89,7 +89,7 @@ x renamed: `*.routeVideo` to `*.public_video` x renamed: `*.routeImage` to `*.public_image` x renamed: `*.routeDownload` to `*.public_download` x renamed: `*.routeStatic` to `*.public` -- renamed: `controller.viewCompile()` to `controller.view_compile()` +x renamed: `controller.viewCompile()` to `controller.view_compile()` x __removed: backward compatibility__ with older version of Node.js < 10 x removed: `F.hash()`, alternative `String.prototype.hash()` From 20fbffec68871a0db5e473d3bd956a1cbe57ce5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 26 Jun 2019 21:03:59 +0200 Subject: [PATCH 1095/1669] Fixed grammar. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 1e0837941..12c598d8a 100755 --- a/changes.txt +++ b/changes.txt @@ -4,7 +4,7 @@ x added: `NEWTASK(name, declaration)` for creating preddefined `TaskBuilder` x added: `TASK(name, taskname, callback, [controller/SchemaOptions/OperationOptions/ErrorBuilder])` for executing preddefined `TaskBuilder` x added: a new config key `directory_tasks` for `TaskBuilder` x added: a global alias `MODIFY()` for `F.modify()` -x added: a global alias `VIEWCOMPILE()` for `F.viewCompile()` +x added: a global alias `VIEWCOMPILE()` for `F.view_compile()` x added: `mail.type = 'html'` can be `html` (default) or `plain` x added: `$.headers` into the `SchemaOptions`, `OperationOptions` and `TaskBuilder` x added: `String.parseCSV([delimiter])` returns `Object Array` From 38bae6e2d511a51697724dff090885d3b9388317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 27 Jun 2019 21:09:10 +0200 Subject: [PATCH 1096/1669] Improved session module. --- session.js | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/session.js b/session.js index 30efd0128..f0586ea95 100644 --- a/session.js +++ b/session.js @@ -155,6 +155,34 @@ SessionProto.getcookie = function(req, opt, callback) { } }; +SessionProto.gettoken = function(req, opt, callback) { + + // opt.token {String} a token + // opt.expire {String} Expiration + // opt.key {String} Encrypt key + + if (req.req) + req = req.req; + + var token = opt.token; + if (!token || token.length < 20) { + callback(); + return; + } + + // IMPORTANT: "req.res" can be null cause of WebSocket + var value = DECRYPTREQ(req, token, opt.key); + if (value && typeof(value) === 'string') { + value = value.split(';'); + this.get(value[0], opt.expire, function(err, data, meta, init) { + if (!err && data) + req.sessionid = meta.sessionid; + callback(err, data, meta, init); + }); + } else + callback(); +}; + SessionProto.usage = function() { var o = {}; o.used = 0; @@ -293,6 +321,36 @@ SessionProto.setcookie = function(res, opt, callback) { }); }; +SessionProto.settoken = function(res, opt, callback) { + + // opt.name {String} A cookie name + // opt.sessionid {String} A unique session ID + // opt.id {String} Optional, custom ID + // opt.expire {String} Expiration + // opt.strict {Boolean} Strict comparing of cookie according to IP (default: false) + // opt.key {String} Encrypt key + // opt.data {Object} A session data + // opt.note {String} A simple note for this session + // opt.settings {String} Settings data for the session + + if (res.res) + res = res.res; + + if (!opt.sessionid) + opt.sessionid = UID(); + + this.set(opt.sessionid, opt.id, opt.data, opt.expire, opt.note, opt.settings, function(err, item, meta) { + if (err) { + callback && callback(err); + } else { + var data = opt.sessionid + ';' + (opt.id || ''); + var token = ENCRYPTREQ(res.req, data, opt.key, opt.strict); + res.req.sessionid = opt.sessionid; + callback && callback(null, item, meta, token); + } + }); +}; + SessionProto.set2 = function(id, data, expire, note, settings, callback) { if (typeof(expire) === 'function') { From 964f3367f0c5ee32a7de1fe3b089a8cf8ae63248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 27 Jun 2019 21:18:47 +0200 Subject: [PATCH 1097/1669] Reoreder arguments in `session.settoken()` callback. --- session.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/session.js b/session.js index f0586ea95..a5494f802 100644 --- a/session.js +++ b/session.js @@ -346,7 +346,7 @@ SessionProto.settoken = function(res, opt, callback) { var data = opt.sessionid + ';' + (opt.id || ''); var token = ENCRYPTREQ(res.req, data, opt.key, opt.strict); res.req.sessionid = opt.sessionid; - callback && callback(null, item, meta, token); + callback && callback(null, token, item, meta); } }); }; From c529835e9b4644cb94963b0f5b6c294ee6d1b004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 08:51:04 +0200 Subject: [PATCH 1098/1669] Renamed methods. --- image.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image.js b/image.js index 4664396e6..686bc0374 100755 --- a/image.js +++ b/image.js @@ -556,7 +556,7 @@ ImageProto.miniature = function(w, h, color, filter) { * @param {String} color Optional, background color. * @return {Image} */ -ImageProto.resizeCenter = function(w, h, color) { +ImageProto.resizeCenter = ImageProto.resize_center = function(w, h, color) { return this.resize(w, h, '^').background(color ? color : 'white').align('center').crop(w, h); }; @@ -568,7 +568,7 @@ ImageProto.resizeCenter = function(w, h, color) { * @param {String} color Optional, background color. * @return {Image} */ -ImageProto.resizeAlign = function(w, h, align, color) { +ImageProto.resizeAlign = ImageProto.resize_align = function(w, h, align, color) { return this.resize(w, h, '^').background(color ? color : 'white').align(align || 'center').crop(w, h); }; From 3c8ee14d4bee16d7dedcbecd54efea304e86369d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 09:47:03 +0200 Subject: [PATCH 1099/1669] Improved `.parseXML()` by adding `replace` argument. --- changes.txt | 1 + index.js | 14 ++++++++------ utils.js | 9 ++++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index 12c598d8a..29dbd3623 100755 --- a/changes.txt +++ b/changes.txt @@ -65,6 +65,7 @@ x updated: `String.hash(true)` returns unsigned int x updated: `CONF:default_image_converter` supports `magick` for new version of ImageMagick x updated: `CONF.default_image_consumption` can contain ZERO value which means disabled optimialization of CPU/memory consumption x updated: `CONF.default_image_consumption` has changed a default value to `0` +x updated: `U.parseXML(str, [replace])` and `String.parseXML([replace])` contains a new argument called `replace` - fixed: schemas validation - fixed: error handling in dynamic schemas diff --git a/index.js b/index.js index fb252b0dc..982b80c33 100755 --- a/index.js +++ b/index.js @@ -1255,7 +1255,7 @@ F.convert = function(value, convertor) { convertor = U.parseInt2; break; default: - return console.log('F.convert unknown convertor type:', convertor); + return console.log('Unknown convertor type:', convertor); } } @@ -1264,9 +1264,11 @@ F.convert = function(value, convertor) { return true; } - for (var i = 0, length = F.convertors.length; i < length; i++) { - if (value[F.convertors[i].name] != null) - value[F.convertors[i].name] = F.convertors[i].convertor(value[F.convertors[i].name]); + if (value) { + for (var i = 0, length = F.convertors.length; i < length; i++) { + if (value[F.convertors[i].name] != null) + value[F.convertors[i].name] = F.convertors[i].convertor(value[F.convertors[i].name]); + } } return value; @@ -5569,8 +5571,8 @@ F.onValidate = null; * @param {String} value * @return {Object} */ -F.onParseXML = function(value) { - var val = U.parseXML(value); +F.onParseXML = function(value, replace) { + var val = U.parseXML(value, replace); F._length_convertors && F.convert(val); return val; }; diff --git a/utils.js b/utils.js index 3dd066590..3e92219e8 100755 --- a/utils.js +++ b/utils.js @@ -2620,7 +2620,7 @@ exports.removeDiacritics = function(str) { * @param {String} xml * @return {Object} */ -exports.parseXML = function(xml) { +exports.parseXML = function(xml, replace) { var beg = -1; var end = 0; @@ -2664,6 +2664,9 @@ exports.parseXML = function(xml) { var path = (current.length ? current.join('.') + '.' : '') + o; var value = xml.substring(from, beg).decode(); + if (replace) + path = path.replace(/\.|_/g, '_'); + if (obj[path] === undefined) obj[path] = value; else if (obj[path] instanceof Array) @@ -3367,8 +3370,8 @@ SP.count = function(text) { return count; }; -SP.parseXML = function() { - return F.onParseXML(this); +SP.parseXML = function(replace) { + return F.onParseXML(this, replace); }; SP.parseJSON = function(date) { From ccd8223035e147727586ca06a6a2c12a172c9a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 14:34:55 +0200 Subject: [PATCH 1100/1669] Improved XML parser. --- builders.js | 8 ++++++-- utils.js | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/builders.js b/builders.js index 13ef9b4f4..b257d9f0d 100755 --- a/builders.js +++ b/builders.js @@ -4881,7 +4881,7 @@ RESTP.accept = function(ext) { return this; }; -RESTP.xml = function(data) { +RESTP.xml = function(data, replace) { if (this.$type !== 3) this.$flags = null; @@ -4890,6 +4890,10 @@ RESTP.xml = function(data) { this.$method = 'post'; this.$type = 3; + + if (replace) + this.$replace = true; + data && this.raw(data); return this; }; @@ -5068,7 +5072,7 @@ RESTP.exec = function(callback) { switch (type.toLowerCase()) { case 'text/xml': case 'application/xml': - output.value = response ? response.parseXML() : {}; + output.value = response ? response.parseXML(self.$replace ? true : false) : {}; break; case 'application/x-www-form-urlencoded': output.value = response ? F.onParseQuery(response) : {}; diff --git a/utils.js b/utils.js index 3e92219e8..c3059000b 100755 --- a/utils.js +++ b/utils.js @@ -96,6 +96,7 @@ const REG_URLEXT = /(https|http|wss|ws|file):\/\/|\/\/[a-z0-9]|[a-z]:/i; const REG_TEXTAPPLICATION = /text|application/i; const REG_DATE = /\.|-|\/|\\|:|\s/g; const REG_TIME = /am|pm/i; +const REG_XMLKEY = /\[|\]|:|\.|_/g; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -2665,7 +2666,7 @@ exports.parseXML = function(xml, replace) { var value = xml.substring(from, beg).decode(); if (replace) - path = path.replace(/\.|_/g, '_'); + path = path.replace(REG_XMLKEY, '_'); if (obj[path] === undefined) obj[path] = value; @@ -2709,7 +2710,10 @@ exports.parseXML = function(xml, replace) { attr[match[i].substring(0, index - 1)] = match[i].substring(index + 1, match[i].length - 1).decode(); } - obj[current.join('.') + (isSingle ? '.' + name : '') + '[]'] = attr; + var k = current.join('.') + (isSingle ? '.' + name : '') + '[]'; + if (replace) + k = k.replace(REG_XMLKEY, '_'); + obj[k] = attr; } return obj; From d11df7dad6ebd65c0ce4d3f78fffaee9e8f4ec50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 14:39:11 +0200 Subject: [PATCH 1101/1669] Depcreated `/workflows` and `/dependencies` files. --- index.js | 4 ++++ test/controllers/default.js | 7 ------- test/dependencies | 1 - test/test-framework-debug.js | 11 +---------- test/workflows | 8 -------- 5 files changed, 5 insertions(+), 26 deletions(-) delete mode 100644 test/dependencies delete mode 100644 test/workflows diff --git a/index.js b/index.js index 982b80c33..16aee3d23 100755 --- a/index.js +++ b/index.js @@ -8800,6 +8800,8 @@ F.$configure_dependencies = function(arr, callback) { if (!arr) return F; + OBSOLETE('/dependencies', 'File "/dependencies" are deprecated and they will be removed in v4.'); + var type; var options; var interval; @@ -8931,6 +8933,8 @@ F.$configure_workflows = function(arr, clean) { arr = null; } + OBSOLETE('/workflows', 'File "/workflows" are deprecated and they will be removed in v4.'); + if (clean) F.workflows = {}; diff --git a/test/controllers/default.js b/test/controllers/default.js index 1431f06a2..4ff5e500b 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -47,7 +47,6 @@ exports.install = function() { F.route('/sync/', synchronize); F.route('/schema-filter/', ['post', '*filter#update']); F.route('/package/', '@testpackage/test'); - F.route('/precompile/', view_precomile); F.route('/homepage/', view_homepage); F.route('/usage/', view_usage); F.route('/sse/', viewSSE_html); @@ -164,12 +163,6 @@ function plain_rest() { this.plain(this.req.method); } -function view_precomile() { - var self = this; - self.layout('precompile._layout'); - self.view('precompile.homepage'); -} - function plain_multiple() { var self = this; self.plain('POST-GET-PUT-DELETE'); diff --git a/test/dependencies b/test/dependencies deleted file mode 100644 index 6e662936e..000000000 --- a/test/dependencies +++ /dev/null @@ -1 +0,0 @@ -module (1 day) : https://www.totaljs.com/framework/include.js --> {"test":true} \ No newline at end of file diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 3bf2ec3c7..a413a7ced 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -416,15 +416,6 @@ function test_routing(next) { }); }); - async.await('precompile', function(complete) { - utils.request(url + 'precompile/', ['get'], function(error, data, code, headers) { - if (error) - throw error; - assert.ok(data.indexOf('precompile') === -1, 'framework.precompile() problem'); - complete(); - }); - }); - async.await('subshare', function(complete) { utils.request(url + 'sub/share/', ['get'], function(error, data, code, headers) { if (error) @@ -924,7 +915,7 @@ function run() { UNINSTALL('source', { uninstall: true }); UNINSTALL('view', 'precompile._layout'); - framework.uninstall('precompile', 'precompile.homepage'); + //framework.uninstall('precompile', 'precompile.homepage'); framework.clear(); setTimeout(function() { diff --git a/test/workflows b/test/workflows deleted file mode 100644 index 9e97848fe..000000000 --- a/test/workflows +++ /dev/null @@ -1,8 +0,0 @@ -// User -user-query : query (response) -user-read : read (response) -user-delete : delete (response) -user-save : workflow:'check' { hidden: true } --> save (response) -user-create : workflow:'check' --> save (response) --> workflow:'confirmation' - -save (User) : save (response) \ No newline at end of file From 23d7654b1d20f025b9f0b3ccf16682f298f0f43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 14:40:32 +0200 Subject: [PATCH 1102/1669] Fixed `deprecated` message handling. --- index.js | 6 +++--- test/test-framework-release.js | 11 +---------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/index.js b/index.js index 16aee3d23..436fd45d7 100755 --- a/index.js +++ b/index.js @@ -8797,7 +8797,7 @@ F.$configure_dependencies = function(arr, callback) { arr = null; } - if (!arr) + if (!arr || !arr.length) return F; OBSOLETE('/dependencies', 'File "/dependencies" are deprecated and they will be removed in v4.'); @@ -8933,14 +8933,14 @@ F.$configure_workflows = function(arr, clean) { arr = null; } - OBSOLETE('/workflows', 'File "/workflows" are deprecated and they will be removed in v4.'); - if (clean) F.workflows = {}; if (!arr || !arr.length) return F; + OBSOLETE('/workflows', 'File "/workflows" are deprecated and they will be removed in v4.'); + arr.forEach(function(line) { line = line.trim(); if (line.startsWith('//')) diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 60be60fbf..c0591eb5d 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -416,15 +416,6 @@ function test_routing(next) { }); }); - async.await('precompile', function(complete) { - utils.request(url + 'precompile/', ['get'], function(error, data, code, headers) { - if (error) - throw error; - assert.ok(data.indexOf('precompile') === -1, 'framework.precompile() problem'); - complete(); - }); - }); - async.await('subshare', function(complete) { utils.request(url + 'sub/share/', ['get'], function(error, data, code, headers) { if (error) @@ -924,7 +915,7 @@ function run() { UNINSTALL('source', { uninstall: true }); UNINSTALL('view', 'precompile._layout'); - framework.uninstall('precompile', 'precompile.homepage'); + //framework.uninstall('precompile', 'precompile.homepage'); framework.clear(); setTimeout(function() { From 5bac919bd8f141fbd8762651cd4e27058035876d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 22:52:55 +0200 Subject: [PATCH 1103/1669] Fixed `filter` in `F.bundle()`. --- changes.txt | 1 + index.js | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 29dbd3623..0cdf77bb8 100755 --- a/changes.txt +++ b/changes.txt @@ -82,6 +82,7 @@ x updated: `U.parseXML(str, [replace])` and `String.parseXML([replace])` contain - fixed: Schema `Boolean` parser - fixed: renamed `F.stats.request.path` to `F.stats.request.patch` - fixed: SMTP sender (a problem with auth mechanism with some mail servers) +- fixed: filter in `F.backup()` x renamed: `*.routeScript` to `*.public_js` x renamed: `*.routeStyle` to `*.public_css` diff --git a/index.js b/index.js index 436fd45d7..c2fcbd68c 100755 --- a/index.js +++ b/index.js @@ -6587,12 +6587,21 @@ F.backup = function(filename, filelist, callback, filter) { } if (stats.isDirectory()) { - var dir = item.replace(/\\/g, '/') + '/'; + var dir = item.replace(/\\/g, '/'); + + if (dir[dir.length - 1] !== '/') + dir += '/'; + if (filter && !filter(dir, true)) return next(); + U.ls(file, function(f, d) { var length = path.length; d.wait(function(item, next) { + + if (filter && !filter(item.substring(length), true)) + return next(); + writer.write(item.substring(length).padRight(padding) + ':#\n', 'utf8'); next(); }, function() { @@ -6604,6 +6613,9 @@ F.backup = function(filename, filelist, callback, filter) { return; } + if (filter && !filter(file.substring(path.length), true)) + return next(); + var data = Buffer.alloc(0); writer.write(item.padRight(padding) + ':'); Fs.createReadStream(file).pipe(Zlib.createGzip(GZIPFILE)).on('data', function(chunk) { From 9cd00540cb24e6db33172f1d6e1fba9d31000792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 22:53:05 +0200 Subject: [PATCH 1104/1669] Added `--bundle` and `--package` command. --- bin/totaljs | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/bin/totaljs b/bin/totaljs index 3cc626f48..ada48066c 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -40,6 +40,8 @@ function display_help() { console.log('-merge source target : merges first resource into the second "-merge source target"'); console.log('-clean source : cleans a resource file "-clean source"'); console.log('-minify filename : minifies .js, .css or .html file into filename.min.[extension]'); + console.log('-bundle filename : makes a bundle from the current directory'); + console.log('-package filename : makes a package from the current directory'); console.log('-install : run "totaljs -install help" to see what can be installed'); console.log('8000 : starts a server'); console.log(''); @@ -567,6 +569,16 @@ function main() { return; } + if (!$type && (cmd === '-bundle' || cmd === 'bundle')) { + makebundle(dir, process.argv[i + 1] || ''); + return; + } + + if (!$type && (cmd === '-package' || cmd === 'package')) { + makepackage(dir, process.argv[i + 1] || ''); + return; + } + if (!$type && (cmd === '-pwa' || cmd === 'pwa')) { git(dir, 'emptyproject-pwa'); return; @@ -823,4 +835,90 @@ function git(dir, type) { }); } +function makebundle(dir, filename) { + + if (!filename) + filename = 'app.bundle'; + + var blacklist = {}; + blacklist['/bundle.json'] = 1; + blacklist['/debug.js'] = 1; + blacklist['/release.js'] = 1; + blacklist['/debug.pid'] = 1; + blacklist['/package.json'] = 1; + blacklist['/readme.md'] = 1; + blacklist['/license.txt'] = 1; + blacklist['/bundles/'] = 1; + blacklist['/tmp/'] = 1; + + if (filename[0] !== '/') + blacklist['/' + filename] = 1; + else + blacklist[filename] = 1; + + blacklist['/.git/'] = 1; + + if (filename.toLowerCase().lastIndexOf('.bundle') === -1) + filename += '.bundle'; + + console.log(''); + console.log('--- CREATE BUNDLE PACKAGE --'); + console.log(''); + console.log('Directory :', dir); + console.log('Filename :', filename); + + F.backup(filename, dir, function(err, path) { + + if (err) + throw err; + + console.log('Success :', path.files.pluralize('# files', '# file', '# files', '# files') + ' (' + path.size.filesize() + ')'); + console.log(''); + + }, path => blacklist[path] == null); +} + +function makepackage(dir, filename) { + + if (!filename) + filename = 'noname.package'; + + var blacklist = {}; + blacklist['/bundle.json'] = 1; + blacklist['/debug.js'] = 1; + blacklist['/release.js'] = 1; + blacklist['/debug.pid'] = 1; + blacklist['/package.json'] = 1; + blacklist['/readme.md'] = 1; + blacklist['/license.txt'] = 1; + blacklist['/bundles/'] = 1; + blacklist['/tmp/'] = 1; + + if (filename[0] !== '/') + blacklist['/' + filename] = 1; + else + blacklist[filename] = 1; + + blacklist['/.git/'] = 1; + + if (filename.toLowerCase().lastIndexOf('.package') === -1) + filename += '.package'; + + console.log(''); + console.log('--- CREATE PACKAGE --'); + console.log(''); + console.log('Directory :', dir); + console.log('Filename :', filename); + + F.backup(filename, dir, function(err, path) { + + if (err) + throw err; + + console.log('Success :', path.files.pluralize('# files', '# file', '# files', '# files') + ' (' + path.size.filesize() + ')'); + console.log(''); + + }, path => blacklist[path] == null); +} + main(); \ No newline at end of file From c02bd1ad225b39a8990af6d234238976e1cfeb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Jun 2019 23:04:20 +0200 Subject: [PATCH 1105/1669] Updated header. --- bin/totaljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/totaljs b/bin/totaljs index ada48066c..317e25480 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -335,7 +335,7 @@ function main() { console.log(''); console.log('|==================================================|'); - console.log('| Total.js - Web framework for Node.js |'); + console.log('| Total.js - www.totaljs.com |'); console.log('| Version: v' + require('total.js').version_header.padRight(39) + '|'); console.log('|==================================================|'); console.log(''); From c19b9f097878b30ce9f0f8f76e1dae2a10efb8a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 29 Jun 2019 20:38:59 +0200 Subject: [PATCH 1106/1669] Improved workers. --- changes.txt | 1 + index.js | 33 ++++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index 0cdf77bb8..fabb45f77 100755 --- a/changes.txt +++ b/changes.txt @@ -50,6 +50,7 @@ x added: __obsolete__ message to older declaration of middleware, schemas and op x added: `U.diffarr(prop, arr_A, arr_B)` for comparing of two arrays x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` x added: `global.REQUIRE()` for local modules within app directory +x added: `global.isWORKER` variable which contains `true` (when the process is executed as a worker) or `false` x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index c2fcbd68c..076d204a7 100755 --- a/index.js +++ b/index.js @@ -107,7 +107,7 @@ Object.freeze(EMPTYREQUEST); global.EMPTYOBJECT = EMPTYOBJECT; global.EMPTYARRAY = EMPTYARRAY; global.NOW = new Date(); - +global.isWORKER = false; global.REQUIRE = function(path) { return require('./' + path); }; @@ -1160,6 +1160,9 @@ global.ON = F.on = function(name, fn) { } } + if (isWORKER && name === 'service' && !F.cache.interval) + F.cache.init_timer(); + if (F.$events[name]) F.$events[name].push(fn); else @@ -6783,6 +6786,7 @@ global.LOAD = F.load = function(debug, types, pwd) { F.isWorker = true; F.isDebug = debug; + global.isWORKER = true; global.DEBUG = debug; global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; @@ -6804,7 +6808,7 @@ global.LOAD = F.load = function(debug, types, pwd) { F.$configure_sitemap(); F.consoledebug('init'); - F.cache.init(); + F.cache.init(true); EMIT('init'); F.$load(types, directory, function() { @@ -6830,10 +6834,11 @@ global.LOAD = F.load = function(debug, types, pwd) { return F; } - setTimeout(function() { - if (!F.isTest) - delete F.test; - }, 5000); + // Because this is worker + // setTimeout(function() { + // if (!F.isTest) + // delete F.test; + // }, 5000); }, 500); @@ -10277,14 +10282,24 @@ function FrameworkCache() { const FrameworkCacheProto = FrameworkCache.prototype; -FrameworkCacheProto.init = function() { +FrameworkCacheProto.init = function(notimer) { var self = this; - clearInterval(self.interval); - self.interval = setInterval(() => F.cache.recycle(), 1000 * 60); + + if (!notimer) + self.init_timer(); + if (CONF.allow_cache_snapshot) self.load(() => self.loadpersistent()); else self.loadpersistent(); + + return self; +}; + +FrameworkCacheProto.init_timer = function() { + var self = this; + self.interval && clearInterval(self.interval); + self.interval = setInterval(() => F.cache.recycle(), 1000 * 60); return self; }; From f78ef490461cfbe00ed2027d148e9e6505fdacdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 29 Jun 2019 20:53:06 +0200 Subject: [PATCH 1107/1669] Improved `LOAD()` by adding `callback` function. --- changes.txt | 1 + index.js | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index fabb45f77..68c1e300e 100755 --- a/changes.txt +++ b/changes.txt @@ -67,6 +67,7 @@ x updated: `CONF:default_image_converter` supports `magick` for new version of I x updated: `CONF.default_image_consumption` can contain ZERO value which means disabled optimialization of CPU/memory consumption x updated: `CONF.default_image_consumption` has changed a default value to `0` x updated: `U.parseXML(str, [replace])` and `String.parseXML([replace])` contains a new argument called `replace` +x updated: `LOAD()` added a callback function - fixed: schemas validation - fixed: error handling in dynamic schemas diff --git a/index.js b/index.js index 076d204a7..84690f78f 100755 --- a/index.js +++ b/index.js @@ -6741,7 +6741,17 @@ F.response503 = function(req, res) { return F; }; -global.LOAD = F.load = function(debug, types, pwd) { +global.LOAD = F.load = function(debug, types, pwd, ready) { + + if (typeof(types) === 'function') { + ready = types; + types = null; + } + + if (typeof(pwd) === 'function') { + ready = pwd; + pwd = null; + } if (pwd && pwd[0] === '.' && pwd.length < 4) F.directory = directory = U.$normalize(Path.normalize(directory + '/..')); @@ -6819,12 +6829,14 @@ global.LOAD = F.load = function(debug, types, pwd) { setTimeout(function() { try { - EMIT('load', F); - EMIT('ready', F); + EMIT('load'); + EMIT('ready'); } catch (err) { - F.error(err, 'F.on("load/ready")'); + F.error(err, 'ON("load/ready")'); } + ready && ready(); + F.removeAllListeners('load'); F.removeAllListeners('ready'); @@ -7006,10 +7018,10 @@ F.initialize = function(http, debug, options) { setTimeout(function() { try { - EMIT('load', F); - EMIT('ready', F); + EMIT('load'); + EMIT('ready'); } catch (err) { - F.error(err, 'F.on("load/ready")'); + F.error(err, 'ON("load/ready")'); } F.removeAllListeners('load'); From 3dac70f7a4ec044df96f71f439f569761e32a85c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 29 Jun 2019 21:43:54 +0200 Subject: [PATCH 1108/1669] Improved `LOAD()`, by default doesn't load bundles, packages or themes --- index.js | 77 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/index.js b/index.js index 84690f78f..e721e94c2 100755 --- a/index.js +++ b/index.js @@ -3811,8 +3811,26 @@ F.$load = function(types, targetdirectory, callback, packageName) { var dependencies = []; var operations = []; var isPackage = targetdirectory.indexOf('.package') !== -1; + var isNo = true; - if (!types || types.indexOf('modules') !== -1) { + if (types) { + for (var i = 0; i < types.length; i++) { + if (types[i].substring(0, 2) !== 'no') { + isNo = false; + break; + } + } + } + + var can = function(type) { + if (!types) + return true; + if (types.indexOf('no' + type) !== -1) + return false; + return isNo ? true : types.indexOf(type) !== -1; + }; + + if (can('modules')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/modules/' : CONF.directory_modules); arr = []; @@ -3822,7 +3840,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('isomorphic') !== -1) { + if (can('isomorphic')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/isomorphic/' : CONF.directory_isomorphic); arr = []; @@ -3832,7 +3850,8 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('packages') !== -1) { + + if (can('packages')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/packages/' : CONF.directory_packages); arr = []; @@ -3879,7 +3898,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('models') !== -1) { + if (can('models')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/models/' : CONF.directory_models); arr = []; @@ -3889,7 +3908,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('schemas') !== -1) { + if (can('schemas')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/schemas/' : CONF.directory_schemas); arr = []; @@ -3899,7 +3918,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('tasks') !== -1) { + if (can('tasks')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/tasks/' : CONF.directory_tasks); arr = []; @@ -3909,7 +3928,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('operations') !== -1) { + if (can('operations')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/operations/' : CONF.directory_operations); arr = []; @@ -3919,7 +3938,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('themes') !== -1) { + if (can('themes')) { operations.push(function(resume) { arr = []; dir = U.combine(targetdirectory, isPackage ? '/themes/' : CONF.directory_themes); @@ -3936,7 +3955,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('definitions') !== -1) { + if (can('definitions')) { operations.push(function(resume) { dir = U.combine(targetdirectory, isPackage ? '/definitions/' : CONF.directory_definitions); arr = []; @@ -3946,7 +3965,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('controllers') !== -1) { + if (can('controllers')) { operations.push(function(resume) { arr = []; dir = U.combine(targetdirectory, isPackage ? '/controllers/' : CONF.directory_controllers); @@ -3956,7 +3975,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('components') !== -1) { + if (can('components')) { operations.push(function(resume) { arr = []; dir = U.combine(targetdirectory, isPackage ? '/components/' : CONF.directory_components); @@ -3966,7 +3985,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { }); } - if (!types || types.indexOf('preferences') !== -1) { + if (can('preferences')) { operations.push(function(resume) { if (F.onPrefLoad) { F.onPrefLoad(function(value) { @@ -4037,6 +4056,8 @@ global.UPTODATE = F.uptodate = function(type, url, options, interval, callback, options = null; } + OBSOLETE('UPTODATE()', 'This method is deprecated and it will be removed in v4.'); + var obj = { type: type, name: '', url: url, interval: interval, options: options, count: 0, updated: NOW, errors: [], callback: callback }; if (!F.uptodates) @@ -6753,12 +6774,17 @@ global.LOAD = F.load = function(debug, types, pwd, ready) { pwd = null; } + if (!types) + types = ['nobundles', 'nopackages', 'nocomponents', 'nothemes']; + if (pwd && pwd[0] === '.' && pwd.length < 4) F.directory = directory = U.$normalize(Path.normalize(directory + '/..')); else if (pwd) F.directory = directory = U.$normalize(pwd); else if (process.env.istotaljsworker) F.directory = process.cwd(); + else if ((/\/scripts\/.*?.js/).test(process.argv[1])) + F.directory = directory = U.$normalize(Path.normalize(directory + '/..')); if (typeof(debug) === 'string') { switch (debug.toLowerCase().replace(/\.|\s/g, '-')) { @@ -6801,6 +6827,25 @@ global.LOAD = F.load = function(debug, types, pwd, ready) { global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; + var isNo = true; + + if (types) { + for (var i = 0; i < types.length; i++) { + if (types[i].substring(0, 2) !== 'no') { + isNo = false; + break; + } + } + } + + var can = function(type) { + if (!types) + return true; + if (types.indexOf('no' + type) !== -1) + return false; + return isNo ? true : types.indexOf(type) !== -1; + }; + F.$bundle(function() { F.consoledebug('startup'); F.$startup(function() { @@ -6808,13 +6853,13 @@ global.LOAD = F.load = function(debug, types, pwd, ready) { F.consoledebug('startup (done)'); F.$configure_configs(); - if (!types || types.indexOf('versions') !== -1) + if (can('versions')) F.$configure_versions(); - if (!types || types.indexOf('workflows') !== -1) + if (can('workflows')) F.$configure_workflows(); - if (!types || types.indexOf('sitemap') !== -1) + if (can('sitemap')) F.$configure_sitemap(); F.consoledebug('init'); @@ -6860,7 +6905,7 @@ global.LOAD = F.load = function(debug, types, pwd, ready) { } }); }); - }, !types || types.indexOf('bundles') !== -1); + }, can('bundles')); return F; }; From fbf582ec9e9976288ee34ca0c32f451e4ca9ab67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Jun 2019 19:58:51 +0200 Subject: [PATCH 1109/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de6d5a6bb..610ff106c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.3.0-33", + "version": "3.3.0-34", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 174770c8c42551e826ec65bcd57ebb81e9f1d448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Jul 2019 09:40:10 +0200 Subject: [PATCH 1110/1669] Improved console output. --- bin/totaljs | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index 317e25480..e8d4c05cf 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -861,7 +861,6 @@ function makebundle(dir, filename) { if (filename.toLowerCase().lastIndexOf('.bundle') === -1) filename += '.bundle'; - console.log(''); console.log('--- CREATE BUNDLE PACKAGE --'); console.log(''); console.log('Directory :', dir); @@ -904,7 +903,6 @@ function makepackage(dir, filename) { if (filename.toLowerCase().lastIndexOf('.package') === -1) filename += '.package'; - console.log(''); console.log('--- CREATE PACKAGE --'); console.log(''); console.log('Directory :', dir); From e728de5d4ca6e92468a282e75d56ffccfd45ff5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Jul 2019 09:40:27 +0200 Subject: [PATCH 1111/1669] Updated `GROUP()`, now it can contain a flag as a string. --- index.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index e721e94c2..969fd5aa3 100755 --- a/index.js +++ b/index.js @@ -1920,9 +1920,14 @@ global.GROUP = F.group = function() { switch (typeof(o)) { case 'string': - if (o.endsWith('/')) - o = o.substring(0, o.length - 1); - _prefix = o; + if (o.indexOf('/') === -1) { + // flags + _flags = o.split(',').trim(); + } else { + if (o.endsWith('/')) + o = o.substring(0, o.length - 1); + _prefix = o; + } break; case 'function': fn = o; From 8ad86e29187ac021e76ffa976c824f74b08ec68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Jul 2019 09:44:27 +0200 Subject: [PATCH 1112/1669] Updated unit-test. --- test/controllers/default.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/controllers/default.js b/test/controllers/default.js index 4ff5e500b..9674242d4 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -32,7 +32,7 @@ exports.install = function() { }); }); - GROUP('prefix2', ['get'], function() { + GROUP('/prefix2/', ['get'], function() { ROUTE('/test/', function() { this.plain('PREFIX2TEST'); }); From 5f49cc773473684824c1a5510732e742b1bf40d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Jul 2019 19:44:09 +0200 Subject: [PATCH 1113/1669] Added `ACTION()` method. --- changes.txt | 1 + index.js | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 68c1e300e..e6ad0b76e 100755 --- a/changes.txt +++ b/changes.txt @@ -51,6 +51,7 @@ x added: `U.diffarr(prop, arr_A, arr_B)` for comparing of two arrays x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` x added: `global.REQUIRE()` for local modules within app directory x added: `global.isWORKER` variable which contains `true` (when the process is executed as a worker) or `false` +- added: `ACTION(url, [data], callback)` can evaluate a route without request creation x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index 969fd5aa3..9ed6492c1 100755 --- a/index.js +++ b/index.js @@ -9845,6 +9845,55 @@ F.lookup = function(req, url, flags, membertype) { return null; }; +F.lookupaction = function(req, url, authorized) { + + var isSystem = url[0] === '#'; + if (isSystem) + return F.routes.system[url]; + + var length = F.routes.web.length; + for (var i = 0; i < length; i++) { + + var route = F.routes.web[i]; + if (route.method !== req.method) + continue; + + if (route.CUSTOM) { + if (!route.CUSTOM(url, req)) + continue; + } else { + if (route.isWILDCARD) { + if (!framework_internal.routeCompare(req.path, route.url, isSystem, true)) + continue; + } else { + if (!framework_internal.routeCompare(req.path, route.url, isSystem)) + continue; + } + } + + if (isSystem) { + if (route.isSYSTEM) + return route; + continue; + } + + if (route.isPARAM && route.regexp) { + var skip = false; + for (var j = 0, l = route.regexpIndexer.length; j < l; j++) { + var p = req.path[route.regexpIndexer[j]]; + if (p === undefined || !route.regexp[route.regexpIndexer[j]].test(p)) { + skip = true; + break; + } + } + if (skip) + continue; + } + return route; + } +}; + + F.lookup_websocket = function(req, url, membertype) { var subdomain = F._length_subdomain_websocket && req.subdomain ? req.subdomain.join('.') : null; @@ -10706,10 +10755,18 @@ Controller.prototype = { return this.req.query; }, + set query(val) { + this.req.query = val; + }, + get body() { return this.req.body; }, + set body(val) { + this.req.body = val; + }, + get files() { return this.req.files; }, @@ -10726,6 +10783,10 @@ Controller.prototype = { return this.req.xhr; }, + set xhr(val) { + this.req.xhr = val; + }, + get url() { return U.path(this.req.uri.pathname); }, @@ -10749,6 +10810,7 @@ Controller.prototype = { }, get controllers() { + OBSOLETE('controller.controllers', 'This property will be removed in v4.'); return F.controllers; }, @@ -10790,6 +10852,10 @@ Controller.prototype = { return this.req.mobile; }, + set mobile(val) { + this.req.mobile = val; + }, + get robot() { return this.req.robot; }, @@ -10798,6 +10864,10 @@ Controller.prototype = { return this.req.sessionid; }, + set sessionid(val) { + this.req.sessionid = val; + }, + get viewname() { var name = this.req.path[this.req.path.length - 1]; return !name || name === '/' ? 'index' : name; @@ -12887,6 +12957,12 @@ ControllerProto.json = function(obj, headers, beautify, replacer) { return self; } + if (self.$evalroutecallback) { + var err = obj instanceof framework_builders.ErrorBuilder ? obj : null; + self.$evalroutecallback(err, err ? null : obj); + return self; + } + if (obj instanceof framework_builders.ErrorBuilder) { self.req.$language && !obj.isResourceCustom && obj.setResource(self.req.$language); @@ -13058,6 +13134,12 @@ ControllerProto.content = function(body, type, headers) { res.options.headers = headers; res.options.code = self.status || 200; + if (self.$evalroutecallback) { + var err = body instanceof ErrorBuilder ? body : null; + self.$evalroutecallback(err, err ? null : body); + return self; + } + if (body instanceof ErrorBuilder) { if (self.language && !body.resourceName) @@ -17059,6 +17141,11 @@ function extend_response(PROTO) { if (res.headersSent) return res; + if (self.$evalroutecallback) { + res.headersSent = true; + self.$evalroutecallback(null, options.body, res.options.encoding || ENCODING); + return self; + } var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); @@ -18093,6 +18180,89 @@ F.ilogger = function(name, req, ts) { } }; +function evalroutehandleraction(controller) { + if (controller.route.isPARAM) + controller.route.execute.apply(controller, framework_internal.routeParam(controller.req.split, controller.route)); + else + controller.route.execute.call(controller); +} + +function evalroutehandler(controller) { + if (!controller.route.schema || !controller.route.schema[1] || controller.req.method === 'DELETE' || controller.req.method === 'GET') + return evalroutehandleraction(controller); + + F.onSchema(controller.req, controller.route, function(err, body) { + if (err) { + controller.$evalroutecallback(err, body); + } else { + controller.body = body; + evalroutehandleraction(controller); + } + }); +} + +global.ACTION = function(url, data, callback) { + + if (typeof(data) === 'function') { + callback = data; + data = null; + } + + var index = url.indexOf(' '); + var method = url.substring(0, index); + var params = ''; + var route; + + url = url.substring(index + 1); + index = url.indexOf('?'); + + if (index !== -1) { + params = url.substring(index + 1); + url = url.substring(0, index); + } + + url = url.trim(); + var routeurl = url; + + if (routeurl.endsWith('/')) + routeurl = routeurl.substring(0, routeurl.length - 1); + + var req = {}; + var res = {}; + + req.res = res; + req.$protocol = 'http'; + req.url = url; + req.ip = F.ip || '127.0.0.1'; + req.host = req.ip + ':' + (F.port || 8000); + req.headers = { 'user-agent': 'Total.js/v' + F.version_header }; + req.uri = framework_internal.parseURI(req); + req.path = framework_internal.routeSplit(req.uri.pathname); + req.body = data || {}; + req.query = params ? F.onParseQuery(params) : {}; + req.files = EMPTYARRAY; + req.method = method; + res.options = req.options = {}; + + var route = F.lookupaction(req, url); + if (!route) + return; + + if (route.isPARAM) + req.split = framework_internal.routeSplit(req.uri.pathname, true); + else + req.split = EMPTYARRAY; + + var controller = new Controller(route.controller, null, null, route.currentViewDirectory); + controller.route = route; + controller.req = req; + controller.res = res; + + controller.$evalroutecallback = callback || NOOP; + setImmediate(evalroutehandler, controller); + return controller; +}; + // Because of controller prototypes // It's used in VIEW() and VIEWCOMPILE() const EMPTYCONTROLLER = new Controller('', null, null, ''); @@ -18103,4 +18273,4 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; +global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file From 03cf7813785ec795c7e6f72fd1c766fa864682b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Jul 2019 19:55:05 +0200 Subject: [PATCH 1114/1669] Fixed routing. --- index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 9ed6492c1..a6bc4fa77 100755 --- a/index.js +++ b/index.js @@ -17141,10 +17141,10 @@ function extend_response(PROTO) { if (res.headersSent) return res; - if (self.$evalroutecallback) { + if (res.$evalroutecallback) { res.headersSent = true; - self.$evalroutecallback(null, options.body, res.options.encoding || ENCODING); - return self; + res.$evalroutecallback(null, options.body, res.options.encoding || ENCODING); + return res; } var accept = req.headers['accept-encoding'] || ''; @@ -18258,7 +18258,7 @@ global.ACTION = function(url, data, callback) { controller.req = req; controller.res = res; - controller.$evalroutecallback = callback || NOOP; + res.$evalroutecallback = controller.$evalroutecallback = callback || NOOP; setImmediate(evalroutehandler, controller); return controller; }; From a90ff67ce8adda8e2dcf237286f560b4990fb9d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Jul 2019 07:19:03 +0200 Subject: [PATCH 1115/1669] Updated changelog. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index e6ad0b76e..992f6c9c7 100755 --- a/changes.txt +++ b/changes.txt @@ -51,7 +51,7 @@ x added: `U.diffarr(prop, arr_A, arr_B)` for comparing of two arrays x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` x added: `global.REQUIRE()` for local modules within app directory x added: `global.isWORKER` variable which contains `true` (when the process is executed as a worker) or `false` -- added: `ACTION(url, [data], callback)` can evaluate a route without request creation +x added: `ACTION(url, [data], callback)` can evaluate a route without request creation x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS From 2fa55aa081ed0743124af1a575d228f229ae6f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Jul 2019 14:01:45 +0200 Subject: [PATCH 1116/1669] Fixed `self`. --- session.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/session.js b/session.js index a5494f802..6f748bb7f 100644 --- a/session.js +++ b/session.js @@ -34,7 +34,7 @@ function Session(name, ondata) { if (m.expire > NOW) storage.push(encodeURIComponent(m.sessionid) + ';' + (m.id ? encodeURIComponent(m.id) : '') + ';' + m.expire.getTime() + ';' + (m.used ? m.used.getTime() : '') + ';' + (m.created ? m.created.getTime() : '') + ';' + (m.note ? encodeURIComponent(m.note) : '') + ';' + (m.settings ? encodeURIComponent(m.settings) : '')); else { - self.onremove && self.onremove(m); + t.onremove && t.onremove(m); t.items.delete(m.sessionid); } } From afd1944915c667de65ee16376cd97ce392f0cf7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Jul 2019 16:07:28 +0200 Subject: [PATCH 1117/1669] Fixed `REQUIRE()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index a6bc4fa77..0c10f6125 100755 --- a/index.js +++ b/index.js @@ -109,7 +109,7 @@ global.EMPTYARRAY = EMPTYARRAY; global.NOW = new Date(); global.isWORKER = false; global.REQUIRE = function(path) { - return require('./' + path); + return require(F.directory + '/' + path); }; var DEF = global.DEF = {}; From d32588e773b3c2e720a5be55312033e6eda4746d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Jul 2019 19:49:31 +0200 Subject: [PATCH 1118/1669] Improved `ROUTE()`. --- changes.txt | 2 ++ index.js | 18 ++++++++---------- package.json | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/changes.txt b/changes.txt index 992f6c9c7..a2951f1e6 100755 --- a/changes.txt +++ b/changes.txt @@ -52,6 +52,8 @@ x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` x added: `global.REQUIRE()` for local modules within app directory x added: `global.isWORKER` variable which contains `true` (when the process is executed as a worker) or `false` x added: `ACTION(url, [data], callback)` can evaluate a route without request creation +- added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` flag +- added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `unauthorized` flag x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index 0c10f6125..8a2fa14eb 100755 --- a/index.js +++ b/index.js @@ -1941,16 +1941,6 @@ global.GROUP = F.group = function() { return F; }; -/** - * Add a route - * @param {String} url - * @param {Function} funcExecute Action. - * @param {String Array} flags - * @param {Number} length Maximum length of request data. - * @param {String Array} middleware Loads custom middleware. - * @param {Number} timeout Response timeout. - * @return {Framework} - */ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, language) { var name; @@ -1986,6 +1976,14 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.replace(/\t/g, ' '); + var first = url.substring(0, 1); + if (first === '+' || first === '-') { + // auth/unauth + url = url.replace(/^(\+|-)+/g, '').trim(); + !flags && (flags = []); + flags.push(first === '+' ? 'authorized' : 'unauthorized'); + } + url = url.replace(/(^|\s?)\*([{}a-z0-9}]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); diff --git a/package.json b/package.json index 610ff106c..aaf79a085 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.3.0-34", + "version": "3.3.0-35", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 29de8f8538709efd1038845dc22e66bb65322777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Jul 2019 19:55:34 +0200 Subject: [PATCH 1119/1669] Improved `ROUTE()`. --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 8a2fa14eb..acd7d72a2 100755 --- a/index.js +++ b/index.js @@ -1977,11 +1977,11 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.replace(/\t/g, ' '); var first = url.substring(0, 1); - if (first === '+' || first === '-') { + if (first === '+' || first === '-' || url.substring(0, 2) === '🔒') { // auth/unauth - url = url.replace(/^(\+|-)+/g, '').trim(); + url = url.replace(/^(\+|-|🔒)+/g, '').trim(); !flags && (flags = []); - flags.push(first === '+' ? 'authorized' : 'unauthorized'); + flags.push(first === '-' ? 'unauthorized' : 'authorized'); } url = url.replace(/(^|\s?)\*([{}a-z0-9}]|\s).*?$/i, function(text) { From 268974437a4b4a948b72f5c579d0fd806bf540b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Jul 2019 19:56:14 +0200 Subject: [PATCH 1120/1669] Updated changelog. --- changes.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index a2951f1e6..e0a5ddbbc 100755 --- a/changes.txt +++ b/changes.txt @@ -52,8 +52,9 @@ x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` x added: `global.REQUIRE()` for local modules within app directory x added: `global.isWORKER` variable which contains `true` (when the process is executed as a worker) or `false` x added: `ACTION(url, [data], callback)` can evaluate a route without request creation +- added: `ROUTE('🔒 METHOD URL')`, it means that 🔒 adds `authorized` flag - added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` flag -- added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `unauthorized` flag +- added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` flag x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS From 98fa1ea25e8711d5f8acc410e2d20f1fa966dc17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Jul 2019 21:27:20 +0200 Subject: [PATCH 1121/1669] Fixed paths in `F.backup()`. --- changes.txt | 1 + index.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/changes.txt b/changes.txt index e0a5ddbbc..ca7a15f0d 100755 --- a/changes.txt +++ b/changes.txt @@ -89,6 +89,7 @@ x updated: `LOAD()` added a callback function - fixed: renamed `F.stats.request.path` to `F.stats.request.patch` - fixed: SMTP sender (a problem with auth mechanism with some mail servers) - fixed: filter in `F.backup()` +- fixed: paths for stored directories in `F.backup()` x renamed: `*.routeScript` to `*.public_js` x renamed: `*.routeStyle` to `*.public_css` diff --git a/index.js b/index.js index acd7d72a2..b8855f2e4 100755 --- a/index.js +++ b/index.js @@ -6623,7 +6623,11 @@ F.backup = function(filename, filelist, callback, filter) { return next(); U.ls(file, function(f, d) { + var length = path.length; + if (path[path.length - 1] === '/') + length--; + d.wait(function(item, next) { if (filter && !filter(item.substring(length), true)) From 6f894b4774a60eb0c1f63e791c5ca9f0c6982e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 10 Jul 2019 12:06:19 +0200 Subject: [PATCH 1122/1669] Updated changelog. --- changes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index e0a5ddbbc..aca5fdb9d 100755 --- a/changes.txt +++ b/changes.txt @@ -52,9 +52,9 @@ x added: `DIFFARR(prop, arr_A, arr_B)` is a global alias for `U.diffarr()` x added: `global.REQUIRE()` for local modules within app directory x added: `global.isWORKER` variable which contains `true` (when the process is executed as a worker) or `false` x added: `ACTION(url, [data], callback)` can evaluate a route without request creation -- added: `ROUTE('🔒 METHOD URL')`, it means that 🔒 adds `authorized` flag -- added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` flag -- added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` flag +x added: `ROUTE('🔒 METHOD URL')`, it means that 🔒 adds `authorized` flag +x added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` flag +x added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` flag x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS From acf629d51aee07a8d43c084e6a3d600248176ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 10:59:36 +0200 Subject: [PATCH 1123/1669] Added schema query argument `filters`. --- builders.js | 85 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/builders.js b/builders.js index b257d9f0d..7b7142db9 100755 --- a/builders.js +++ b/builders.js @@ -133,6 +133,13 @@ TaskBuilder.prototype = { get headers() { return this.controller && this.controller.req ? this.controller.req.headers : null; + }, + + get filter() { + var ctrl = this.controller; + if (ctrl && !ctrl.$filter) + ctrl.$filter = ctrl.$filterschema ? CONVERT(ctrl.query, ctrl.$filterschema) : ctrl.query; + return ctrl ? ctrl.$filter : EMPTYOBJECT; } }; @@ -194,6 +201,13 @@ SchemaOptions.prototype = { get headers() { return this.controller && this.controller.req ? this.controller.req.headers : null; + }, + + get filter() { + var ctrl = this.controller; + if (ctrl && !ctrl.$filter) + ctrl.$filter = ctrl.$filterschema ? CONVERT(ctrl.query, ctrl.$filterschema) : ctrl.query; + return ctrl ? ctrl.$filter : EMPTYOBJECT; } }; @@ -1040,10 +1054,11 @@ SchemaBuilderEntityProto.setPrepare = function(fn) { * @param {Function(error, model, helper, next(value), controller)} fn * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setSave = function(fn, description) { +SchemaBuilderEntityProto.setSave = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onSave = fn; this.meta.save = description || null; + this.meta.savefilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setSave()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1053,10 +1068,11 @@ SchemaBuilderEntityProto.setSave = function(fn, description) { * @param {Function(error, model, helper, next(value), controller)} fn * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setInsert = function(fn, description) { +SchemaBuilderEntityProto.setInsert = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onInsert = fn; this.meta.insert = description || null; + this.meta.insertfilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setInsert()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1066,10 +1082,11 @@ SchemaBuilderEntityProto.setInsert = function(fn, description) { * @param {Function(error, model, helper, next(value), controller)} fn * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setUpdate = function(fn, description) { +SchemaBuilderEntityProto.setUpdate = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onUpdate = fn; this.meta.update = description || null; + this.meta.updatefilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setUpdate()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1079,10 +1096,11 @@ SchemaBuilderEntityProto.setUpdate = function(fn, description) { * @param {Function(error, model, helper, next(value), controller)} fn * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setPatch = function(fn, description) { +SchemaBuilderEntityProto.setPatch = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onPatch = fn; this.meta.patch = description || null; + this.meta.patchfilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setPatch()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1102,10 +1120,11 @@ SchemaBuilderEntityProto.setError = function(fn) { * @param {Function(error, model, helper, next(value), controller)} fn * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setGet = SchemaBuilderEntityProto.setRead = function(fn, description) { +SchemaBuilderEntityProto.setGet = SchemaBuilderEntityProto.setRead = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onGet = fn; this.meta.get = this.meta.read = description || null; + this.meta.getfilter = this.meta.readfilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setGet()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1116,10 +1135,11 @@ SchemaBuilderEntityProto.setGet = SchemaBuilderEntityProto.setRead = function(fn * @param {String} description Optional. * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setQuery = function(fn, description) { +SchemaBuilderEntityProto.setQuery = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onQuery = fn; this.meta.query = description || null; + this.meta.queryfilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setQuery()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1130,10 +1150,11 @@ SchemaBuilderEntityProto.setQuery = function(fn, description) { * @param {String} description Optional. * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.setRemove = function(fn, description) { +SchemaBuilderEntityProto.setRemove = function(fn, description, filter) { fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onRemove = fn; this.meta.remove = description || null; + this.meta.removefilter = filter; !fn.$newversion && OBSOLETE('Schema("{0}").setRemove()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1165,7 +1186,7 @@ SchemaBuilderEntityProto.constant = function(name, value, description) { * @param {String} description Optional. * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.addTransform = function(name, fn, description) { +SchemaBuilderEntityProto.addTransform = function(name, fn, description, filter) { if (typeof(name) === 'function') { fn = name; @@ -1176,6 +1197,7 @@ SchemaBuilderEntityProto.addTransform = function(name, fn, description) { !this.transforms && (this.transforms = {}); this.transforms[name] = fn; this.meta['transform#' + name] = description || null; + this.meta['transformfilter#' + name] = filter; !fn.$newversion && OBSOLETE('Schema("{0}").addTransform("{1}")'.format(this.name, name), MSG_OBSOLETE_NEW); return this; }; @@ -1187,7 +1209,7 @@ SchemaBuilderEntityProto.addTransform = function(name, fn, description) { * @param {String} description Optional. * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.addOperation = function(name, fn, description) { +SchemaBuilderEntityProto.addOperation = function(name, fn, description, filter) { if (typeof(name) === 'function') { fn = name; @@ -1198,6 +1220,7 @@ SchemaBuilderEntityProto.addOperation = function(name, fn, description) { !this.operations && (this.operations = {}); this.operations[name] = fn; this.meta['operation#' + name] = description || null; + this.meta['operationfilter#' + name] = filter; !fn.$newversion && OBSOLETE('Schema("{0}").addOperation("{1}")'.format(this.name, name), MSG_OBSOLETE_NEW); return this; }; @@ -1209,7 +1232,7 @@ SchemaBuilderEntityProto.addOperation = function(name, fn, description) { * @param {String} description Optional. * @return {SchemaBuilderEntity} */ -SchemaBuilderEntityProto.addWorkflow = function(name, fn, description) { +SchemaBuilderEntityProto.addWorkflow = function(name, fn, description, filter) { if (typeof(name) === 'function') { fn = name; @@ -1220,11 +1243,12 @@ SchemaBuilderEntityProto.addWorkflow = function(name, fn, description) { !this.workflows && (this.workflows = {}); this.workflows[name] = fn; this.meta['workflow#' + name] = description || null; + this.meta['workflowfilter#' + name] = filter; !fn.$newversion && OBSOLETE('Schema("{0}").addWorkflow("{1}")'.format(this.name, name), MSG_OBSOLETE_NEW); return this; }; -SchemaBuilderEntityProto.addHook = function(name, fn, description) { +SchemaBuilderEntityProto.addHook = function(name, fn, description, filter) { if (!this.hooks) this.hooks = {}; @@ -1233,6 +1257,7 @@ SchemaBuilderEntityProto.addHook = function(name, fn, description) { !this.hooks[name] && (this.hooks[name] = []); this.hooks[name].push({ owner: F.$owner(), fn: fn }); this.meta['hook#' + name] = description || null; + this.meta['hookfilter#' + name] = filter; !fn.$newversion && OBSOLETE('Schema("{0}").addHook("{1}")'.format(this.name, name), MSG_OBSOLETE_NEW); return this; }; @@ -1474,6 +1499,11 @@ SchemaBuilderEntityProto.get = SchemaBuilderEntityProto.read = function(options, if (controller instanceof SchemaOptions || controller instanceof OperationOptions) controller = controller.controller; + if (self.meta.getfilter && controller) { + controller.$filterschema = self.meta.getfilter; + controller.$filter = null; + } + if (CONF.logger) $now = Date.now(); @@ -1559,6 +1589,11 @@ SchemaBuilderEntityProto.remove = function(options, callback, controller) { if (controller instanceof SchemaOptions || controller instanceof OperationOptions) controller = controller.controller; + if (self.meta.removefilter && controller) { + controller.$filterschema = self.meta.removefilter; + controller.$filter = null; + } + if (CONF.logger) $now = Date.now(); @@ -1641,6 +1676,11 @@ SchemaBuilderEntityProto.query = function(options, callback, controller) { var $type = 'query'; var $now; + if (self.meta.queryfilter && controller) { + controller.$filterschema = self.meta.queryfilter; + controller.$filter = null; + } + self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2614,6 +2654,12 @@ SchemaBuilderEntityProto.$execute = function(type, name, model, options, callbac if (model && !controller && model.$$controller) controller = model.$$controller; + var opfilter = [type + 'filter#' + name]; + if (opfilter && controller) { + controller.$filterschema = opfilter; + controller.$filter = null; + } + if (skip === true || model instanceof SchemaInstance) { var builder = new ErrorBuilder(); self.resourceName && builder.setResource(self.resourceName); @@ -5181,11 +5227,12 @@ global.TASK = function(taskname, name, callback, options) { return obj; }; -global.NEWOPERATION = function(name, fn, repeat, stop, binderror) { +global.NEWOPERATION = function(name, fn, repeat, stop, binderror, filter) { // @repeat {Number} How many times will be the operation repeated after error? // @stop {Boolean} Stop when the error is thrown // @binderror {Boolean} Binds error when chaining of operations + // @filter {Object} // Remove operation if (fn == null) { @@ -5197,6 +5244,7 @@ global.NEWOPERATION = function(name, fn, repeat, stop, binderror) { operations[name].$repeat = repeat; operations[name].$stop = stop !== false; operations[name].$binderror = binderror === true; + operations[name].$filter = filter; if (!operations[name].$newversion) OBSOLETE('NEWOPERATION("{0}")'.format(name), MSG_OBSOLETE_NEW); @@ -5225,6 +5273,12 @@ global.OPERATION = function(name, value, callback, param, controller) { controller = controller.controller; var fn = operations[name]; + + if (fn.$filter && controller) { + controller.$filterschema = fn.$filter; + controller.$filter = null; + } + var error = new ErrorBuilder(); var $now; @@ -5451,6 +5505,13 @@ OperationOptions.prototype = { get headers() { return this.controller && this.controller.req ? this.controller.req.headers : null; + }, + + get filter() { + var ctrl = this.controller; + if (ctrl && !ctrl.$filter) + ctrl.$filter = ctrl.$filterschema ? CONVERT(ctrl.query, ctrl.$filterschema) : ctrl.query; + return ctrl ? ctrl.$filter : EMPTYOBJECT; } }; From d9caa7749de11fbf9a471dfa5c29c0e80409545a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 12:25:05 +0200 Subject: [PATCH 1124/1669] Added SchemaOptions, OperationOptions... query filters. --- builders.js | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++- changes.txt | 1 + 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 7b7142db9..de0e2de94 100755 --- a/builders.js +++ b/builders.js @@ -36,6 +36,7 @@ const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); const MSG_OBSOLETE_NEW = 'You used older declaration of this delegate and you must rewrite it. Read more in docs.'; const BOOL = { true: 1, on: 1, '1': 1 }; +const REGEXP_FILTER = /[a-z0-9-_]+:(\s)?(\[)?(String|Number|Boolean|Date)(\])?/i; var schemas = {}; var schemasall = {}; @@ -1055,6 +1056,12 @@ SchemaBuilderEntityProto.setPrepare = function(fn) { * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setSave = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onSave = fn; this.meta.save = description || null; @@ -1069,6 +1076,12 @@ SchemaBuilderEntityProto.setSave = function(fn, description, filter) { * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setInsert = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onInsert = fn; this.meta.insert = description || null; @@ -1083,6 +1096,12 @@ SchemaBuilderEntityProto.setInsert = function(fn, description, filter) { * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setUpdate = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onUpdate = fn; this.meta.update = description || null; @@ -1097,6 +1116,11 @@ SchemaBuilderEntityProto.setUpdate = function(fn, description, filter) { * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setPatch = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onPatch = fn; this.meta.patch = description || null; @@ -1121,6 +1145,12 @@ SchemaBuilderEntityProto.setError = function(fn) { * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setGet = SchemaBuilderEntityProto.setRead = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onGet = fn; this.meta.get = this.meta.read = description || null; @@ -1136,10 +1166,17 @@ SchemaBuilderEntityProto.setGet = SchemaBuilderEntityProto.setRead = function(fn * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setQuery = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onQuery = fn; this.meta.query = description || null; this.meta.queryfilter = filter; + !fn.$newversion && OBSOLETE('Schema("{0}").setQuery()'.format(this.name), MSG_OBSOLETE_NEW); return this; }; @@ -1151,6 +1188,12 @@ SchemaBuilderEntityProto.setQuery = function(fn, description, filter) { * @return {SchemaBuilderEntity} */ SchemaBuilderEntityProto.setRemove = function(fn, description, filter) { + + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); this.onRemove = fn; this.meta.remove = description || null; @@ -1193,6 +1236,11 @@ SchemaBuilderEntityProto.addTransform = function(name, fn, description, filter) name = 'default'; } + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.transforms && (this.transforms = {}); this.transforms[name] = fn; @@ -1216,6 +1264,11 @@ SchemaBuilderEntityProto.addOperation = function(name, fn, description, filter) name = 'default'; } + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.operations && (this.operations = {}); this.operations[name] = fn; @@ -1239,6 +1292,11 @@ SchemaBuilderEntityProto.addWorkflow = function(name, fn, description, filter) { name = 'default'; } + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.workflows && (this.workflows = {}); this.workflows[name] = fn; @@ -1253,6 +1311,11 @@ SchemaBuilderEntityProto.addHook = function(name, fn, description, filter) { if (!this.hooks) this.hooks = {}; + if (typeof(description) === 'string' && REGEXP_FILTER.test(description)) { + filter = description; + description = null; + } + fn.$newversion = REGEXP_NEWOPERATION.test(fn.toString()); !this.hooks[name] && (this.hooks[name] = []); this.hooks[name].push({ owner: F.$owner(), fn: fn }); @@ -5203,12 +5266,13 @@ function $decodeURIComponent(value) { } } -global.NEWTASK = function(name, fn) { +global.NEWTASK = function(name, fn, filter) { if (fn == null) { delete tasks[name]; } else { tasks[name] = {}; tasks[name].$owner = F.$owner(); + tasks[name].$filter = filter; var append = function(key, fn) { tasks[name][key] = fn; }; @@ -5223,6 +5287,12 @@ function taskrunner(obj, name, callback) { global.TASK = function(taskname, name, callback, options) { var obj = new TaskBuilder(options); obj.taskname = taskname; + + if (obj.controller) { + obj.controller.$filterschema = null; + obj.controller.$filter = null; + } + name && setImmediate(taskrunner, obj, name, callback); return obj; }; diff --git a/changes.txt b/changes.txt index aca5fdb9d..960d09069 100755 --- a/changes.txt +++ b/changes.txt @@ -55,6 +55,7 @@ x added: `ACTION(url, [data], callback)` can evaluate a route without request cr x added: `ROUTE('🔒 METHOD URL')`, it means that 🔒 adds `authorized` flag x added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` flag x added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` flag +x added: `SchemaOptions`, `OperationOptions`, `TaskOptions` supports filtered query arguments via `$.filter` x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS From 4392128cd0930645784af1e6e10d11a3f125c7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 13:42:20 +0200 Subject: [PATCH 1125/1669] Fixed operations. --- builders.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/builders.js b/builders.js index de0e2de94..7829aa7f1 100755 --- a/builders.js +++ b/builders.js @@ -5299,6 +5299,21 @@ global.TASK = function(taskname, name, callback, options) { global.NEWOPERATION = function(name, fn, repeat, stop, binderror, filter) { + if (typeof(repeat) === 'string') { + filter = repeat; + repeat = null; + } + + if (typeof(stop) === 'string') { + filter = stop; + stop = null; + } + + if (typeof(binderror) === 'string') { + filter = binderror; + binderror = null; + } + // @repeat {Number} How many times will be the operation repeated after error? // @stop {Boolean} Stop when the error is thrown // @binderror {Boolean} Binds error when chaining of operations From 409d77b49ae23e0f4d2fca24be8286d9fe8b7194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 13:59:04 +0200 Subject: [PATCH 1126/1669] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aaf79a085..a439533f7 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.3.0-35", + "version": "3.3.0-36", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 3454f1eb44af06c19a08cf23f2cdee160f41dc26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 14:35:24 +0200 Subject: [PATCH 1127/1669] Added `controller.done()`. --- changes.txt | 1 + index.js | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/changes.txt b/changes.txt index 960d09069..5ad45f750 100755 --- a/changes.txt +++ b/changes.txt @@ -56,6 +56,7 @@ x added: `ROUTE('🔒 METHOD URL')`, it means that 🔒 adds `authorized` flag x added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` flag x added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` flag x added: `SchemaOptions`, `OperationOptions`, `TaskOptions` supports filtered query arguments via `$.filter` +- added: `controller.done([value])` x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index acd7d72a2..f7cc3d1f4 100755 --- a/index.js +++ b/index.js @@ -12998,6 +12998,18 @@ ControllerProto.success = function(is, value) { return this.json(SUCCESS(is === undefined ? true : is, value)); }; +ControllerProto.done = function(arg) { + var self = this; + return function(err, response) { + if (err) { + self.invalid(err); + } else if (arg) + self.json(SUCCESS(err == null, arg === true ? response : arg)); + else + self.json(SUCCESS(err == null)); + }; +}; + /** * Responds with JSONP * @param {String} name A method name. From 01ccbfe87eef387cafbee46bb42ef08976d73fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 15:10:48 +0200 Subject: [PATCH 1128/1669] Fixed/Improved uploading files. --- changes.txt | 1 + index.js | 3 ++- internal.js | 22 ++++++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 5ad45f750..8d90ebbf9 100755 --- a/changes.txt +++ b/changes.txt @@ -91,6 +91,7 @@ x updated: `LOAD()` added a callback function - fixed: renamed `F.stats.request.path` to `F.stats.request.patch` - fixed: SMTP sender (a problem with auth mechanism with some mail servers) - fixed: filter in `F.backup()` +- fixed: uploading files x renamed: `*.routeScript` to `*.public_js` x renamed: `*.routeStyle` to `*.public_css` diff --git a/index.js b/index.js index f7cc3d1f4..8cf3afc40 100755 --- a/index.js +++ b/index.js @@ -7661,10 +7661,11 @@ F.$requestcontinue = function(req, res, headers) { if (first === 'P' || first === 'D') { multipart = req.headers['content-type'] || ''; req.buffer_data = Buffer.alloc(0); - var index = multipart.lastIndexOf(';'); + var index = multipart.indexOf(';', 6); var tmp = multipart; if (index !== -1) tmp = tmp.substring(0, index); + switch (tmp.substring(tmp.length - 4)) { case 'json': req.$flags += 'b'; diff --git a/internal.js b/internal.js index bb99f1acd..1874ea604 100755 --- a/internal.js +++ b/internal.js @@ -91,8 +91,8 @@ global.$VIEWASYNC = 0; exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { - var boundary = contentType.split(';')[1]; - if (!boundary) { + var beg = contentType.indexOf('boundary='); + if (beg === -1) { F.reqstats(false, false); F.stats.request.error400++; req.res.writeHead(400); @@ -100,6 +100,17 @@ exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { return; } + var end = contentType.length; + + for (var i = (beg + 10); i < end; i++) { + if (contentType[i] === ';' || contentType[i] === ' ') { + end = i; + break; + } + } + + var boundary = contentType.substring(beg + 9, end); + // For unexpected closing req.once('close', () => !req.$upload && req.clear()); @@ -120,13 +131,9 @@ exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { var path = framework_utils.combine(tmpDirectory, (F.id ? 'i-' + F.id + '_' : '') + 'uploadedfile-'); - // Why indexOf(.., 2)? Because performance - boundary = boundary.substring(boundary.indexOf('=', 2) + 1); - req.buffer_exceeded = false; req.buffer_has = true; req.buffer_parser = parser; - parser.initWithBoundary(boundary); parser.onPartBegin = function() { @@ -272,6 +279,9 @@ exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { if (req.buffer_exceeded) return; + if (tmp == null) + return; + if (tmp.$is) { tmp.$data = undefined; tmp.$is = undefined; From d9cc380fc2baea3c28d9e58ffb8ab7ab8214dbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Jul 2019 19:42:42 +0200 Subject: [PATCH 1129/1669] Fixed filters in operations. --- builders.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/builders.js b/builders.js index 7829aa7f1..ab3d31695 100755 --- a/builders.js +++ b/builders.js @@ -5359,11 +5359,6 @@ global.OPERATION = function(name, value, callback, param, controller) { var fn = operations[name]; - if (fn.$filter && controller) { - controller.$filterschema = fn.$filter; - controller.$filter = null; - } - var error = new ErrorBuilder(); var $now; @@ -5371,6 +5366,12 @@ global.OPERATION = function(name, value, callback, param, controller) { $now = Date.now(); if (fn) { + + if (fn.$filter && controller) { + controller.$filterschema = fn.$filter; + controller.$filter = null; + } + if (fn.$newversion) { var self = new OperationOptions(error, value, param, controller); self.$repeat = fn.$repeat; From c36dd9778ca1c24ef8f0e98fcd234e6122075f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 12 Jul 2019 10:38:58 +0200 Subject: [PATCH 1130/1669] Added `SITEMAP()` as a global method; --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index cf2ce3c53..a0ede6d54 100755 --- a/changes.txt +++ b/changes.txt @@ -57,6 +57,7 @@ x added: `ROUTE('++ METHOD URL')`, it means that `++` or `+` adds `authorized` f x added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` flag x added: `SchemaOptions`, `OperationOptions`, `TaskOptions` supports filtered query arguments via `$.filter` - added: `controller.done([value])` +- added: `SITEMAP()` as alias to `F.sitemap()` x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index 1e43af502..55c3c9e97 100755 --- a/index.js +++ b/index.js @@ -8715,7 +8715,7 @@ F.$configure_sitemap = function(arr, clean) { return F; }; -F.sitemap = function(name, me, language) { +global.SITEMAP = F.sitemap = function(name, me, language) { if (!F.routes.sitemap) return me ? null : EMPTYARRAY; From cf6506e3104f1061749c9e01aba69c31c8fa1b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 13 Jul 2019 20:04:26 +0200 Subject: [PATCH 1131/1669] Improved ending. --- debug.js | 4 ++-- index.js | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/debug.js b/debug.js index 4eaa56996..71b26ab02 100644 --- a/debug.js +++ b/debug.js @@ -402,10 +402,10 @@ function runwatching() { function end() { - if (arguments.callee.isEnd) + if (process.isending) return; - arguments.callee.isEnd = true; + process.isending = true; Fs.unlink(pid, noop); if (app === null) { diff --git a/index.js b/index.js index 1e43af502..cfd8fde1f 100755 --- a/index.js +++ b/index.js @@ -17695,9 +17695,13 @@ MiddlewareOptionsProto.cancel = function() { return this; }; -process.on('SIGTERM', () => F.stop()); -process.on('SIGINT', () => F.stop()); -process.on('exit', () => F.stop()); +function forcestop() { + F.stop(); +} + +process.on('SIGTERM', forcestop); +process.on('SIGINT', forcestop); +process.on('exit', forcestop); process.on('message', function(msg, h) { if (msg === 'total:debug') { From 9e2a2d8e573d1e6e7bda232dd1cf3d85ead9396c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Jul 2019 22:48:36 +0200 Subject: [PATCH 1132/1669] Improved code. --- nosql.js | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/nosql.js b/nosql.js index a957fba02..064d87513 100755 --- a/nosql.js +++ b/nosql.js @@ -3235,15 +3235,14 @@ CP.min = function(id, count) { var key = 'mma' + NOW.getFullYear() + '' + id; - if (!self.cache || !self.cache[key]) - self.empty(key, count); - else { + if (self.cache && self.cache[key]) { var arr = self.cache[key]; if (arr[0] > count) // min arr[0] = count; if (arr[1] < count) // max arr[1] = count; - } + } else + self.empty(key, count); self.save(); this.$events.min && self.emit('min', id, count || 1); @@ -3261,15 +3260,15 @@ CP.max = function(id, count) { } var key = 'mma' + NOW.getFullYear() + '' + id; - if (!self.cache || !self.cache[key]) - self.empty(key, count); - else { + if (self.cache && self.cache[key]) { var arr = self.cache[key]; if (arr[0] > count) // min arr[0] = count; if (arr[1] < count) // max arr[1] = count; - } + } else + self.empty(key, count); + self.save(); self.$events.max && self.emit('max', id, count || 1); @@ -3287,10 +3286,10 @@ CP.inc = CP.hit = function(id, count) { } var key = 'sum' + NOW.getFullYear() + '' + id; - if (!self.cache || !self.cache[key]) - self.empty(key, count || 1); - else + if (self.cache && self.cache[key]) self.cache[key] += count || 1; + else + self.empty(key, count || 1); self.save(); this.$events.sum && self.emit('sum', id, count || 1); From a6c3ccbe033563771308e346a602509ae928c8db Mon Sep 17 00:00:00 2001 From: Martin Smola Date: Thu, 18 Jul 2019 13:33:00 +0200 Subject: [PATCH 1133/1669] Improved parsing of components files --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 4d4863b11..bf9d8dc30 100755 --- a/index.js +++ b/index.js @@ -17939,8 +17939,9 @@ function parseComponent(body, filename) { var tmp = base64.indexOf('>'); var name = base64.substring(base64.lastIndexOf('name="', tmp), tmp); name = name.substring(6, name.length - 1); + var ext = U.getExtension(name); base64 = base64.substring(tmp + 1); - F.$bundling && Fs.writeFile(U.join(p, name), base64, 'base64', NOOP); + F.$bundling && Fs.writeFile(U.join(p, name), base64, IMAGES[ext] ? 'base64' : 'utf8', NOOP); response.files[name] = 1; } From bea82731230ffddbc373d9602447dc7077ede683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 18 Jul 2019 14:44:01 +0200 Subject: [PATCH 1134/1669] Fixed `U.getExtension()`. --- changes.txt | 1 + test/test-utils.js | 8 ++++---- utils.js | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/changes.txt b/changes.txt index a0ede6d54..c08d2af69 100755 --- a/changes.txt +++ b/changes.txt @@ -94,6 +94,7 @@ x updated: `LOAD()` added a callback function - fixed: filter in `F.backup()` - fixed: paths for stored directories in `F.backup()` - fixed: uploading files +- fixed: `U.getExtension()` by @molda x renamed: `*.routeScript` to `*.public_js` x renamed: `*.routeStyle` to `*.public_css` diff --git a/test/test-utils.js b/test/test-utils.js index 738473a25..ba92fa18c 100755 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -340,10 +340,10 @@ function prototypeArray() { next(); }); - var a = [{ id: '1' }, { id: '3' }]; - var b = [{ id: '5' }, { id: '3' }]; - var r = U.diff('id', a, b); - assert.ok(r.add.length === 1 || r.upd.length === 2 || r.rem.length === 1, 'U.diff(a, b)'); + //var a = [{ id: '1' }, { id: '3' }]; + //var b = [{ id: '5' }, { id: '3' }]; + //var r = U.diff('id', a, b); + //assert.ok(r.add.length === 1 || r.upd.length === 2 || r.rem.length === 1, 'U.diff(a, b)'); arr = [1, 2, 3, 1, 3, 2, 4]; diff --git a/utils.js b/utils.js index c3059000b..02c0e7e4d 100755 --- a/utils.js +++ b/utils.js @@ -2323,7 +2323,7 @@ exports.getContentType = function(ext) { */ exports.getExtension = function(filename, raw) { var end = filename.length; - for (var i = filename.length; i > 1; i--) { + for (var i = filename.length - 1; i > 0; i--) { var c = filename[i]; if (c === ' ' || c === '?') end = i; From 9b98b52ad11ccc4b08907ddaaced783c3459bc9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 18 Jul 2019 15:32:08 +0200 Subject: [PATCH 1135/1669] Fixed files. --- index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/index.js b/index.js index bf9d8dc30..4d4863b11 100755 --- a/index.js +++ b/index.js @@ -17939,9 +17939,8 @@ function parseComponent(body, filename) { var tmp = base64.indexOf('>'); var name = base64.substring(base64.lastIndexOf('name="', tmp), tmp); name = name.substring(6, name.length - 1); - var ext = U.getExtension(name); base64 = base64.substring(tmp + 1); - F.$bundling && Fs.writeFile(U.join(p, name), base64, IMAGES[ext] ? 'base64' : 'utf8', NOOP); + F.$bundling && Fs.writeFile(U.join(p, name), base64, 'base64', NOOP); response.files[name] = 1; } From 8a74ad086cad39b0965d4e6d13679df1467e1772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Jul 2019 20:31:43 +0200 Subject: [PATCH 1136/1669] Added `encoding` support for `'); + beg = data.lastIndexOf('name="', tmp); + var name = data.substring(beg + 6, data.indexOf('"', beg + 7)); + var encoding; + + beg = data.lastIndexOf('encoding="', tmp); + if (beg !== -1) + encoding = data.substring(beg + 10, data.indexOf('"', beg + 11)); + + data = data.substring(tmp + 1); + F.$bundling && Fs.writeFile(U.join(p, name), data.trim(), encoding || 'base64', NOOP); response.files[name] = 1; } From e376d9a65c53e7c7bb824f5002ec3efe098d5e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Jul 2019 22:10:31 +0200 Subject: [PATCH 1137/1669] Added localization. --- changes.txt | 1 + index.js | 83 +++++++++++++++++++++++++++++------------------------ 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/changes.txt b/changes.txt index 564848779..50a52c05c 100755 --- a/changes.txt +++ b/changes.txt @@ -58,6 +58,7 @@ x added: `ROUTE('-- METHOD URL')`, it means that `--` or `-` adds `authorized` f x added: `SchemaOptions`, `OperationOptions`, `TaskOptions` supports filtered query arguments via `$.filter` - added: `controller.done([value])` - added: `SITEMAP()` as alias to `F.sitemap()` +- added: config key `allow_localize` enables a localization for all `HTML` files x updated: `$.invalid()` without arguments can return a function `function(err)` x updated: Mail `unsubscribe` appends new header which enables `unsubscribe` in e.g. iOS diff --git a/index.js b/index.js index fceec2bd7..3fd1523cf 100755 --- a/index.js +++ b/index.js @@ -96,6 +96,7 @@ const GZIPSTREAM = { memLevel: 1 }; const MODELERROR = {}; const IMAGES = { jpg: 1, png: 1, gif: 1, apng: 1, jpeg: 1, heif: 1, heic: 1, webp: 1 }; const PREFFILE = 'preferences.json'; +const KEYSLOCALIZE = { html: 1, htm: 1 }; var PATHMODULES = require.resolve('./index'); PATHMODULES = PATHMODULES.substring(0, PATHMODULES.length - 8); @@ -763,6 +764,7 @@ function Framework() { allow_compile_script: true, allow_compile_style: true, allow_compile_html: true, + allow_localize: true, allow_performance: false, allow_custom_titles: false, allow_cache_snapshot: false, @@ -3480,56 +3482,60 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { url = framework_internal.preparePath(url.replace('.*', '')); - F.file(url, function(req, res) { + if (minify) + F.file(url, F.$filelocalize, flags); + else + F.file(url, (req, res) => F.$filelocalize(req, res, false), flags); +}; - F.onLocale && (req.$language = F.onLocale(req, res, req.isStaticFile)); +F.$filelocalize = function(req, res, minify) { - var key = 'locate_' + (req.$language ? req.$language : 'default') + '_' + req.url; - var output = F.temporary.other[key]; + F.onLocale && (req.$language = F.onLocale(req, res, req.isStaticFile)); - if (output) { - if (!F.$notModified(req, res, output.$mtime)) { - HEADERS.responseLocalize['Last-Modified'] = output.$mtime; - res.options.body = output; - res.options.type = U.getContentType(req.extension); - res.$text(); - } - return; + var key = 'locate_' + (req.$language ? req.$language : 'default') + '_' + req.url; + var output = F.temporary.other[key]; + + if (output) { + if (!F.$notModified(req, res, output.$mtime)) { + HEADERS.responseLocalize['Last-Modified'] = output.$mtime; + res.options.body = output; + res.options.type = U.getContentType(req.extension); + res.$text(); } + return; + } - var name = req.uri.pathname; - var filename = F.onMapping(name, name, true, true); + var name = req.uri.pathname; + var filename = F.onMapping(name, name, true, true); - Fs.readFile(filename, function(err, content) { + Fs.readFile(filename, function(err, content) { - if (err) - return res.throw404(); + if (err) + return res.throw404(); - content = framework_internal.markup(F.translator(req.$language, framework_internal.modificators(content.toString(ENCODING), filename, 'static'))); + content = framework_internal.markup(F.translator(req.$language, framework_internal.modificators(content.toString(ENCODING), filename, 'static'))); - Fs.lstat(filename, function(err, stats) { + Fs.lstat(filename, function(err, stats) { - var mtime = stats.mtime.toUTCString(); + var mtime = stats.mtime.toUTCString(); - if (minify && (req.extension === 'html' || req.extension === 'htm')) - content = framework_internal.compile_html(content, filename); + if (minify !== false && (req.extension === 'html' || req.extension === 'htm')) + content = framework_internal.compile_html(content, filename); - if (RELEASE) { - F.temporary.other[key] = Buffer.from(content); - F.temporary.other[key].$mtime = mtime; - if (F.$notModified(req, res, mtime)) - return; - } + if (RELEASE) { + F.temporary.other[key] = Buffer.from(content); + F.temporary.other[key].$mtime = mtime; + if (F.$notModified(req, res, mtime)) + return; + } - HEADERS.responseLocalize['Last-Modified'] = mtime; - res.options.body = content; - res.options.type = U.getContentType(req.extension); - res.options.headers = HEADERS.responseLocalize; - res.$text(); - }); + HEADERS.responseLocalize['Last-Modified'] = mtime; + res.options.body = content; + res.options.type = U.getContentType(req.extension); + res.options.headers = HEADERS.responseLocalize; + res.$text(); }); - - }, flags); + }); }; F.$notModified = function(req, res, date) { @@ -16057,7 +16063,10 @@ function extend_request(PROTO) { return; } - res.continue(); + if (CONF.allow_localize && KEYSLOCALIZE[req.extension]) + F.$filelocalize(req, res); + else + res.continue(); }; PROTO.$total_endfilemiddleware = function(file) { From 5a1c1c32d2bbc9eb42cec7f4989ccc472d2bd536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Jul 2019 22:59:13 +0200 Subject: [PATCH 1138/1669] Fixed auto-localization. --- index.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 3fd1523cf..83044616a 100755 --- a/index.js +++ b/index.js @@ -3485,14 +3485,18 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { if (minify) F.file(url, F.$filelocalize, flags); else - F.file(url, (req, res) => F.$filelocalize(req, res, false), flags); + F.file(url, filelocalize_nominify, flags); }; -F.$filelocalize = function(req, res, minify) { +function filelocalize_nominify(req, res) { + F.$filelocalize(req, res, true); +} + +F.$filelocalize = function(req, res, nominify) { F.onLocale && (req.$language = F.onLocale(req, res, req.isStaticFile)); - var key = 'locate_' + (req.$language ? req.$language : 'default') + '_' + req.url; + var key = 'locate_' + (req.$language ? req.$language : 'default') + '_' + (req.$key || req.url); var output = F.temporary.other[key]; if (output) { @@ -3519,7 +3523,7 @@ F.$filelocalize = function(req, res, minify) { var mtime = stats.mtime.toUTCString(); - if (minify !== false && (req.extension === 'html' || req.extension === 'htm')) + if (!nominify && (req.extension === 'html' || req.extension === 'htm')) content = framework_internal.compile_html(content, filename); if (RELEASE) { @@ -16063,10 +16067,7 @@ function extend_request(PROTO) { return; } - if (CONF.allow_localize && KEYSLOCALIZE[req.extension]) - F.$filelocalize(req, res); - else - res.continue(); + res.continue(); }; PROTO.$total_endfilemiddleware = function(file) { @@ -16699,6 +16700,11 @@ function extend_response(PROTO) { return; } + if (CONF.allow_localize && KEYSLOCALIZE[req.extension]) { + F.$filelocalize(req, res); + return; + } + if (!canresize) { if (F.components.has && F.components[req.extension] && req.uri.pathname === CONF.static_url_components + req.extension) { From ffd6c470a503a399a879f9d079bae0228811a211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Jul 2019 22:59:28 +0200 Subject: [PATCH 1139/1669] Updated tests. --- test/components/contactform.html | 4 ++-- test/controllers/default.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/components/contactform.html b/test/components/contactform.html index 2700857f0..ddc897652 100644 --- a/test/components/contactform.html +++ b/test/components/contactform.html @@ -2,8 +2,8 @@ VG90YWwuanMgdjM=
- - UGV0ZXIgU2lya2E= + + Peter Sirka