From d8c23fc327f9d9b372d37f685424f62a337c4b5f Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 4 Sep 2017 21:26:45 -0500 Subject: [PATCH 01/12] Intermediate commit: Very first pass at setting up .archive() by copying from .destroy(). (Still needs some love.) --- lib/waterline/methods/archive.js | 493 +++++++++++++++++++++++++++++++ 1 file changed, 493 insertions(+) create mode 100644 lib/waterline/methods/archive.js diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js new file mode 100644 index 000000000..27e530016 --- /dev/null +++ b/lib/waterline/methods/archive.js @@ -0,0 +1,493 @@ +/** + * Module Dependencies + */ + +var util = require('util'); +var async = require('async'); +var _ = require('@sailshq/lodash'); +var flaverr = require('flaverr'); +var parley = require('parley'); +var buildOmen = require('../utils/query/build-omen'); +var forgeAdapterError = require('../utils/query/forge-adapter-error'); +var forgeStageTwoQuery = require('../utils/query/forge-stage-two-query'); +var forgeStageThreeQuery = require('../utils/query/forge-stage-three-query'); +var getQueryModifierMethods = require('../utils/query/get-query-modifier-methods'); +var processAllRecords = require('../utils/query/process-all-records'); +var verifyModelMethodContext = require('../utils/query/verify-model-method-context'); + + +/** + * Module constants + */ + +var DEFERRED_METHODS = getQueryModifierMethods('archive'); + + + +/** + * archive() + * + * Archive (s.k.a. "soft-delete") records that match the specified criteria, + * saving them as new records in the built-in Archive model, then destroying + * the originals. + * + * ``` + * // Archive all bank accounts with more than $32,000 in them. + * BankAccount.archive().where({ + * balance: { '>': 32000 } + * }).exec(function(err) { + * // ... + * }); + * ``` + * + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * Usage without deferred object: + * ================================================ + * + * @param {Dictionary?} criteria + * + * @param {Function?} explicitCbMaybe + * Callback function to run when query has either finished successfully or errored. + * (If unspecified, will return a Deferred object instead of actually doing anything.) + * + * @param {Ref?} meta + * For internal use. + * + * @returns {Ref?} Deferred object if no `explicitCbMaybe` callback was provided + * + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * The underlying query keys: + * ============================== + * + * @qkey {Dictionary?} criteria + * + * @qkey {Dictionary?} meta + * @qkey {String} using + * @qkey {String} method + * + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ + +module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */) { + + // Verify `this` refers to an actual Sails/Waterline model. + verifyModelMethodContext(this); + + // Set up a few, common local vars for convenience / familiarity. + var WLModel = this; + var orm = this.waterline; + var modelIdentity = this.identity; + + // Build an omen for potential use in the asynchronous callback below. + var omen = buildOmen(archive); + + // Build initial query. + var query = { + method: 'archive', + using: modelIdentity, + criteria: undefined, + meta: undefined + }; + + // ██╗ ██╗ █████╗ ██████╗ ██╗ █████╗ ██████╗ ██╗ ██████╗███████╗ + // ██║ ██║██╔══██╗██╔══██╗██║██╔══██╗██╔══██╗██║██╔════╝██╔════╝ + // ██║ ██║███████║██████╔╝██║███████║██║ ██║██║██║ ███████╗ + // ╚██╗ ██╔╝██╔══██║██╔══██╗██║██╔══██║██║ ██║██║██║ ╚════██║ + // ╚████╔╝ ██║ ██║██║ ██║██║██║ ██║██████╔╝██║╚██████╗███████║ + // ╚═══╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═════╝╚══════╝ + // + // FUTURE: when time allows, update this to match the "VARIADICS" format + // used in the other model methods. + + // The explicit callback, if one was provided. + var explicitCbMaybe; + + // Handle double meaning of first argument: + // + // • archive(criteria, ...) + if (!_.isFunction(arguments[0])) { + query.criteria = arguments[0]; + explicitCbMaybe = arguments[1]; + query.meta = arguments[2]; + } + // • archive(explicitCbMaybe, ...) + else { + explicitCbMaybe = arguments[0]; + query.meta = arguments[1]; + } + + + + // ██████╗ ███████╗███████╗███████╗██████╗ + // ██╔══██╗██╔════╝██╔════╝██╔════╝██╔══██╗ + // ██║ ██║█████╗ █████╗ █████╗ ██████╔╝ + // ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██╔══██╗ + // ██████╔╝███████╗██║ ███████╗██║ ██║ + // ╚═════╝ ╚══════╝╚═╝ ╚══════╝╚═╝ ╚═╝ + // + // ██╗███╗ ███╗ █████╗ ██╗ ██╗██████╗ ███████╗██╗ + // ██╔╝████╗ ████║██╔══██╗╚██╗ ██╔╝██╔══██╗██╔════╝╚██╗ + // ██║ ██╔████╔██║███████║ ╚████╔╝ ██████╔╝█████╗ ██║ + // ██║ ██║╚██╔╝██║██╔══██║ ╚██╔╝ ██╔══██╗██╔══╝ ██║ + // ╚██╗██║ ╚═╝ ██║██║ ██║ ██║ ██████╔╝███████╗██╔╝ + // ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ + // + // ┌┐ ┬ ┬┬┬ ┌┬┐ ┬ ┬─┐┌─┐┌┬┐┬ ┬┬─┐┌┐┌ ┌┐┌┌─┐┬ ┬ ┌┬┐┌─┐┌─┐┌─┐┬─┐┬─┐┌─┐┌┬┐ + // ├┴┐│ │││ ││ ┌┼─ ├┬┘├┤ │ │ │├┬┘│││ │││├┤ │││ ││├┤ ├┤ ├┤ ├┬┘├┬┘├┤ ││ + // └─┘└─┘┴┴─┘─┴┘ └┘ ┴└─└─┘ ┴ └─┘┴└─┘└┘ ┘└┘└─┘└┴┘ ─┴┘└─┘└ └─┘┴└─┴└─└─┘─┴┘ + // ┌─ ┬┌─┐ ┬─┐┌─┐┬ ┌─┐┬ ┬┌─┐┌┐┌┌┬┐ ─┐ + // │─── │├┤ ├┬┘├┤ │ ├┤ └┐┌┘├─┤│││ │ ───│ + // └─ ┴└ ┴└─└─┘┴─┘└─┘ └┘ ┴ ┴┘└┘ ┴ ─┘ + // If a callback function was not specified, then build a new Deferred and bail now. + // + // > This method will be called AGAIN automatically when the Deferred is executed. + // > and next time, it'll have a callback. + return parley( + + function (done){ + + // Otherwise, IWMIH, we know that a callback was specified. + // So... + + // ███████╗██╗ ██╗███████╗ ██████╗██╗ ██╗████████╗███████╗ + // ██╔════╝╚██╗██╔╝██╔════╝██╔════╝██║ ██║╚══██╔══╝██╔════╝ + // █████╗ ╚███╔╝ █████╗ ██║ ██║ ██║ ██║ █████╗ + // ██╔══╝ ██╔██╗ ██╔══╝ ██║ ██║ ██║ ██║ ██╔══╝ + // ███████╗██╔╝ ██╗███████╗╚██████╗╚██████╔╝ ██║ ███████╗ + // ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚══════╝ + // + // ╔═╗╔═╗╦═╗╔═╗╔═╗ ┌─┐┌┬┐┌─┐┌─┐┌─┐ ┌┬┐┬ ┬┌─┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ + // ╠╣ ║ ║╠╦╝║ ╦║╣ └─┐ │ ├─┤│ ┬├┤ │ ││││ │ │─┼┐│ │├┤ ├┬┘└┬┘ + // ╚ ╚═╝╩╚═╚═╝╚═╝ └─┘ ┴ ┴ ┴└─┘└─┘ ┴ └┴┘└─┘ └─┘└└─┘└─┘┴└─ ┴ + // + // Forge a stage 2 query (aka logical protostatement) + // This ensures a normalized format. + try { + forgeStageTwoQuery(query, orm); + } catch (e) { + switch (e.code) { + case 'E_INVALID_CRITERIA': + return done( + flaverr( + { name: 'UsageError' }, + new Error( + 'Invalid criteria.\n'+ + 'Details:\n'+ + ' '+e.details+'\n' + ) + ) + ); + + case 'E_NOOP': + // Determine the appropriate no-op result. + // If `fetch` meta key is set, use `[]`-- otherwise use `undefined`. + var noopResult = undefined; + if (query.meta && query.meta.fetch) { + noopResult = []; + }//>- + return done(undefined, noopResult); + + default: + return done(e); + } + } + + + // ╦ ╦╔═╗╔╗╔╔╦╗╦ ╔═╗ ┬ ┬┌─┐┌─┐┌─┐┬ ┬┌─┐┬ ┌─┐ ┌─┐┌─┐┬ ┬ ┌┐ ┌─┐┌─┐┬┌─ + // ╠═╣╠═╣║║║ ║║║ ║╣ BEFORE │ │├┤ ├┤ │ └┬┘│ │ ├┤ │ ├─┤│ │ ├┴┐├─┤│ ├┴┐ + // ╩ ╩╩ ╩╝╚╝═╩╝╩═╝╚═╝ ┴─┘┴└ └─┘└─┘ ┴ └─┘┴─┘└─┘ └─┘┴ ┴┴─┘┴─┘└─┘┴ ┴└─┘┴ ┴ + (function (proceed) { + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: `beforeArchive` lifecycle callback? + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + return proceed(); + })(function (err, query) { + if (err) { return done(err); } + + // ┬ ┌─┐┌─┐┬┌─┬ ┬┌─┐ ┌─┐┌┬┐┌─┐┌─┐┌┬┐┌─┐┬─┐ + // │ │ ││ │├┴┐│ │├─┘ ├─┤ ││├─┤├─┘ │ ├┤ ├┬┘ + // ┴─┘└─┘└─┘┴ ┴└─┘┴ ┴ ┴─┴┘┴ ┴┴ ┴ └─┘┴└─ + // Look up the appropriate adapter to use for this model. + + // Get a reference to the adapter. + var adapter = WLModel._adapter; + if (!adapter) { + // ^^One last sanity check to make sure the adapter exists-- again, for compatibility's sake. + return done(new Error('Consistency violation: Cannot find adapter for model (`' + modelIdentity + '`). This model appears to be using datastore `'+WLModel.datastore+'`, but the adapter for that datastore cannot be located.')); + } + + // Verify the adapter has `destroy` & `find` methods. + if (!adapter.destroy || !adapter.find) { + return done(new Error('The adapter used by this model (`' + modelIdentity + '`) doesn\'t support the `destroy`+`find` methods.')); + } + + // ╔═╗╔═╗╦═╗╔═╗╔═╗ ┌─┐┌┬┐┌─┐┌─┐┌─┐ ┌┬┐┬ ┬┬─┐┌─┐┌─┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ + // ╠╣ ║ ║╠╦╝║ ╦║╣ └─┐ │ ├─┤│ ┬├┤ │ ├─┤├┬┘├┤ ├┤ │─┼┐│ │├┤ ├┬┘└┬┘ + // ╚ ╚═╝╩╚═╚═╝╚═╝ └─┘ ┴ ┴ ┴└─┘└─┘ ┴ ┴ ┴┴└─└─┘└─┘ └─┘└└─┘└─┘┴└─ ┴ + // Now, destructively forge this S2Q into a S3Q. + try { + query = forgeStageThreeQuery({ + stageTwoQuery: query, + identity: modelIdentity, + transformer: WLModel._transformer, + originalModels: orm.collections + }); + } catch (e) { return done(e); } + + + // ┬┌─┐ ╔═╗╔═╗╔═╗╔═╗╔═╗╔╦╗╔═╗ ┌─┐┌┐┌┌─┐┌┐ ┬ ┌─┐┌┬┐ ┌┬┐┬ ┬┌─┐┌┐┌ + // │├┤ ║ ╠═╣╚═╗║ ╠═╣ ║║║╣ ├┤ │││├─┤├┴┐│ ├┤ ││ │ ├─┤├┤ │││ + // ┴└ ╚═╝╩ ╩╚═╝╚═╝╩ ╩═╩╝╚═╝ └─┘┘└┘┴ ┴└─┘┴─┘└─┘─┴┘┘ ┴ ┴ ┴└─┘┘└┘ + // ┌─┐┬┌┐┌┌┬┐ ╦╔╦╗╔═╗ ┌┬┐┌─┐ ┌┬┐┌─┐┌─┐┌┬┐┬─┐┌─┐┬ ┬ + // ├┤ ││││ ││ ║ ║║╚═╗ │ │ │ ││├┤ └─┐ │ ├┬┘│ │└┬┘ + // └ ┴┘└┘─┴┘ ╩═╩╝╚═╝ ┴ └─┘ ─┴┘└─┘└─┘ ┴ ┴└─└─┘ ┴ + (function _maybeFindIdsToDestroy(proceed) { + + // If `cascade` meta key is NOT enabled, then just proceed. + if (!query.meta || !query.meta.cascade) { + return proceed(); + } + + // Look up the ids of records that will be destroyed. + // (We need these because, later, since `cascade` is enabled, we'll need + // to empty out all of their associated collections.) + // + // > FUTURE: instead of doing this, consider forcing `fetch: true` in the + // > implementation of `.destroy()` when `cascade` meta key is enabled (mainly + // > for consistency w/ the approach used in createEach()/create()) + + // To do this, we'll grab the appropriate adapter method and call it with a stage 3 + // "find" query, using almost exactly the same QKs as in the incoming "destroy". + // The only tangible difference is that its criteria has a `select` clause so that + // records only contain the primary key field (by column name, of course.) + var pkColumnName = WLModel.schema[WLModel.primaryKey].columnName; + if (!pkColumnName) { + return done(new Error('Consistency violation: model `' + WLModel.identity + '` schema has no primary key column name!')); + } + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // > Note: We have to look up the column name this way (instead of simply using the + // > getAttribute() utility) because it is currently only fully normalized on the + // > `schema` dictionary-- the model's attributes don't necessarily have valid, + // > normalized column names. For more context, see: + // > https://github.com/balderdashy/waterline/commit/19889b7ee265e9850657ec2b4c7f3012f213a0ae#commitcomment-20668097 + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + adapter.find(WLModel.datastore, { + method: 'find', + using: query.using, + criteria: { + where: query.criteria.where, + skip: query.criteria.skip, + limit: query.criteria.limit, + sort: query.criteria.sort, + select: [ pkColumnName ] + }, + meta: query.meta //<< this is how we know that the same db connection will be used + }, function _afterPotentiallyFindingIdsToDestroy(err, pRecords) { + if (err) { + err = forgeAdapterError(err, omen, 'find', modelIdentity, orm); + return proceed(err); + } + + // Slurp out just the array of ids (pk values), and send that back. + var ids = _.pluck(pRecords, pkColumnName); + return proceed(undefined, ids); + + });// + + })(function _afterPotentiallyLookingUpRecordsToCascade(err, idsOfRecordsBeingDestroyedMaybe) { + if (err) { return done(err); } + + + // Now we'll actually perform the `destroy`. + + // ┌─┐┌─┐┌┐┌┌┬┐ ┌┬┐┌─┐ ╔═╗╔╦╗╔═╗╔═╗╔╦╗╔═╗╦═╗ + // └─┐├┤ │││ ││ │ │ │ ╠═╣ ║║╠═╣╠═╝ ║ ║╣ ╠╦╝ + // └─┘└─┘┘└┘─┴┘ ┴ └─┘ ╩ ╩═╩╝╩ ╩╩ ╩ ╚═╝╩╚═ + // Call the `destroy` adapter method. + adapter.destroy(WLModel.datastore, query, function _afterTalkingToAdapter(err, rawAdapterResult) { + if (err) { + err = forgeAdapterError(err, omen, 'destroy', modelIdentity, orm); + return done(err); + }//-• + + + // ╦═╗╔═╗╦╔╗╔ ╔╦╗╔═╗╦ ╦╔╗╔ ╔╦╗╔═╗╔═╗╔╦╗╦═╗╦ ╦╔═╗╔╦╗╦╔═╗╔╗╔ ┌─┐┌┐┌┌┬┐┌─┐ + // ╠╦╝╠═╣║║║║ ║║║ ║║║║║║║ ║║║╣ ╚═╗ ║ ╠╦╝║ ║║ ║ ║║ ║║║║ │ ││││ │ │ │ + // ╩╚═╩ ╩╩╝╚╝ ═╩╝╚═╝╚╩╝╝╚╝ ═╩╝╚═╝╚═╝ ╩ ╩╚═╚═╝╚═╝ ╩ ╩╚═╝╝╚╝ └─┘┘└┘ ┴ └─┘ + // ┌─┐┌─┐┌─┐┌─┐┌─┐┬┌─┐┌┬┐┬┌─┐┌┐┌┌─┐ ┌─ ┬ ┌─┐ ┌─┐┌─┐┌─┐┌─┐┌─┐┌┬┐┌─┐ ─┐ + // ├─┤└─┐└─┐│ ││ │├─┤ │ ││ ││││└─┐ │ │ ├┤ │ ├─┤└─┐│ ├─┤ ││├┤ │ + // ┴ ┴└─┘└─┘└─┘└─┘┴┴ ┴ ┴ ┴└─┘┘└┘└─┘ └─ ┴o└─┘o └─┘┴ ┴└─┘└─┘┴ ┴─┴┘└─┘ ─┘ + (function _maybeWipeAssociatedCollections(proceed) { + + // If `cascade` meta key is NOT enabled, then just proceed. + if (!query.meta || !query.meta.cascade) { + return proceed(); + } + + // Otherwise, then we should have the records we looked up before. + // (Here we do a quick sanity check.) + if (!_.isArray(idsOfRecordsBeingDestroyedMaybe)) { + return proceed(new Error('Consistency violation: Should have an array of records looked up before! But instead, got: '+util.inspect(idsOfRecordsBeingDestroyedMaybe, {depth: 5})+'')); + } + + // --• + // Now we'll clear out collections belonging to the specified records. + // (i.e. use `replaceCollection` to wipe them all out to be `[]`) + + + // First, if there are no target records, then gracefully bail without complaint. + // (i.e. this is a no-op) + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: Revisit this and verify that it's unnecessary. While this isn't a bad micro-optimization, + // its existence makes it seem like this wouldn't work or would cause a warning or something. And it + // really shouldn't be necessary. (It's doubtful that it adds any real tangible performance benefit anyway.) + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + if (idsOfRecordsBeingDestroyedMaybe.length === 0) { + return proceed(); + }//-• + + // Otherwise, we have work to do. + // + // Run .replaceCollection() for each associated collection of the targets, wiping them all out. + // (if n..m, this destroys junction records; otherwise, it's n..1, so this just nulls out the other side) + // + // > Note that we pass through `meta` here, ensuring that the same db connection is used, if possible. + async.each(_.keys(WLModel.attributes), function _eachAttribute(attrName, next) { + + var attrDef = WLModel.attributes[attrName]; + + // Skip everything other than collection attributes. + if (!attrDef.collection){ return next(); } + + // But otherwise, this is a collection attribute. So wipe it. + WLModel.replaceCollection(idsOfRecordsBeingDestroyedMaybe, attrName, [], function (err) { + if (err) { return next(err); } + + return next(); + + }, query.meta);// + + },// ~∞%° + function _afterwards(err) { + if (err) { return proceed(err); } + + return proceed(); + + });// + + })(function _afterPotentiallyWipingCollections(err) {// ~∞%° + if (err) { + return done(err); + } + + // ╔╦╗╦═╗╔═╗╔╗╔╔═╗╔═╗╔═╗╦═╗╔╦╗ ┬─┐┌─┐┌─┐┌─┐┬─┐┌┬┐┌─┐ ┌┐ ┬ ┬┌┬┐ ┌─┐┌┐┌┬ ┬ ┬ ┬┌─┐ + // ║ ╠╦╝╠═╣║║║╚═╗╠╣ ║ ║╠╦╝║║║ ├┬┘├┤ │ │ │├┬┘ ││└─┐ ├┴┐│ │ │ │ │││││ └┬┘ │├┤ + // ╩ ╩╚═╩ ╩╝╚╝╚═╝╚ ╚═╝╩╚═╩ ╩ ┴└─└─┘└─┘└─┘┴└──┴┘└─┘ooo└─┘└─┘ ┴ └─┘┘└┘┴─┘┴ ┴└ + // ╔═╗╔═╗╔╦╗╔═╗╦ ╦ ┌┬┐┌─┐┌┬┐┌─┐ ┬┌─┌─┐┬ ┬ ┬ ┬┌─┐┌─┐ ┌─┐┌─┐┌┬┐ ┌┬┐┌─┐ ┌┬┐┬─┐┬ ┬┌─┐ + // ╠╣ ║╣ ║ ║ ╠═╣ │││├┤ │ ├─┤ ├┴┐├┤ └┬┘ │││├─┤└─┐ └─┐├┤ │ │ │ │ │ ├┬┘│ │├┤ + // ╚ ╚═╝ ╩ ╚═╝╩ ╩ ┴ ┴└─┘ ┴ ┴ ┴ ┴ ┴└─┘ ┴ └┴┘┴ ┴└─┘ └─┘└─┘ ┴ ┴ └─┘ ┴ ┴└─└─┘└─┘ + (function _maybeTransformRecords(proceed){ + + // If `fetch` was not enabled, return. + if (!_.has(query.meta, 'fetch') || query.meta.fetch === false) { + + // > Note: This `if` statement is a convenience, for cases where the result from + // > the adapter may have been coerced from `undefined` to `null` automatically. + // > (we want it to be `undefined` still, for consistency) + if (_.isNull(rawAdapterResult)) { + return proceed(); + }//-• + + if (!_.isUndefined(rawAdapterResult)) { + console.warn('\n'+ + 'Warning: Unexpected behavior in database adapter:\n'+ + 'Since `fetch` is NOT enabled, this adapter (for datastore `'+WLModel.datastore+'`)\n'+ + 'should NOT have sent back anything as the 2nd argument when triggering the callback\n'+ + 'from its `destroy` method. But it did!\n'+ + '\n'+ + '(Displaying this warning to help avoid confusion and draw attention to the bug.\n'+ + 'Specifically, got:\n'+ + util.inspect(rawAdapterResult, {depth:5})+'\n'+ + '(Ignoring it and proceeding anyway...)'+'\n' + ); + }//>- + + // Continue on. + return proceed(); + + }//-• + + // IWMIH then we know that `fetch: true` meta key was set, and so the + // adapter should have sent back an array. + + // Verify that the raw result from the adapter is an array. + if (!_.isArray(rawAdapterResult)) { + return proceed(new Error( + 'Unexpected behavior in database adapter: Since `fetch: true` was enabled, this adapter '+ + '(for datastore `'+WLModel.datastore+'`) should have sent back an array of records as the 2nd argument when triggering '+ + 'the callback from its `archive` method. But instead, got: '+util.inspect(rawAdapterResult, {depth:5})+'' + )); + }//-• + + // Attempt to convert the column names in each record back into attribute names. + var transformedRecords; + try { + transformedRecords = rawAdapterResult.map(function(record) { + return WLModel._transformer.unserialize(record); + }); + } catch (e) { return proceed(e); } + + // Check the records to verify compliance with the adapter spec, + // as well as any issues related to stale data that might not have been + // been migrated to keep up with the logical schema (`type`, etc. in + // attribute definitions). + try { + processAllRecords(transformedRecords, query.meta, modelIdentity, orm); + } catch (e) { return proceed(e); } + + // Now continue on. + return proceed(undefined, transformedRecords); + + })(function (err, transformedRecordsMaybe){ + if (err) { return done(err); } + + // ╔═╗╔═╗╔╦╗╔═╗╦═╗ ┬ ┬┌─┐┌─┐┌─┐┬ ┬┌─┐┬ ┌─┐ ┌─┐┌─┐┬ ┬ ┌┐ ┌─┐┌─┐┬┌─ + // ╠═╣╠╣ ║ ║╣ ╠╦╝ │ │├┤ ├┤ │ └┬┘│ │ ├┤ │ ├─┤│ │ ├┴┐├─┤│ ├┴┐ + // ╩ ╩╚ ╩ ╚═╝╩╚═ ┴─┘┴└ └─┘└─┘ ┴ └─┘┴─┘└─┘ └─┘┴ ┴┴─┘┴─┘└─┘┴ ┴└─┘┴ ┴ + (function (proceed){ + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: `afterArchive` lifecycle callback? + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + return proceed(); + })(function(err){ + if (err){ return done(err); } + return done(undefined, transformedRecordsMaybe); + });//_∏_ (†: "after" LC) + });//_∏_ (†: after determining (and potentially transforming) the result from the adapter) + });//_∏_ (†: _afterPotentiallyWipingCollections) + });//_∏_ (adapter.destroy) + }); //_∏_ (†: after potentially looking up records to cascade) + }); //_∏_ (†: "before" LC) + }, + + + explicitCbMaybe, + + + _.extend(DEFERRED_METHODS, { + + // Provide access to this model for use in query modifier methods. + _WLModel: WLModel, + + // Set up initial query metadata. + _wlQueryInfo: query, + + }) + + + );// + +}; From ea462b7ffc1cf90c03c6cd2796dea84cead64f20 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 4 Sep 2017 23:28:40 -0500 Subject: [PATCH 02/12] Implement first complete pass at .archive(), minus actually looking up the Archived model (which doesn't exist yet) --- lib/waterline/methods/archive.js | 351 ++++-------------- lib/waterline/methods/destroy.js | 12 +- .../utils/query/forge-stage-two-query.js | 8 +- .../utils/query/get-query-modifier-methods.js | 2 + 4 files changed, 77 insertions(+), 296 deletions(-) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index 27e530016..902c26fcc 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -2,17 +2,12 @@ * Module Dependencies */ -var util = require('util'); -var async = require('async'); var _ = require('@sailshq/lodash'); var flaverr = require('flaverr'); var parley = require('parley'); var buildOmen = require('../utils/query/build-omen'); -var forgeAdapterError = require('../utils/query/forge-adapter-error'); var forgeStageTwoQuery = require('../utils/query/forge-stage-two-query'); -var forgeStageThreeQuery = require('../utils/query/forge-stage-three-query'); var getQueryModifierMethods = require('../utils/query/get-query-modifier-methods'); -var processAllRecords = require('../utils/query/process-all-records'); var verifyModelMethodContext = require('../utils/query/verify-model-method-context'); @@ -166,18 +161,17 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // This ensures a normalized format. try { forgeStageTwoQuery(query, orm); - } catch (e) { - switch (e.code) { + } catch (err) { + switch (err.code) { case 'E_INVALID_CRITERIA': return done( - flaverr( - { name: 'UsageError' }, - new Error( - 'Invalid criteria.\n'+ - 'Details:\n'+ - ' '+e.details+'\n' - ) - ) + flaverr({ + name: 'UsageError', + message: + 'Invalid criteria.\n'+ + 'Details:\n'+ + ' '+err.details+'\n' + }, omen) ); case 'E_NOOP': @@ -190,287 +184,70 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ return done(undefined, noopResult); default: - return done(e); + return done(err); } - } - - - // ╦ ╦╔═╗╔╗╔╔╦╗╦ ╔═╗ ┬ ┬┌─┐┌─┐┌─┐┬ ┬┌─┐┬ ┌─┐ ┌─┐┌─┐┬ ┬ ┌┐ ┌─┐┌─┐┬┌─ - // ╠═╣╠═╣║║║ ║║║ ║╣ BEFORE │ │├┤ ├┤ │ └┬┘│ │ ├┤ │ ├─┤│ │ ├┴┐├─┤│ ├┴┐ - // ╩ ╩╩ ╩╝╚╝═╩╝╩═╝╚═╝ ┴─┘┴└ └─┘└─┘ ┴ └─┘┴─┘└─┘ └─┘┴ ┴┴─┘┴─┘└─┘┴ ┴└─┘┴ ┴ - (function (proceed) { - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // FUTURE: `beforeArchive` lifecycle callback? - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return proceed(); - })(function (err, query) { - if (err) { return done(err); } + }//fi - // ┬ ┌─┐┌─┐┬┌─┬ ┬┌─┐ ┌─┐┌┬┐┌─┐┌─┐┌┬┐┌─┐┬─┐ - // │ │ ││ │├┴┐│ │├─┘ ├─┤ ││├─┤├─┘ │ ├┤ ├┬┘ - // ┴─┘└─┘└─┘┴ ┴└─┘┴ ┴ ┴─┴┘┴ ┴┴ ┴ └─┘┴└─ - // Look up the appropriate adapter to use for this model. - // Get a reference to the adapter. - var adapter = WLModel._adapter; - if (!adapter) { - // ^^One last sanity check to make sure the adapter exists-- again, for compatibility's sake. - return done(new Error('Consistency violation: Cannot find adapter for model (`' + modelIdentity + '`). This model appears to be using datastore `'+WLModel.datastore+'`, but the adapter for that datastore cannot be located.')); - } + // Look up the Archived model. + var Archived;// TODO + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // TODO: provide a way of choosing which datastore is "dominant" + // as far as the Archived model is concerned (e.g. where does it live). + // + // TODO: provide a way of choosing the `tableName` and `columnName`s + // for the Archived model + // + // TODO: provide a way of disabling the Archived model (and thus + // disabling support for the `.archive()` model method) + // + // TODO: automigrate the built-in Archived model + // + // TODO: inject the built-in Archived model into the ORM's ontology + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Verify the adapter has `destroy` & `find` methods. - if (!adapter.destroy || !adapter.find) { - return done(new Error('The adapter used by this model (`' + modelIdentity + '`) doesn\'t support the `destroy`+`find` methods.')); - } - // ╔═╗╔═╗╦═╗╔═╗╔═╗ ┌─┐┌┬┐┌─┐┌─┐┌─┐ ┌┬┐┬ ┬┬─┐┌─┐┌─┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ - // ╠╣ ║ ║╠╦╝║ ╦║╣ └─┐ │ ├─┤│ ┬├┤ │ ├─┤├┬┘├┤ ├┤ │─┼┐│ │├┤ ├┬┘└┬┘ - // ╚ ╚═╝╩╚═╚═╝╚═╝ └─┘ ┴ ┴ ┴└─┘└─┘ ┴ ┴ ┴┴└─└─┘└─┘ └─┘└└─┘└─┘┴└─ ┴ - // Now, destructively forge this S2Q into a S3Q. - try { - query = forgeStageThreeQuery({ - stageTwoQuery: query, - identity: modelIdentity, - transformer: WLModel._transformer, - originalModels: orm.collections + // - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: pass through the `omen` in the metadata. + // - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬┌┐┌┌┬┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ + // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ├┤ ││││ ││ │─┼┐│ │├┤ ├┬┘└┬┘ + // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ └ ┴┘└┘─┴┘ └─┘└└─┘└─┘┴└─ ┴ + // Note that we pass in `meta` here, as well as in the other queries + // below. (This ensures we're on the same db connection, provided one + // was explicitly passed in!) + WLModel.find(query.criteria, function _afterFinding(err, foundRecords) { + if (err) { return done(err); } + + + var itemsToArchive = []; + _.each(foundRecords, function(record){ + itemsToArchive.push({ + ofModel: WLModel.identity, + ofRecord: record }); - } catch (e) { return done(e); } - - - // ┬┌─┐ ╔═╗╔═╗╔═╗╔═╗╔═╗╔╦╗╔═╗ ┌─┐┌┐┌┌─┐┌┐ ┬ ┌─┐┌┬┐ ┌┬┐┬ ┬┌─┐┌┐┌ - // │├┤ ║ ╠═╣╚═╗║ ╠═╣ ║║║╣ ├┤ │││├─┤├┴┐│ ├┤ ││ │ ├─┤├┤ │││ - // ┴└ ╚═╝╩ ╩╚═╝╚═╝╩ ╩═╩╝╚═╝ └─┘┘└┘┴ ┴└─┘┴─┘└─┘─┴┘┘ ┴ ┴ ┴└─┘┘└┘ - // ┌─┐┬┌┐┌┌┬┐ ╦╔╦╗╔═╗ ┌┬┐┌─┐ ┌┬┐┌─┐┌─┐┌┬┐┬─┐┌─┐┬ ┬ - // ├┤ ││││ ││ ║ ║║╚═╗ │ │ │ ││├┤ └─┐ │ ├┬┘│ │└┬┘ - // └ ┴┘└┘─┴┘ ╩═╩╝╚═╝ ┴ └─┘ ─┴┘└─┘└─┘ ┴ ┴└─└─┘ ┴ - (function _maybeFindIdsToDestroy(proceed) { - - // If `cascade` meta key is NOT enabled, then just proceed. - if (!query.meta || !query.meta.cascade) { - return proceed(); - } - - // Look up the ids of records that will be destroyed. - // (We need these because, later, since `cascade` is enabled, we'll need - // to empty out all of their associated collections.) - // - // > FUTURE: instead of doing this, consider forcing `fetch: true` in the - // > implementation of `.destroy()` when `cascade` meta key is enabled (mainly - // > for consistency w/ the approach used in createEach()/create()) - - // To do this, we'll grab the appropriate adapter method and call it with a stage 3 - // "find" query, using almost exactly the same QKs as in the incoming "destroy". - // The only tangible difference is that its criteria has a `select` clause so that - // records only contain the primary key field (by column name, of course.) - var pkColumnName = WLModel.schema[WLModel.primaryKey].columnName; - if (!pkColumnName) { - return done(new Error('Consistency violation: model `' + WLModel.identity + '` schema has no primary key column name!')); - } - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // > Note: We have to look up the column name this way (instead of simply using the - // > getAttribute() utility) because it is currently only fully normalized on the - // > `schema` dictionary-- the model's attributes don't necessarily have valid, - // > normalized column names. For more context, see: - // > https://github.com/balderdashy/waterline/commit/19889b7ee265e9850657ec2b4c7f3012f213a0ae#commitcomment-20668097 - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - adapter.find(WLModel.datastore, { - method: 'find', - using: query.using, - criteria: { - where: query.criteria.where, - skip: query.criteria.skip, - limit: query.criteria.limit, - sort: query.criteria.sort, - select: [ pkColumnName ] - }, - meta: query.meta //<< this is how we know that the same db connection will be used - }, function _afterPotentiallyFindingIdsToDestroy(err, pRecords) { - if (err) { - err = forgeAdapterError(err, omen, 'find', modelIdentity, orm); - return proceed(err); - } - - // Slurp out just the array of ids (pk values), and send that back. - var ids = _.pluck(pRecords, pkColumnName); - return proceed(undefined, ids); - - });// - - })(function _afterPotentiallyLookingUpRecordsToCascade(err, idsOfRecordsBeingDestroyedMaybe) { + });//∞ + + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬─┐┌─┐┌─┐┌┬┐┌─┐┌─┐┌─┐┌─┐┬ ┬ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ + // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ │ ├┬┘├┤ ├─┤ │ ├┤ ├┤ ├─┤│ ├─┤ │─┼┐│ │├┤ ├┬┘└┬┘ + // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ └─┘┴└─└─┘┴ ┴ ┴ └─┘└─┘┴ ┴└─┘┴ ┴ └─┘└└─┘└─┘┴└─ ┴ + Archived.createEach(itemsToArchive, function _afterCreatingEach(err) { if (err) { return done(err); } + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌┬┐┌─┐┌─┐┌┬┐┬─┐┌─┐┬ ┬ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ + // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ││├┤ └─┐ │ ├┬┘│ │└┬┘ │─┼┐│ │├┤ ├┬┘└┬┘ + // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ ─┴┘└─┘└─┘ ┴ ┴└─└─┘ ┴ └─┘└└─┘└─┘┴└─ ┴ + WLModel.destroy(query.criteria, function _afterDestroying(err) { + if (err) { return done(err); } + + return done(); + + }, query.meta);// + }, query.meta);// + }, query.meta);// - // Now we'll actually perform the `destroy`. - - // ┌─┐┌─┐┌┐┌┌┬┐ ┌┬┐┌─┐ ╔═╗╔╦╗╔═╗╔═╗╔╦╗╔═╗╦═╗ - // └─┐├┤ │││ ││ │ │ │ ╠═╣ ║║╠═╣╠═╝ ║ ║╣ ╠╦╝ - // └─┘└─┘┘└┘─┴┘ ┴ └─┘ ╩ ╩═╩╝╩ ╩╩ ╩ ╚═╝╩╚═ - // Call the `destroy` adapter method. - adapter.destroy(WLModel.datastore, query, function _afterTalkingToAdapter(err, rawAdapterResult) { - if (err) { - err = forgeAdapterError(err, omen, 'destroy', modelIdentity, orm); - return done(err); - }//-• - - - // ╦═╗╔═╗╦╔╗╔ ╔╦╗╔═╗╦ ╦╔╗╔ ╔╦╗╔═╗╔═╗╔╦╗╦═╗╦ ╦╔═╗╔╦╗╦╔═╗╔╗╔ ┌─┐┌┐┌┌┬┐┌─┐ - // ╠╦╝╠═╣║║║║ ║║║ ║║║║║║║ ║║║╣ ╚═╗ ║ ╠╦╝║ ║║ ║ ║║ ║║║║ │ ││││ │ │ │ - // ╩╚═╩ ╩╩╝╚╝ ═╩╝╚═╝╚╩╝╝╚╝ ═╩╝╚═╝╚═╝ ╩ ╩╚═╚═╝╚═╝ ╩ ╩╚═╝╝╚╝ └─┘┘└┘ ┴ └─┘ - // ┌─┐┌─┐┌─┐┌─┐┌─┐┬┌─┐┌┬┐┬┌─┐┌┐┌┌─┐ ┌─ ┬ ┌─┐ ┌─┐┌─┐┌─┐┌─┐┌─┐┌┬┐┌─┐ ─┐ - // ├─┤└─┐└─┐│ ││ │├─┤ │ ││ ││││└─┐ │ │ ├┤ │ ├─┤└─┐│ ├─┤ ││├┤ │ - // ┴ ┴└─┘└─┘└─┘└─┘┴┴ ┴ ┴ ┴└─┘┘└┘└─┘ └─ ┴o└─┘o └─┘┴ ┴└─┘└─┘┴ ┴─┴┘└─┘ ─┘ - (function _maybeWipeAssociatedCollections(proceed) { - - // If `cascade` meta key is NOT enabled, then just proceed. - if (!query.meta || !query.meta.cascade) { - return proceed(); - } - - // Otherwise, then we should have the records we looked up before. - // (Here we do a quick sanity check.) - if (!_.isArray(idsOfRecordsBeingDestroyedMaybe)) { - return proceed(new Error('Consistency violation: Should have an array of records looked up before! But instead, got: '+util.inspect(idsOfRecordsBeingDestroyedMaybe, {depth: 5})+'')); - } - - // --• - // Now we'll clear out collections belonging to the specified records. - // (i.e. use `replaceCollection` to wipe them all out to be `[]`) - - - // First, if there are no target records, then gracefully bail without complaint. - // (i.e. this is a no-op) - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // FUTURE: Revisit this and verify that it's unnecessary. While this isn't a bad micro-optimization, - // its existence makes it seem like this wouldn't work or would cause a warning or something. And it - // really shouldn't be necessary. (It's doubtful that it adds any real tangible performance benefit anyway.) - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if (idsOfRecordsBeingDestroyedMaybe.length === 0) { - return proceed(); - }//-• - - // Otherwise, we have work to do. - // - // Run .replaceCollection() for each associated collection of the targets, wiping them all out. - // (if n..m, this destroys junction records; otherwise, it's n..1, so this just nulls out the other side) - // - // > Note that we pass through `meta` here, ensuring that the same db connection is used, if possible. - async.each(_.keys(WLModel.attributes), function _eachAttribute(attrName, next) { - - var attrDef = WLModel.attributes[attrName]; - - // Skip everything other than collection attributes. - if (!attrDef.collection){ return next(); } - - // But otherwise, this is a collection attribute. So wipe it. - WLModel.replaceCollection(idsOfRecordsBeingDestroyedMaybe, attrName, [], function (err) { - if (err) { return next(err); } - - return next(); - - }, query.meta);// - - },// ~∞%° - function _afterwards(err) { - if (err) { return proceed(err); } - - return proceed(); - - });// - - })(function _afterPotentiallyWipingCollections(err) {// ~∞%° - if (err) { - return done(err); - } - - // ╔╦╗╦═╗╔═╗╔╗╔╔═╗╔═╗╔═╗╦═╗╔╦╗ ┬─┐┌─┐┌─┐┌─┐┬─┐┌┬┐┌─┐ ┌┐ ┬ ┬┌┬┐ ┌─┐┌┐┌┬ ┬ ┬ ┬┌─┐ - // ║ ╠╦╝╠═╣║║║╚═╗╠╣ ║ ║╠╦╝║║║ ├┬┘├┤ │ │ │├┬┘ ││└─┐ ├┴┐│ │ │ │ │││││ └┬┘ │├┤ - // ╩ ╩╚═╩ ╩╝╚╝╚═╝╚ ╚═╝╩╚═╩ ╩ ┴└─└─┘└─┘└─┘┴└──┴┘└─┘ooo└─┘└─┘ ┴ └─┘┘└┘┴─┘┴ ┴└ - // ╔═╗╔═╗╔╦╗╔═╗╦ ╦ ┌┬┐┌─┐┌┬┐┌─┐ ┬┌─┌─┐┬ ┬ ┬ ┬┌─┐┌─┐ ┌─┐┌─┐┌┬┐ ┌┬┐┌─┐ ┌┬┐┬─┐┬ ┬┌─┐ - // ╠╣ ║╣ ║ ║ ╠═╣ │││├┤ │ ├─┤ ├┴┐├┤ └┬┘ │││├─┤└─┐ └─┐├┤ │ │ │ │ │ ├┬┘│ │├┤ - // ╚ ╚═╝ ╩ ╚═╝╩ ╩ ┴ ┴└─┘ ┴ ┴ ┴ ┴ ┴└─┘ ┴ └┴┘┴ ┴└─┘ └─┘└─┘ ┴ ┴ └─┘ ┴ ┴└─└─┘└─┘ - (function _maybeTransformRecords(proceed){ - - // If `fetch` was not enabled, return. - if (!_.has(query.meta, 'fetch') || query.meta.fetch === false) { - - // > Note: This `if` statement is a convenience, for cases where the result from - // > the adapter may have been coerced from `undefined` to `null` automatically. - // > (we want it to be `undefined` still, for consistency) - if (_.isNull(rawAdapterResult)) { - return proceed(); - }//-• - - if (!_.isUndefined(rawAdapterResult)) { - console.warn('\n'+ - 'Warning: Unexpected behavior in database adapter:\n'+ - 'Since `fetch` is NOT enabled, this adapter (for datastore `'+WLModel.datastore+'`)\n'+ - 'should NOT have sent back anything as the 2nd argument when triggering the callback\n'+ - 'from its `destroy` method. But it did!\n'+ - '\n'+ - '(Displaying this warning to help avoid confusion and draw attention to the bug.\n'+ - 'Specifically, got:\n'+ - util.inspect(rawAdapterResult, {depth:5})+'\n'+ - '(Ignoring it and proceeding anyway...)'+'\n' - ); - }//>- - - // Continue on. - return proceed(); - - }//-• - - // IWMIH then we know that `fetch: true` meta key was set, and so the - // adapter should have sent back an array. - - // Verify that the raw result from the adapter is an array. - if (!_.isArray(rawAdapterResult)) { - return proceed(new Error( - 'Unexpected behavior in database adapter: Since `fetch: true` was enabled, this adapter '+ - '(for datastore `'+WLModel.datastore+'`) should have sent back an array of records as the 2nd argument when triggering '+ - 'the callback from its `archive` method. But instead, got: '+util.inspect(rawAdapterResult, {depth:5})+'' - )); - }//-• - - // Attempt to convert the column names in each record back into attribute names. - var transformedRecords; - try { - transformedRecords = rawAdapterResult.map(function(record) { - return WLModel._transformer.unserialize(record); - }); - } catch (e) { return proceed(e); } - - // Check the records to verify compliance with the adapter spec, - // as well as any issues related to stale data that might not have been - // been migrated to keep up with the logical schema (`type`, etc. in - // attribute definitions). - try { - processAllRecords(transformedRecords, query.meta, modelIdentity, orm); - } catch (e) { return proceed(e); } - - // Now continue on. - return proceed(undefined, transformedRecords); - - })(function (err, transformedRecordsMaybe){ - if (err) { return done(err); } - - // ╔═╗╔═╗╔╦╗╔═╗╦═╗ ┬ ┬┌─┐┌─┐┌─┐┬ ┬┌─┐┬ ┌─┐ ┌─┐┌─┐┬ ┬ ┌┐ ┌─┐┌─┐┬┌─ - // ╠═╣╠╣ ║ ║╣ ╠╦╝ │ │├┤ ├┤ │ └┬┘│ │ ├┤ │ ├─┤│ │ ├┴┐├─┤│ ├┴┐ - // ╩ ╩╚ ╩ ╚═╝╩╚═ ┴─┘┴└ └─┘└─┘ ┴ └─┘┴─┘└─┘ └─┘┴ ┴┴─┘┴─┘└─┘┴ ┴└─┘┴ ┴ - (function (proceed){ - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // FUTURE: `afterArchive` lifecycle callback? - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return proceed(); - })(function(err){ - if (err){ return done(err); } - return done(undefined, transformedRecordsMaybe); - });//_∏_ (†: "after" LC) - });//_∏_ (†: after determining (and potentially transforming) the result from the adapter) - });//_∏_ (†: _afterPotentiallyWipingCollections) - });//_∏_ (adapter.destroy) - }); //_∏_ (†: after potentially looking up records to cascade) - }); //_∏_ (†: "before" LC) }, diff --git a/lib/waterline/methods/destroy.js b/lib/waterline/methods/destroy.js index 50af930e1..f4f2e3015 100644 --- a/lib/waterline/methods/destroy.js +++ b/lib/waterline/methods/destroy.js @@ -531,12 +531,12 @@ module.exports = function destroy(/* criteria, explicitCbMaybe, metaContainer */ } return done(undefined, transformedRecordsMaybe); - });// - });// - }); // - }); // - }); // - }); // + });//_∏_ (†: async.each() -- ran "after" lifecycle callback on each record) + });//_∏_ (†: after determining (and potentially transforming) the result from the adapter) + });//_∏_ (†: _afterPotentiallyWipingCollections) + });//_∏_ (adapter.destroy) + }); //_∏_ (†: after potentially looking up records to cascade) + }); //_∏_ (†: "before" LC) }, diff --git a/lib/waterline/utils/query/forge-stage-two-query.js b/lib/waterline/utils/query/forge-stage-two-query.js index 33038c4e1..bda8d76bc 100644 --- a/lib/waterline/utils/query/forge-stage-two-query.js +++ b/lib/waterline/utils/query/forge-stage-two-query.js @@ -330,6 +330,8 @@ module.exports = function forgeStageTwoQuery(query, orm) { case 'update': return [ 'criteria', 'valuesToSet' ]; case 'destroy': return [ 'criteria' ]; + case 'archive': return [ 'criteria' ]; + case 'addToCollection': return [ 'targetRecordIds', 'collectionAttrName', 'associatedIds' ]; case 'removeFromCollection': return [ 'targetRecordIds', 'collectionAttrName', 'associatedIds' ]; case 'replaceCollection': return [ 'targetRecordIds', 'collectionAttrName', 'associatedIds' ]; @@ -446,9 +448,9 @@ module.exports = function forgeStageTwoQuery(query, orm) { }//>-• // If the criteria is not defined, then in most cases, we treat it like `{}`. - // BUT if this query will be running as a result of an `update()` or a `destroy()`, - // then we'll be a bit more picky in order to prevent accidents. - if (_.isUndefined(query.criteria) && (query.method === 'update' || query.method === 'destroy')) { + // BUT if this query will be running as a result of an `update()`, or a `destroy()`, + // or an `.archive()`, then we'll be a bit more picky in order to prevent accidents. + if (_.isUndefined(query.criteria) && (query.method === 'update' || query.method === 'destroy' || query.method === 'archive')) { throw buildUsageError('E_INVALID_CRITERIA', 'Cannot use this method (`'+query.method+'`) with a criteria of `undefined`. (This is just a simple failsafe to help protect your data: if you really want to '+query.method+' ALL records, no problem-- please just be explicit and provide a criteria of `{}`.)', query.using); diff --git a/lib/waterline/utils/query/get-query-modifier-methods.js b/lib/waterline/utils/query/get-query-modifier-methods.js index 0f12d0e55..7e24cc029 100644 --- a/lib/waterline/utils/query/get-query-modifier-methods.js +++ b/lib/waterline/utils/query/get-query-modifier-methods.js @@ -657,6 +657,8 @@ module.exports = function getQueryModifierMethods(category){ case 'update': _.extend(queryMethods, FILTER_Q_METHODS, SET_Q_METHODS); break; case 'destroy': _.extend(queryMethods, FILTER_Q_METHODS); break; + case 'archive': _.extend(queryMethods, FILTER_Q_METHODS); break; + case 'addToCollection': _.extend(queryMethods, COLLECTION_Q_METHODS); break; case 'removeFromCollection': _.extend(queryMethods, COLLECTION_Q_METHODS); break; case 'replaceCollection': _.extend(queryMethods, COLLECTION_Q_METHODS); break; From 805dece84f5e62797280c7611e3d4ed29098cdea Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 4 Sep 2017 23:31:24 -0500 Subject: [PATCH 03/12] Add note about streaming optimization. --- lib/waterline/methods/archive.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index 902c26fcc..fe9b9fc0e 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -212,6 +212,12 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // FUTURE: pass through the `omen` in the metadata. // - - - - - - - - - - - - - - - - - - - - - - - - - - - + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: as an optimization, fetch records batch-at-a-time + // using .stream(). (This would allow you to potentially archive + // millions of records at a time without overflowing RAM.) + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬┌┐┌┌┬┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ├┤ ││││ ││ │─┼┐│ │├┤ ├┬┘└┬┘ // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ └ ┴┘└┘─┴┘ └─┘└└─┘└─┘┴└─ ┴ @@ -221,7 +227,6 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ WLModel.find(query.criteria, function _afterFinding(err, foundRecords) { if (err) { return done(err); } - var itemsToArchive = []; _.each(foundRecords, function(record){ itemsToArchive.push({ From aa18562e27699b51d369d9f51b748e0d06bca2b3 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 4 Sep 2017 23:41:25 -0500 Subject: [PATCH 04/12] Change attr names for built-in Archived model for clarity, and add some additional notes for future reference. --- lib/waterline/methods/archive.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index fe9b9fc0e..044761d8d 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -205,6 +205,12 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // TODO: automigrate the built-in Archived model // // TODO: inject the built-in Archived model into the ORM's ontology + // • id (pk-- string or number, depending on where the Archived model is being stored) + // • createdAt (timestamp-- this is effectively ≈ "archivedAt") + // • originalRecord (json-- the original record, completely unpopulated) + // • fromModel (string-- the original model identity) + // + // > Note there's no updatedAt! // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -212,6 +218,12 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // FUTURE: pass through the `omen` in the metadata. // - - - - - - - - - - - - - - - - - - - - - - - - - - - + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: Maybe refactor this into more-generic `.move()` and/or + // `.copy()` methods for migrating data between models/datastores. + // Then just leverage those methods here in `.archive()`. + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // FUTURE: as an optimization, fetch records batch-at-a-time // using .stream(). (This would allow you to potentially archive @@ -230,8 +242,8 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ var itemsToArchive = []; _.each(foundRecords, function(record){ itemsToArchive.push({ - ofModel: WLModel.identity, - ofRecord: record + originalRecord: record, + fromModel: WLModel.identity, }); });//∞ From 5f01593b68c79656a91b1d31cdf1a1aa05e09a2f Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 4 Sep 2017 23:58:36 -0500 Subject: [PATCH 05/12] Added first pass at Archive model lookup. --- lib/waterline/methods/archive.js | 83 +++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index 044761d8d..47510a04e 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -188,24 +188,72 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ } }//fi + // Bail now if archival has been disabled. + if (!WLModel.Archive) { + return done(flaverr({ + name: 'UsageError', + message: 'Since the `Archive` setting was explicitly disabled, .archive() cannot be used.' + }, omen)); + }//• + + // Look up the Archive model. + var Archive = WLModel.Archive; + if (Archive.identity && Archive.identity !== '_archive') { + return done(new Error('Consistency violation: Cannot override the `identity` of the built-in Archive model! (expecting "_archive", but instead got "'+Archive.identity+'")')); + } - // Look up the Archived model. - var Archived;// TODO + try { + Archive = getModel('_archive', orm); + } + catch (err) { + switch (err.code) { + case 'E_MODEL_NOT_REGISTERED': + return done(new Error('Since the `Archive` setting was explicitly disabled, .archive() ')); + // TODO: check to see if this was deliberately disabled + return done(err); + default: return done(err); + } + }//fi // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // TODO: provide a way of choosing which datastore is "dominant" - // as far as the Archived model is concerned (e.g. where does it live). + // as far as the Archive model is concerned (e.g. where does it live). + // { + // …(in top-level orm settings)… + // Archive: { datastore: 'foo' } + // } // // TODO: provide a way of choosing the `tableName` and `columnName`s - // for the Archived model + // for the Archive model + // { + // …(in top-level orm settings)… + // Archive: { + // tableName: 'foo', + // attributes: { + // originalRecord: { type: 'json', columnName: 'barbaz' }, + // fromModel: { type: 'string', columnName: 'bingbong' } + // } + // } + // } // - // TODO: provide a way of disabling the Archived model (and thus + // TODO: provide a way of disabling the Archive model (and thus // disabling support for the `.archive()` model method) + // { + // …(in top-level orm settings)… + // Archive: false + // } // - // TODO: automigrate the built-in Archived model - // - // TODO: inject the built-in Archived model into the ORM's ontology - // • id (pk-- string or number, depending on where the Archived model is being stored) + // TODO: prevent overriding the Archive model's `identity`, which + // is immutable & private, for simplicity's sake + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // TODO: automigrate the built-in Archive model + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // TODO: inject the built-in Archive model into the ORM's ontology + // • id (pk-- string or number, depending on where the Archive model is being stored) // • createdAt (timestamp-- this is effectively ≈ "archivedAt") // • originalRecord (json-- the original record, completely unpopulated) // • fromModel (string-- the original model identity) @@ -224,11 +272,6 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // Then just leverage those methods here in `.archive()`. // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // FUTURE: as an optimization, fetch records batch-at-a-time - // using .stream(). (This would allow you to potentially archive - // millions of records at a time without overflowing RAM.) - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬┌┐┌┌┬┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ├┤ ││││ ││ │─┼┐│ │├┤ ├┬┘└┬┘ @@ -239,9 +282,15 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ WLModel.find(query.criteria, function _afterFinding(err, foundRecords) { if (err) { return done(err); } - var itemsToArchive = []; + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // FUTURE: as an optimization, fetch records batch-at-a-time + // using .stream() instead of just doing a naïve `.find()`. + // (This would allow you to potentially archive millions of records + // at a time without overflowing RAM.) + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + var archives = []; _.each(foundRecords, function(record){ - itemsToArchive.push({ + archives.push({ originalRecord: record, fromModel: WLModel.identity, }); @@ -250,7 +299,7 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬─┐┌─┐┌─┐┌┬┐┌─┐┌─┐┌─┐┌─┐┬ ┬ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ │ ├┬┘├┤ ├─┤ │ ├┤ ├┤ ├─┤│ ├─┤ │─┼┐│ │├┤ ├┬┘└┬┘ // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ └─┘┴└─└─┘┴ ┴ ┴ └─┘└─┘┴ ┴└─┘┴ ┴ └─┘└└─┘└─┘┴└─ ┴ - Archived.createEach(itemsToArchive, function _afterCreatingEach(err) { + Archive.createEach(archives, function _afterCreatingEach(err) { if (err) { return done(err); } // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌┬┐┌─┐┌─┐┌┬┐┬─┐┌─┐┬ ┬ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ From c4272ff7b7f3396babdd6118b812a4d6f62ca0a8 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Tue, 5 Sep 2017 00:10:14 -0500 Subject: [PATCH 06/12] Simplify spec and add more descriptive examples. --- lib/waterline/methods/archive.js | 63 ++++++++++++-------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index 47510a04e..2257fb66e 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -6,11 +6,11 @@ var _ = require('@sailshq/lodash'); var flaverr = require('flaverr'); var parley = require('parley'); var buildOmen = require('../utils/query/build-omen'); +var getModel = require('../utils/ontology/get-model'); var forgeStageTwoQuery = require('../utils/query/forge-stage-two-query'); var getQueryModifierMethods = require('../utils/query/get-query-modifier-methods'); var verifyModelMethodContext = require('../utils/query/verify-model-method-context'); - /** * Module constants */ @@ -188,69 +188,49 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ } }//fi - // Bail now if archival has been disabled. - if (!WLModel.Archive) { + // Bail now if archiving has been disabled. + if (!WLModel.archiveModelIdentity) { return done(flaverr({ name: 'UsageError', - message: 'Since the `Archive` setting was explicitly disabled, .archive() cannot be used.' + message: 'Since the `archiveModelIdentity` setting was explicitly disabled, .archive() cannot be used.' }, omen)); }//• // Look up the Archive model. - var Archive = WLModel.Archive; - if (Archive.identity && Archive.identity !== '_archive') { - return done(new Error('Consistency violation: Cannot override the `identity` of the built-in Archive model! (expecting "_archive", but instead got "'+Archive.identity+'")')); - } - + var Archive = WLModel.archiveModelIdentity; try { - Archive = getModel('_archive', orm); - } - catch (err) { - switch (err.code) { - case 'E_MODEL_NOT_REGISTERED': - return done(new Error('Since the `Archive` setting was explicitly disabled, .archive() ')); - // TODO: check to see if this was deliberately disabled - return done(err); - default: return done(err); - } - }//fi + Archive = getModel(WLModel.archiveModelIdentity, orm); + } catch (err) { return done(err); }//fi + // Generally: // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // TODO: provide a way of choosing which datastore is "dominant" // as far as the Archive model is concerned (e.g. where does it live). - // { - // …(in top-level orm settings)… - // Archive: { datastore: 'foo' } - // } + // …in top-level orm settings: + // archiveModelIdentity: 'archive', + // + // …in 'archive' model: + // datastore: 'foo' // // TODO: provide a way of choosing the `tableName` and `columnName`s // for the Archive model - // { - // …(in top-level orm settings)… - // Archive: { + // …in top-level orm settings: + // archiveModelIdentity: 'archive', + // + // …in 'archive' model: // tableName: 'foo', // attributes: { // originalRecord: { type: 'json', columnName: 'barbaz' }, // fromModel: { type: 'string', columnName: 'bingbong' } // } - // } - // } // // TODO: provide a way of disabling the Archive model (and thus // disabling support for the `.archive()` model method) - // { - // …(in top-level orm settings)… - // Archive: false - // } - // - // TODO: prevent overriding the Archive model's `identity`, which - // is immutable & private, for simplicity's sake - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // TODO: automigrate the built-in Archive model + // …in top-level orm settings: + // archiveModelIdentity: false // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // In `../waterline.js`: // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // TODO: inject the built-in Archive model into the ORM's ontology // • id (pk-- string or number, depending on where the Archive model is being stored) @@ -259,6 +239,9 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // • fromModel (string-- the original model identity) // // > Note there's no updatedAt! + // + // TODO: validate the configured model's attributes and so forth, + // to make sure it's actually usable as the orm's Archive. // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 1521931424142923d3d23d8be43b30eb79bccfcf Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Tue, 5 Sep 2017 00:13:57 -0500 Subject: [PATCH 07/12] Expose .archive() --- lib/waterline/MetaModel.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/waterline/MetaModel.js b/lib/waterline/MetaModel.js index bf89260a9..ebb58efce 100644 --- a/lib/waterline/MetaModel.js +++ b/lib/waterline/MetaModel.js @@ -157,6 +157,7 @@ _.extend( createEach: require('./methods/create-each'), update: require('./methods/update'), destroy: require('./methods/destroy'), + archive: require('./methods/archive'), addToCollection: require('./methods/add-to-collection'), removeFromCollection: require('./methods/remove-from-collection'), replaceCollection: require('./methods/replace-collection'), From 67f6987af0af8a773390d1aa0a945a0106fca8a6 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Tue, 5 Sep 2017 00:17:51 -0500 Subject: [PATCH 08/12] Remove limit/skip/sort/select/omit (which were automatically added in the forging from the find query) so that the destroy() query works. Note: At this point, you can take this and use .archive() at the app-level in your own project, provided you hook up your own Archive model. But in subsequent commits, I'll write some code that injects this model automatically, unless configured otherwise. --- lib/waterline/methods/archive.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index 2257fb66e..b6c638012 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -242,6 +242,9 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // // TODO: validate the configured model's attributes and so forth, // to make sure it's actually usable as the orm's Archive. + // + // TODO: if there is an existing archive model with a conflicting + // identity, throw a descriptive error on ORM initialization // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -285,6 +288,15 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ Archive.createEach(archives, function _afterCreatingEach(err) { if (err) { return done(err); } + // Remove the `limit`, `skip`, `sort`, `select`, and `omit` clauses so + // that our `destroy` query is valid. + // (This is because they were automatically attached above in the forging.) + delete query.criteria.limit; + delete query.criteria.skip; + delete query.criteria.sort; + delete query.criteria.select; + delete query.criteria.omit; + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌┬┐┌─┐┌─┐┌┬┐┬─┐┌─┐┬ ┬ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ││├┤ └─┐ │ ├┬┘│ │└┬┘ │─┼┐│ │├┤ ├┬┘└┬┘ // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ ─┴┘└─┘└─┘ ┴ ┴└─└─┘ ┴ └─┘└└─┘└─┘┴└─ ┴ From 92e9373cd1d5497b4d6cf42ebacefc479a0b8f36 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Tue, 5 Sep 2017 00:18:53 -0500 Subject: [PATCH 09/12] One last todo about defaulting the model setting --- lib/waterline/methods/archive.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index b6c638012..c6898a72d 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -232,6 +232,8 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // In `../waterline.js`: // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // TODO: default the `archiveModelIdentity` orm setting to "archive" + // // TODO: inject the built-in Archive model into the ORM's ontology // • id (pk-- string or number, depending on where the Archive model is being stored) // • createdAt (timestamp-- this is effectively ≈ "archivedAt") From 1e00704a6755c5a8d1fba392cf9954c42aaea4c6 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 2 Oct 2017 11:29:06 -0500 Subject: [PATCH 10/12] Merge in the other notes about the naive cloneDeep --- .../query/private/normalize-where-clause.js | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/waterline/utils/query/private/normalize-where-clause.js b/lib/waterline/utils/query/private/normalize-where-clause.js index 1ca3fcaa4..8e64c6336 100644 --- a/lib/waterline/utils/query/private/normalize-where-clause.js +++ b/lib/waterline/utils/query/private/normalize-where-clause.js @@ -85,7 +85,36 @@ module.exports = function normalizeWhereClause(whereClause, modelIdentity, orm, // (This isn't great because it can mess up things like Buffers... which you // shouldn't really be using in a `where` clause anyway, but still-- it makes // it way harder to figure out what's wrong when folks find themselves in that - // situation.) + // situation. It could also affect any weird custom constraints for `type:'ref'` + // attrs. And if the current approach were also used in valuesToSet, newRecord, + // newRecords etc, it would matter even more.) + // + // The full list of query keys that need to be carefully checked: + // • criteria + // • populates + // • newRecord + // • newRecords + // • valuesToSet + // • targetRecordIds + // • associatedIds + // + // The solution will probably mean distributing this deep clone behavior out + // to the various places it's liable to come up. In reality, this will be + // more performant anyway, since we won't be unnecessarily cloning things like + // big JSON values, etc. + // + // The important thing is that this should do shallow clones of deeply-nested + // control structures like top level query key dictionaries, criteria clauses, + // predicates/constraints/modifiers in `where` clauses, etc. + // + // > And remember: Don't deep-clone functions. + // > Note that, weirdly, it's fine to deep-clone dictionaries/arrays + // > that contain nested functions (they just don't get cloned-- they're + // > the same reference). But if you try to deep-clone a function at the + // > top level, it gets messed up. + // > + // > More background on this: https://trello.com/c/VLXPvyN5 + // > (Note that this behavior maintains backwards compatibility with Waterline <=0.12.) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - }//fi From 329f27736efe96d021fdd041f8011e5b6d68c5f5 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 2 Oct 2017 11:50:01 -0500 Subject: [PATCH 11/12] Add clarification about what mutateArgs is and isn't. --- lib/waterline/utils/query/forge-stage-two-query.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/waterline/utils/query/forge-stage-two-query.js b/lib/waterline/utils/query/forge-stage-two-query.js index 2ee7e145c..f5683fdd9 100644 --- a/lib/waterline/utils/query/forge-stage-two-query.js +++ b/lib/waterline/utils/query/forge-stage-two-query.js @@ -413,6 +413,12 @@ module.exports = function forgeStageTwoQuery(query, orm) { // ┌┬┐┬ ┬┌┬┐┌─┐┌┬┐┌─┐ ┌─┐┬─┐┌─┐┌─┐ // ││││ │ │ ├─┤ │ ├┤ ├─┤├┬┘│ ┬└─┐ // ┴ ┴└─┘ ┴ ┴ ┴ ┴ └─┘ ┴ ┴┴└─└─┘└─┘ + // + // EXPERIMENTAL: The `mutateArgs` meta key enabled optimizations by preventing + // unnecessary cloning of arguments. + // + // > Note that this is ONLY respected at the stage 2 level! + // > That is, it doesn't matter if this meta key is set or not when you call adapters. if (query.meta.mutateArgs !== undefined) { if (!_.isBoolean(query.meta.mutateArgs)) { From 165559debbc2645f77e68054db35620a7bb8c02a Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 2 Oct 2017 11:50:37 -0500 Subject: [PATCH 12/12] Initial pass at using mutateArgs internally. Needs benchmarks proving it's actually faster, otherwise no reason to include any of this extra complexity. --- lib/waterline/methods/add-to-collection.js | 5 +++-- lib/waterline/methods/archive.js | 11 ++++++++--- lib/waterline/methods/create-each.js | 7 ++++++- lib/waterline/methods/create.js | 8 +++++++- lib/waterline/methods/destroy.js | 7 ++++++- lib/waterline/methods/find-or-create.js | 9 +++++++-- lib/waterline/methods/remove-from-collection.js | 5 +++-- lib/waterline/methods/replace-collection.js | 5 +++-- lib/waterline/methods/stream.js | 7 ++++++- lib/waterline/methods/update.js | 8 +++++++- 10 files changed, 56 insertions(+), 16 deletions(-) diff --git a/lib/waterline/methods/add-to-collection.js b/lib/waterline/methods/add-to-collection.js index e9f0a9c41..ced096d5f 100644 --- a/lib/waterline/methods/add-to-collection.js +++ b/lib/waterline/methods/add-to-collection.js @@ -296,9 +296,10 @@ module.exports = function addToCollection(/* targetRecordIds, collectionAttrName manyToMany = true; } - // Ensure the query skips lifecycle callbacks // Build a modified shallow clone of the originally-provided `meta` - var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true }); + // To ensure that internal queries skip lifecycle callbacks, and that they don't worry + // about protecting arguments from destructive mutation. + var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true, mutateArgs: true }); // ███╗ ███╗ █████╗ ███╗ ██╗██╗ ██╗ ████████╗ ██████╗ ███╗ ███╗ █████╗ ███╗ ██╗██╗ ██╗ diff --git a/lib/waterline/methods/archive.js b/lib/waterline/methods/archive.js index c6898a72d..c0df476df 100644 --- a/lib/waterline/methods/archive.js +++ b/lib/waterline/methods/archive.js @@ -261,6 +261,11 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // In our `meta` query key for use in these internal queries, enable the + // `mutateArgs` flag to increase performance (by avoiding the unnecessary + // duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬┌┐┌┌┬┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ├┤ ││││ ││ │─┼┐│ │├┤ ├┬┘└┬┘ // ╚═╝╩ ╚═╚═╝╚═╝╚═╝ ╩ ╚═╝ └ ┴┘└┘─┴┘ └─┘└└─┘└─┘┴└─ ┴ @@ -307,9 +312,9 @@ module.exports = function archive(/* criteria, explicitCbMaybe, metaContainer */ return done(); - }, query.meta);// - }, query.meta);// - }, query.meta);// + }, metaForInternalQueries);// + }, metaForInternalQueries);// + }, metaForInternalQueries);// }, diff --git a/lib/waterline/methods/create-each.js b/lib/waterline/methods/create-each.js index 2b0d2de28..e1cccc112 100644 --- a/lib/waterline/methods/create-each.js +++ b/lib/waterline/methods/create-each.js @@ -397,6 +397,11 @@ module.exports = function createEach( /* newRecords?, explicitCbMaybe?, meta? */ });// });// + // In our `meta` query key for use in these internal queries, enable the + // `mutateArgs` flag to increase performance (by avoiding the unnecessary + // duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + async.each(argsForEachReplaceOp, function _eachReplaceCollectionOp(argsForReplace, next) { // Note that, by using the same `meta`, we use same db connection @@ -404,7 +409,7 @@ module.exports = function createEach( /* newRecords?, explicitCbMaybe?, meta? */ WLModel.replaceCollection(argsForReplace[0], argsForReplace[1], argsForReplace[2], function(err) { if (err) { return next(err); } return next(); - }, query.meta); + }, metaForInternalQueries); },// ~∞%° function _afterReplacingAllCollections(err) { diff --git a/lib/waterline/methods/create.js b/lib/waterline/methods/create.js index 1f787aeef..92cf48552 100644 --- a/lib/waterline/methods/create.js +++ b/lib/waterline/methods/create.js @@ -331,12 +331,18 @@ module.exports = function create(newRecord, explicitCbMaybe, metaContainer) { // ├┤ ┌┴┬┘├─┘│ ││ │ │ │ └┬┘───└─┐├─┘├┤ │ │├┤ │├┤ ││ ├─┤└─┐└─┐│ ││ │├─┤ │ ││ ││││└─┐ // └─┘┴ └─┴ ┴─┘┴└─┘┴ ┴ ┴─┘┴ └─┘┴ └─┘└─┘┴└ ┴└─┘─┴┘ ┴ ┴└─┘└─┘└─┘└─┘┴┴ ┴ ┴ ┴└─┘┘└┘└─┘ var targetId = transformedRecord[WLModel.primaryKey]; + + // In our `meta` query key for use in these internal queries, enable the + // `mutateArgs` flag to increase performance (by avoiding the unnecessary + // duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + async.each(_.keys(collectionResets), function _eachReplaceCollectionOp(collectionAttrName, next) { WLModel.replaceCollection(targetId, collectionAttrName, collectionResets[collectionAttrName], function(err){ if (err) { return next(err); } return next(); - }, query.meta); + }, metaForInternalQueries); },// ~∞%° function _afterReplacingAllCollections(err) { diff --git a/lib/waterline/methods/destroy.js b/lib/waterline/methods/destroy.js index 31c0c2805..108cf93da 100644 --- a/lib/waterline/methods/destroy.js +++ b/lib/waterline/methods/destroy.js @@ -395,6 +395,11 @@ module.exports = function destroy(/* criteria, explicitCbMaybe, metaContainer */ // (if n..m, this destroys junction records; otherwise, it's n..1, so this just nulls out the other side) // // > Note that we pass through `meta` here, ensuring that the same db connection is used, if possible. + // > But in our `meta` query key for use in these internal queries, we enable the + // > `mutateArgs` flag to increase performance (by avoiding the unnecessary + // > duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + async.each(_.keys(WLModel.attributes), function _eachAttribute(attrName, next) { var attrDef = WLModel.attributes[attrName]; @@ -424,7 +429,7 @@ module.exports = function destroy(/* criteria, explicitCbMaybe, metaContainer */ return next(); - }, query.meta);// + }, metaForInternalQueries);// },// ~∞%° function _afterwards(err) { diff --git a/lib/waterline/methods/find-or-create.js b/lib/waterline/methods/find-or-create.js index 22564b087..c80f2f360 100644 --- a/lib/waterline/methods/find-or-create.js +++ b/lib/waterline/methods/find-or-create.js @@ -208,6 +208,11 @@ module.exports = function findOrCreate( /* criteria?, newRecord?, explicitCbMayb delete query.criteria.skip; delete query.criteria.sort; + // In our `meta` query key for use in these internal queries, enable the + // `mutateArgs` flag to increase performance (by avoiding the unnecessary + // duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬┌┐┌┌┬┐ ┌─┐┌┐┌┌─┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ ├┤ ││││ ││ │ ││││├┤ │─┼┐│ │├┤ ├┬┘└┬┘ @@ -240,7 +245,7 @@ module.exports = function findOrCreate( /* criteria?, newRecord?, explicitCbMayb // Build a modified shallow clone of the originally-provided `meta` // that also has `fetch: true`. - var modifiedMeta = _.extend({}, query.meta || {}, { fetch: true }); + var modifiedMeta = _.extend({}, metaForInternalQueries, { fetch: true }); // ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗ ┌─┐┬─┐┌─┐┌─┐┌┬┐┌─┐ ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬ // ║╣ ╔╩╦╝║╣ ║ ║ ║ ║ ║╣ │ ├┬┘├┤ ├─┤ │ ├┤ │─┼┐│ │├┤ ├┬┘└┬┘ @@ -261,7 +266,7 @@ module.exports = function findOrCreate( /* criteria?, newRecord?, explicitCbMayb return done(undefined, createdRecord, true); }, modifiedMeta);// - }, query.meta);// + }, metaForInternalQueries);// }, diff --git a/lib/waterline/methods/remove-from-collection.js b/lib/waterline/methods/remove-from-collection.js index e1c9b47b4..801a0ebf3 100644 --- a/lib/waterline/methods/remove-from-collection.js +++ b/lib/waterline/methods/remove-from-collection.js @@ -299,9 +299,10 @@ module.exports = function removeFromCollection(/* targetRecordIds?, collectionAt manyToMany = true; } - // Ensure the query skips lifecycle callbacks // Build a modified shallow clone of the originally-provided `meta` - var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true }); + // To ensure that internal queries skip lifecycle callbacks, and that they don't worry + // about protecting arguments from destructive mutation. + var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true, mutateArgs: true }); // ██╗███╗ ██╗ ███╗ ███╗██╗ diff --git a/lib/waterline/methods/replace-collection.js b/lib/waterline/methods/replace-collection.js index debee622c..69838584d 100644 --- a/lib/waterline/methods/replace-collection.js +++ b/lib/waterline/methods/replace-collection.js @@ -297,9 +297,10 @@ module.exports = function replaceCollection(/* targetRecordIds?, collectionAttrN } - // Ensure the query skips lifecycle callbacks // Build a modified shallow clone of the originally-provided `meta` - var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true }); + // To ensure that internal queries skip lifecycle callbacks, and that they don't worry + // about protecting arguments from destructive mutation. + var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true, mutateArgs: true }); diff --git a/lib/waterline/methods/stream.js b/lib/waterline/methods/stream.js index 852afeaa9..7a6154d60 100644 --- a/lib/waterline/methods/stream.js +++ b/lib/waterline/methods/stream.js @@ -261,6 +261,11 @@ module.exports = function stream( /* criteria?, eachRecordFn?, explicitCbMaybe?, var i = 0; + // In our `meta` query key for use in these internal queries, enable the + // `mutateArgs` flag to increase performance (by avoiding the unnecessary + // duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + async.whilst(function _checkHasntReachedLastBatchYet(){ if (!reachedLastBatch) { return true; } else { return false; } @@ -318,7 +323,7 @@ module.exports = function stream( /* criteria?, eachRecordFn?, explicitCbMaybe?, // Pass through `meta` so we're sure to use the same db connection // and settings (i.e. esp. relevant if we happen to be inside a transaction) - deferredForThisBatch.meta(query.meta); + deferredForThisBatch.meta(metaForInternalQueries); deferredForThisBatch.exec(function (err, batchOfRecords){ if (err) { return next(err); } diff --git a/lib/waterline/methods/update.js b/lib/waterline/methods/update.js index 254b3e477..b4118a233 100644 --- a/lib/waterline/methods/update.js +++ b/lib/waterline/methods/update.js @@ -382,12 +382,18 @@ module.exports = function update(criteria, valuesToSet, explicitCbMaybe, metaCon // ├┤ ┌┴┬┘├─┘│ ││ │ │ │ └┬┘───└─┐├─┘├┤ │ │├┤ │├┤ ││ ├─┤└─┐└─┐│ ││ │├─┤ │ ││ ││││└─┐ // └─┘┴ └─┴ ┴─┘┴└─┘┴ ┴ ┴─┘┴ └─┘┴ └─┘└─┘┴└ ┴└─┘─┴┘ ┴ ┴└─┘└─┘└─┘└─┘┴┴ ┴ ┴ ┴└─┘┘└┘└─┘ var targetIds = _.pluck(transformedRecords, WLModel.primaryKey); + + // In our `meta` query key for use in these internal queries, enable the + // `mutateArgs` flag to increase performance (by avoiding the unnecessary + // duplication of objects and allowing arguments to be destructively consumed). + var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true }); + async.each(_.keys(collectionResets), function _eachReplaceCollectionOp(collectionAttrName, next) { WLModel.replaceCollection(targetIds, collectionAttrName, collectionResets[collectionAttrName], function(err){ if (err) { return next(err); } return next(); - }, query.meta); + }, metaForInternalQueries); },// ~∞%° function _afterReplacingAllCollections(err) {